diff --git a/extensions/browser/index.test.ts b/extensions/browser/index.test.ts new file mode 100644 index 00000000000..0e857236281 --- /dev/null +++ b/extensions/browser/index.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from "vitest"; +import { createTestPluginApi } from "../../test/helpers/extensions/plugin-api.js"; +import type { OpenClawPluginApi } from "./runtime-api.js"; + +const runtimeApiMocks = vi.hoisted(() => ({ + createBrowserPluginService: vi.fn(() => ({ id: "browser-control", start: vi.fn() })), + createBrowserTool: vi.fn(() => ({ + name: "browser", + description: "browser", + parameters: { type: "object", properties: {} }, + execute: vi.fn(), + })), + handleBrowserGatewayRequest: vi.fn(), + registerBrowserCli: vi.fn(), +})); + +vi.mock("./runtime-api.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createBrowserPluginService: runtimeApiMocks.createBrowserPluginService, + createBrowserTool: runtimeApiMocks.createBrowserTool, + handleBrowserGatewayRequest: runtimeApiMocks.handleBrowserGatewayRequest, + registerBrowserCli: runtimeApiMocks.registerBrowserCli, + }; +}); + +import browserPlugin from "./index.js"; + +function createApi() { + const registerCli = vi.fn(); + const registerGatewayMethod = vi.fn(); + const registerService = vi.fn(); + const registerTool = vi.fn(); + const api = createTestPluginApi({ + id: "browser", + name: "Browser", + source: "test", + config: {}, + runtime: {} as OpenClawPluginApi["runtime"], + registerCli, + registerGatewayMethod, + registerService, + registerTool, + }) as OpenClawPluginApi; + return { api, registerCli, registerGatewayMethod, registerService, registerTool }; +} + +describe("browser plugin", () => { + it("registers browser tool, cli, gateway method, and service ownership", () => { + const { api, registerCli, registerGatewayMethod, registerService, registerTool } = createApi(); + browserPlugin.register(api); + + expect(registerTool).toHaveBeenCalledTimes(1); + expect(registerCli).toHaveBeenCalledWith(expect.any(Function), { commands: ["browser"] }); + expect(registerGatewayMethod).toHaveBeenCalledWith( + "browser.request", + runtimeApiMocks.handleBrowserGatewayRequest, + { scope: "operator.write" }, + ); + expect(runtimeApiMocks.createBrowserPluginService).toHaveBeenCalledTimes(1); + expect(registerService).toHaveBeenCalledWith( + runtimeApiMocks.createBrowserPluginService.mock.results[0]?.value, + ); + }); + + it("forwards per-session browser options into the tool factory", () => { + const { api, registerTool } = createApi(); + browserPlugin.register(api); + + const tool = registerTool.mock.calls[0]?.[0]; + if (typeof tool !== "function") { + throw new Error("expected browser plugin to register a tool factory"); + } + + tool({ + sessionKey: "agent:main:webchat:direct:123", + browser: { + sandboxBridgeUrl: "http://127.0.0.1:9999", + allowHostControl: true, + }, + }); + + expect(runtimeApiMocks.createBrowserTool).toHaveBeenCalledWith({ + sandboxBridgeUrl: "http://127.0.0.1:9999", + allowHostControl: true, + agentSessionKey: "agent:main:webchat:direct:123", + }); + }); +}); diff --git a/extensions/browser/index.ts b/extensions/browser/index.ts new file mode 100644 index 00000000000..d4c92f6d695 --- /dev/null +++ b/extensions/browser/index.ts @@ -0,0 +1,28 @@ +import { + createBrowserPluginService, + createBrowserTool, + definePluginEntry, + handleBrowserGatewayRequest, + registerBrowserCli, + type OpenClawPluginToolContext, + type OpenClawPluginToolFactory, +} from "./runtime-api.js"; + +export default definePluginEntry({ + id: "browser", + name: "Browser", + description: "Default browser tool plugin", + register(api) { + api.registerTool(((ctx: OpenClawPluginToolContext) => + createBrowserTool({ + sandboxBridgeUrl: ctx.browser?.sandboxBridgeUrl, + allowHostControl: ctx.browser?.allowHostControl, + agentSessionKey: ctx.sessionKey, + })) as OpenClawPluginToolFactory); + api.registerCli(({ program }) => registerBrowserCli(program), { commands: ["browser"] }); + api.registerGatewayMethod("browser.request", handleBrowserGatewayRequest, { + scope: "operator.write", + }); + api.registerService(createBrowserPluginService()); + }, +}); diff --git a/extensions/browser/openclaw.plugin.json b/extensions/browser/openclaw.plugin.json new file mode 100644 index 00000000000..ec319d9fab4 --- /dev/null +++ b/extensions/browser/openclaw.plugin.json @@ -0,0 +1,9 @@ +{ + "id": "browser", + "enabledByDefault": true, + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/extensions/browser/package.json b/extensions/browser/package.json new file mode 100644 index 00000000000..7ac8708ef63 --- /dev/null +++ b/extensions/browser/package.json @@ -0,0 +1,12 @@ +{ + "name": "@openclaw/browser-plugin", + "version": "2026.3.25", + "private": true, + "description": "OpenClaw browser tool plugin", + "type": "module", + "openclaw": { + "extensions": [ + "./index.ts" + ] + } +} diff --git a/extensions/browser/runtime-api.ts b/extensions/browser/runtime-api.ts new file mode 100644 index 00000000000..ca68a9159bc --- /dev/null +++ b/extensions/browser/runtime-api.ts @@ -0,0 +1,10 @@ +export { createBrowserTool } from "./src/browser-tool.js"; +export { registerBrowserCli } from "./src/cli/browser-cli.js"; +export { createBrowserPluginService } from "./src/plugin-service.js"; +export { handleBrowserGatewayRequest } from "./src/gateway/browser-request.js"; +export { + definePluginEntry, + type OpenClawPluginApi, + type OpenClawPluginToolContext, + type OpenClawPluginToolFactory, +} from "openclaw/plugin-sdk/plugin-entry"; diff --git a/extensions/browser/src/browser-runtime.ts b/extensions/browser/src/browser-runtime.ts new file mode 100644 index 00000000000..d48dad1c214 --- /dev/null +++ b/extensions/browser/src/browser-runtime.ts @@ -0,0 +1,70 @@ +export { + browserAct, + browserArmDialog, + browserArmFileChooser, + browserConsoleMessages, + browserNavigate, + browserPdfSave, + browserScreenshotAction, +} from "./browser/client-actions.js"; +export { + browserCloseTab, + browserFocusTab, + browserOpenTab, + browserCreateProfile, + browserDeleteProfile, + browserProfiles, + browserResetProfile, + browserSnapshot, + browserStart, + browserStatus, + browserStop, + browserTabAction, + browserTabs, +} from "./browser/client.js"; +export type { + BrowserCreateProfileResult, + BrowserDeleteProfileResult, + BrowserResetProfileResult, + BrowserStatus, + BrowserTab, + BrowserTransport, + ProfileStatus, + SnapshotResult, +} from "./browser/client.js"; +export { resolveBrowserConfig, resolveProfile } from "./browser/config.js"; +export { DEFAULT_AI_SNAPSHOT_MAX_CHARS } from "./browser/constants.js"; +export { redactCdpUrl } from "./browser/cdp.helpers.js"; +export { DEFAULT_UPLOAD_DIR, resolveExistingPathsWithinRoot } from "./browser/paths.js"; +export { getBrowserProfileCapabilities } from "./browser/profile-capabilities.js"; +export { applyBrowserProxyPaths, persistBrowserProxyFiles } from "./browser/proxy-files.js"; +export { + isPersistentBrowserProfileMutation, + normalizeBrowserRequestPath, + resolveRequestedBrowserProfile, +} from "./browser/request-policy.js"; +export { + trackSessionBrowserTab, + untrackSessionBrowserTab, +} from "./browser/session-tab-registry.js"; +export { ensureBrowserControlAuth, resolveBrowserControlAuth } from "./browser/control-auth.js"; +export { + createBrowserControlContext, + getBrowserControlState, + startBrowserControlServiceFromConfig, + stopBrowserControlService, +} from "./control-service.js"; +export { createBrowserRuntimeState, stopBrowserRuntime } from "./browser/runtime-lifecycle.js"; +export { type BrowserServerState, createBrowserRouteContext } from "./browser/server-context.js"; +export { registerBrowserRoutes } from "./browser/routes/index.js"; +export { createBrowserRouteDispatcher } from "./browser/routes/dispatcher.js"; +export type { BrowserRouteRegistrar } from "./browser/routes/types.js"; +export { + installBrowserAuthMiddleware, + installBrowserCommonMiddleware, +} from "./browser/server-middleware.js"; +export type { BrowserFormField } from "./browser/client-actions-core.js"; +export { + normalizeBrowserFormField, + normalizeBrowserFormFieldValue, +} from "./browser/form-fields.js"; diff --git a/extensions/browser/src/browser-tool.actions.ts b/extensions/browser/src/browser-tool.actions.ts new file mode 100644 index 00000000000..bbd92145c04 --- /dev/null +++ b/extensions/browser/src/browser-tool.actions.ts @@ -0,0 +1,402 @@ +import type { AgentToolResult } from "@mariozechner/pi-agent-core"; +import { + DEFAULT_AI_SNAPSHOT_MAX_CHARS, + browserAct, + browserConsoleMessages, + browserSnapshot, + browserTabs, + getBrowserProfileCapabilities, + imageResultFromFile, + jsonResult, + loadConfig, + resolveBrowserConfig, + resolveProfile, + wrapExternalContent, +} from "./core-api.js"; + +const browserToolActionDeps = { + browserAct, + browserConsoleMessages, + browserSnapshot, + browserTabs, + imageResultFromFile, + loadConfig, +}; + +export const __testing = { + setDepsForTest( + overrides: Partial<{ + browserAct: typeof browserAct; + browserConsoleMessages: typeof browserConsoleMessages; + browserSnapshot: typeof browserSnapshot; + browserTabs: typeof browserTabs; + imageResultFromFile: typeof imageResultFromFile; + loadConfig: typeof loadConfig; + }> | null, + ) { + browserToolActionDeps.browserAct = overrides?.browserAct ?? browserAct; + browserToolActionDeps.browserConsoleMessages = + overrides?.browserConsoleMessages ?? browserConsoleMessages; + browserToolActionDeps.browserSnapshot = overrides?.browserSnapshot ?? browserSnapshot; + browserToolActionDeps.browserTabs = overrides?.browserTabs ?? browserTabs; + browserToolActionDeps.imageResultFromFile = + overrides?.imageResultFromFile ?? imageResultFromFile; + browserToolActionDeps.loadConfig = overrides?.loadConfig ?? loadConfig; + }, +}; + +type BrowserProxyRequest = (opts: { + method: string; + path: string; + query?: Record; + body?: unknown; + timeoutMs?: number; + profile?: string; +}) => Promise; + +function wrapBrowserExternalJson(params: { + kind: "snapshot" | "console" | "tabs"; + payload: unknown; + includeWarning?: boolean; +}): { wrappedText: string; safeDetails: Record } { + const extractedText = JSON.stringify(params.payload, null, 2); + const wrappedText = wrapExternalContent(extractedText, { + source: "browser", + includeWarning: params.includeWarning ?? true, + }); + return { + wrappedText, + safeDetails: { + ok: true, + externalContent: { + untrusted: true, + source: "browser", + kind: params.kind, + wrapped: true, + }, + }, + }; +} + +function formatTabsToolResult(tabs: unknown[]): AgentToolResult { + const wrapped = wrapBrowserExternalJson({ + kind: "tabs", + payload: { tabs }, + includeWarning: false, + }); + const content: AgentToolResult["content"] = [ + { type: "text", text: wrapped.wrappedText }, + ]; + return { + content, + details: { ...wrapped.safeDetails, tabCount: tabs.length }, + }; +} + +function formatConsoleToolResult(result: { + targetId?: string; + messages?: unknown[]; +}): AgentToolResult { + const wrapped = wrapBrowserExternalJson({ + kind: "console", + payload: result, + includeWarning: false, + }); + return { + content: [{ type: "text" as const, text: wrapped.wrappedText }], + details: { + ...wrapped.safeDetails, + targetId: typeof result.targetId === "string" ? result.targetId : undefined, + messageCount: Array.isArray(result.messages) ? result.messages.length : undefined, + }, + }; +} + +function isChromeStaleTargetError(profile: string | undefined, err: unknown): boolean { + if (!profile) { + return false; + } + if (profile === "user") { + const msg = String(err); + return msg.includes("404:") && msg.includes("tab not found"); + } + const cfg = browserToolActionDeps.loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + const browserProfile = resolveProfile(resolved, profile); + if (!browserProfile || !getBrowserProfileCapabilities(browserProfile).usesChromeMcp) { + return false; + } + const msg = String(err); + return msg.includes("404:") && msg.includes("tab not found"); +} + +function stripTargetIdFromActRequest( + request: Parameters[1], +): Parameters[1] | null { + const targetId = typeof request.targetId === "string" ? request.targetId.trim() : undefined; + if (!targetId) { + return null; + } + const retryRequest = { ...request }; + delete retryRequest.targetId; + return retryRequest as Parameters[1]; +} + +function canRetryChromeActWithoutTargetId(request: Parameters[1]): boolean { + const typedRequest = request as Partial>; + const kind = + typeof typedRequest.kind === "string" + ? typedRequest.kind + : typeof typedRequest.action === "string" + ? typedRequest.action + : ""; + return kind === "hover" || kind === "scrollIntoView" || kind === "wait"; +} + +export async function executeTabsAction(params: { + baseUrl?: string; + profile?: string; + proxyRequest: BrowserProxyRequest | null; +}): Promise> { + const { baseUrl, profile, proxyRequest } = params; + if (proxyRequest) { + const result = await proxyRequest({ + method: "GET", + path: "/tabs", + profile, + }); + const tabs = (result as { tabs?: unknown[] }).tabs ?? []; + return formatTabsToolResult(tabs); + } + const tabs = await browserToolActionDeps.browserTabs(baseUrl, { profile }); + return formatTabsToolResult(tabs); +} + +export async function executeSnapshotAction(params: { + input: Record; + baseUrl?: string; + profile?: string; + proxyRequest: BrowserProxyRequest | null; +}): Promise> { + const { input, baseUrl, profile, proxyRequest } = params; + const snapshotDefaults = browserToolActionDeps.loadConfig().browser?.snapshotDefaults; + const format: "ai" | "aria" | undefined = + input.snapshotFormat === "ai" || input.snapshotFormat === "aria" + ? input.snapshotFormat + : undefined; + const mode: "efficient" | undefined = + input.mode === "efficient" + ? "efficient" + : format !== "aria" && snapshotDefaults?.mode === "efficient" + ? "efficient" + : undefined; + const labels = typeof input.labels === "boolean" ? input.labels : undefined; + const refs: "aria" | "role" | undefined = + input.refs === "aria" || input.refs === "role" ? input.refs : undefined; + const hasMaxChars = Object.hasOwn(input, "maxChars"); + const targetId = typeof input.targetId === "string" ? input.targetId.trim() : undefined; + const limit = + typeof input.limit === "number" && Number.isFinite(input.limit) ? input.limit : undefined; + const maxChars = + typeof input.maxChars === "number" && Number.isFinite(input.maxChars) && input.maxChars > 0 + ? Math.floor(input.maxChars) + : undefined; + const interactive = typeof input.interactive === "boolean" ? input.interactive : undefined; + const compact = typeof input.compact === "boolean" ? input.compact : undefined; + const depth = + typeof input.depth === "number" && Number.isFinite(input.depth) ? input.depth : undefined; + const selector = typeof input.selector === "string" ? input.selector.trim() : undefined; + const frame = typeof input.frame === "string" ? input.frame.trim() : undefined; + const resolvedMaxChars = + format === "ai" + ? hasMaxChars + ? maxChars + : mode === "efficient" + ? undefined + : DEFAULT_AI_SNAPSHOT_MAX_CHARS + : hasMaxChars + ? maxChars + : undefined; + const snapshotQuery = { + ...(format ? { format } : {}), + targetId, + limit, + ...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}), + refs, + interactive, + compact, + depth, + selector, + frame, + labels, + mode, + }; + const snapshot = proxyRequest + ? ((await proxyRequest({ + method: "GET", + path: "/snapshot", + profile, + query: snapshotQuery, + })) as Awaited>) + : await browserToolActionDeps.browserSnapshot(baseUrl, { + ...snapshotQuery, + profile, + }); + if (snapshot.format === "ai") { + const extractedText = snapshot.snapshot ?? ""; + const wrappedSnapshot = wrapExternalContent(extractedText, { + source: "browser", + includeWarning: true, + }); + const safeDetails = { + ok: true, + format: snapshot.format, + targetId: snapshot.targetId, + url: snapshot.url, + truncated: snapshot.truncated, + stats: snapshot.stats, + refs: snapshot.refs ? Object.keys(snapshot.refs).length : undefined, + labels: snapshot.labels, + labelsCount: snapshot.labelsCount, + labelsSkipped: snapshot.labelsSkipped, + imagePath: snapshot.imagePath, + imageType: snapshot.imageType, + externalContent: { + untrusted: true, + source: "browser", + kind: "snapshot", + format: "ai", + wrapped: true, + }, + }; + if (labels && snapshot.imagePath) { + return await browserToolActionDeps.imageResultFromFile({ + label: "browser:snapshot", + path: snapshot.imagePath, + extraText: wrappedSnapshot, + details: safeDetails, + }); + } + return { + content: [{ type: "text" as const, text: wrappedSnapshot }], + details: safeDetails, + }; + } + { + const wrapped = wrapBrowserExternalJson({ + kind: "snapshot", + payload: snapshot, + }); + return { + content: [{ type: "text" as const, text: wrapped.wrappedText }], + details: { + ...wrapped.safeDetails, + format: "aria", + targetId: snapshot.targetId, + url: snapshot.url, + nodeCount: snapshot.nodes.length, + externalContent: { + untrusted: true, + source: "browser", + kind: "snapshot", + format: "aria", + wrapped: true, + }, + }, + }; + } +} + +export async function executeConsoleAction(params: { + input: Record; + baseUrl?: string; + profile?: string; + proxyRequest: BrowserProxyRequest | null; +}): Promise> { + const { input, baseUrl, profile, proxyRequest } = params; + const level = typeof input.level === "string" ? input.level.trim() : undefined; + const targetId = typeof input.targetId === "string" ? input.targetId.trim() : undefined; + if (proxyRequest) { + const result = (await proxyRequest({ + method: "GET", + path: "/console", + profile, + query: { + level, + targetId, + }, + })) as { ok?: boolean; targetId?: string; messages?: unknown[] }; + return formatConsoleToolResult(result); + } + const result = await browserToolActionDeps.browserConsoleMessages(baseUrl, { + level, + targetId, + profile, + }); + return formatConsoleToolResult(result); +} + +export async function executeActAction(params: { + request: Parameters[1]; + baseUrl?: string; + profile?: string; + proxyRequest: BrowserProxyRequest | null; +}): Promise> { + const { request, baseUrl, profile, proxyRequest } = params; + try { + const result = proxyRequest + ? await proxyRequest({ + method: "POST", + path: "/act", + profile, + body: request, + }) + : await browserToolActionDeps.browserAct(baseUrl, request, { + profile, + }); + return jsonResult(result); + } catch (err) { + if (isChromeStaleTargetError(profile, err)) { + const retryRequest = stripTargetIdFromActRequest(request); + const tabs = proxyRequest + ? (( + (await proxyRequest({ + method: "GET", + path: "/tabs", + profile, + })) as { tabs?: unknown[] } + ).tabs ?? []) + : await browserToolActionDeps.browserTabs(baseUrl, { profile }).catch(() => []); + // Some user-browser targetIds can go stale between snapshots and actions. + // Only retry safe read-only actions, and only when exactly one tab remains attached. + if (retryRequest && canRetryChromeActWithoutTargetId(request) && tabs.length === 1) { + try { + const retryResult = proxyRequest + ? await proxyRequest({ + method: "POST", + path: "/act", + profile, + body: retryRequest, + }) + : await browserToolActionDeps.browserAct(baseUrl, retryRequest, { + profile, + }); + return jsonResult(retryResult); + } catch { + // Fall through to explicit stale-target guidance. + } + } + if (!tabs.length) { + throw new Error( + `No browser tabs found for profile="${profile}". Make sure the configured Chromium-based browser (v144+) is running and has open tabs, then retry.`, + { cause: err }, + ); + } + throw new Error( + `Chrome tab not found (stale targetId?). Run action=tabs profile="${profile}" and use one of the returned targetIds.`, + { cause: err }, + ); + } + throw err; + } +} diff --git a/extensions/browser/src/browser-tool.schema.ts b/extensions/browser/src/browser-tool.schema.ts new file mode 100644 index 00000000000..605a4fad563 --- /dev/null +++ b/extensions/browser/src/browser-tool.schema.ts @@ -0,0 +1,138 @@ +import { Type } from "@sinclair/typebox"; +import { optionalStringEnum, stringEnum } from "./core-api.js"; + +const BROWSER_ACT_KINDS = [ + "click", + "type", + "press", + "hover", + "drag", + "select", + "fill", + "resize", + "wait", + "evaluate", + "close", +] as const; + +const BROWSER_TOOL_ACTIONS = [ + "status", + "start", + "stop", + "profiles", + "tabs", + "open", + "focus", + "close", + "snapshot", + "screenshot", + "navigate", + "console", + "pdf", + "upload", + "dialog", + "act", +] as const; + +const BROWSER_TARGETS = ["sandbox", "host", "node"] as const; + +const BROWSER_SNAPSHOT_FORMATS = ["aria", "ai"] as const; +const BROWSER_SNAPSHOT_MODES = ["efficient"] as const; +const BROWSER_SNAPSHOT_REFS = ["role", "aria"] as const; + +const BROWSER_IMAGE_TYPES = ["png", "jpeg"] as const; + +// NOTE: Using a flattened object schema instead of Type.Union([Type.Object(...), ...]) +// because Claude API on Vertex AI rejects nested anyOf schemas as invalid JSON Schema. +// The discriminator (kind) determines which properties are relevant; runtime validates. +const BrowserActSchema = Type.Object({ + kind: stringEnum(BROWSER_ACT_KINDS), + // Common fields + targetId: Type.Optional(Type.String()), + ref: Type.Optional(Type.String()), + // click + doubleClick: Type.Optional(Type.Boolean()), + button: Type.Optional(Type.String()), + modifiers: Type.Optional(Type.Array(Type.String())), + // type + text: Type.Optional(Type.String()), + submit: Type.Optional(Type.Boolean()), + slowly: Type.Optional(Type.Boolean()), + // press + key: Type.Optional(Type.String()), + delayMs: Type.Optional(Type.Number()), + // drag + startRef: Type.Optional(Type.String()), + endRef: Type.Optional(Type.String()), + // select + values: Type.Optional(Type.Array(Type.String())), + // fill - use permissive array of objects + fields: Type.Optional(Type.Array(Type.Object({}, { additionalProperties: true }))), + // resize + width: Type.Optional(Type.Number()), + height: Type.Optional(Type.Number()), + // wait + timeMs: Type.Optional(Type.Number()), + selector: Type.Optional(Type.String()), + url: Type.Optional(Type.String()), + loadState: Type.Optional(Type.String()), + textGone: Type.Optional(Type.String()), + timeoutMs: Type.Optional(Type.Number()), + // evaluate + fn: Type.Optional(Type.String()), +}); + +// IMPORTANT: OpenAI function tool schemas must have a top-level `type: "object"`. +// A root-level `Type.Union([...])` compiles to `{ anyOf: [...] }` (no `type`), +// which OpenAI rejects ("Invalid schema ... type: None"). Keep this schema an object. +export const BrowserToolSchema = Type.Object({ + action: stringEnum(BROWSER_TOOL_ACTIONS), + target: optionalStringEnum(BROWSER_TARGETS), + node: Type.Optional(Type.String()), + profile: Type.Optional(Type.String()), + targetUrl: Type.Optional(Type.String()), + url: Type.Optional(Type.String()), + targetId: Type.Optional(Type.String()), + limit: Type.Optional(Type.Number()), + maxChars: Type.Optional(Type.Number()), + mode: optionalStringEnum(BROWSER_SNAPSHOT_MODES), + snapshotFormat: optionalStringEnum(BROWSER_SNAPSHOT_FORMATS), + refs: optionalStringEnum(BROWSER_SNAPSHOT_REFS), + interactive: Type.Optional(Type.Boolean()), + compact: Type.Optional(Type.Boolean()), + depth: Type.Optional(Type.Number()), + selector: Type.Optional(Type.String()), + frame: Type.Optional(Type.String()), + labels: Type.Optional(Type.Boolean()), + fullPage: Type.Optional(Type.Boolean()), + ref: Type.Optional(Type.String()), + element: Type.Optional(Type.String()), + type: optionalStringEnum(BROWSER_IMAGE_TYPES), + level: Type.Optional(Type.String()), + paths: Type.Optional(Type.Array(Type.String())), + inputRef: Type.Optional(Type.String()), + timeoutMs: Type.Optional(Type.Number()), + accept: Type.Optional(Type.Boolean()), + promptText: Type.Optional(Type.String()), + // Legacy flattened act params (preferred: request={...}) + kind: Type.Optional(stringEnum(BROWSER_ACT_KINDS)), + doubleClick: Type.Optional(Type.Boolean()), + button: Type.Optional(Type.String()), + modifiers: Type.Optional(Type.Array(Type.String())), + text: Type.Optional(Type.String()), + submit: Type.Optional(Type.Boolean()), + slowly: Type.Optional(Type.Boolean()), + key: Type.Optional(Type.String()), + delayMs: Type.Optional(Type.Number()), + startRef: Type.Optional(Type.String()), + endRef: Type.Optional(Type.String()), + values: Type.Optional(Type.Array(Type.String())), + fields: Type.Optional(Type.Array(Type.Object({}, { additionalProperties: true }))), + width: Type.Optional(Type.Number()), + height: Type.Optional(Type.Number()), + timeMs: Type.Optional(Type.Number()), + textGone: Type.Optional(Type.String()), + loadState: Type.Optional(Type.String()), + fn: Type.Optional(Type.String()), + request: Type.Optional(BrowserActSchema), +}); diff --git a/extensions/browser/src/browser-tool.ts b/extensions/browser/src/browser-tool.ts new file mode 100644 index 00000000000..cf0792fe316 --- /dev/null +++ b/extensions/browser/src/browser-tool.ts @@ -0,0 +1,755 @@ +import crypto from "node:crypto"; +import { + executeActAction, + executeConsoleAction, + executeSnapshotAction, + executeTabsAction, +} from "./browser-tool.actions.js"; +import { BrowserToolSchema } from "./browser-tool.schema.js"; +import { + type AnyAgentTool, + type NodeListNode, + DEFAULT_UPLOAD_DIR, + applyBrowserProxyPaths, + browserAct, + browserArmDialog, + browserArmFileChooser, + browserCloseTab, + browserFocusTab, + browserNavigate, + browserOpenTab, + browserPdfSave, + browserProfiles, + browserScreenshotAction, + browserStart, + browserStatus, + browserStop, + getBrowserProfileCapabilities, + imageResultFromFile, + jsonResult, + listNodes, + loadConfig, + persistBrowserProxyFiles, + readStringParam, + resolveBrowserConfig, + resolveExistingPathsWithinRoot, + resolveNodeIdFromList, + resolveProfile, + selectDefaultNodeFromList, + trackSessionBrowserTab, + untrackSessionBrowserTab, +} from "./core-api.js"; +import { callGatewayTool } from "./core-api.js"; + +const browserToolDeps = { + browserAct, + browserArmDialog, + browserArmFileChooser, + browserCloseTab, + browserFocusTab, + browserNavigate, + browserOpenTab, + browserPdfSave, + browserProfiles, + browserScreenshotAction, + browserStart, + browserStatus, + browserStop, + imageResultFromFile, + loadConfig, + listNodes, + callGatewayTool, + trackSessionBrowserTab, + untrackSessionBrowserTab, +}; + +export const __testing = { + setDepsForTest( + overrides: Partial<{ + browserAct: typeof browserAct; + browserArmDialog: typeof browserArmDialog; + browserArmFileChooser: typeof browserArmFileChooser; + browserCloseTab: typeof browserCloseTab; + browserFocusTab: typeof browserFocusTab; + browserNavigate: typeof browserNavigate; + browserOpenTab: typeof browserOpenTab; + browserPdfSave: typeof browserPdfSave; + browserProfiles: typeof browserProfiles; + browserScreenshotAction: typeof browserScreenshotAction; + browserStart: typeof browserStart; + browserStatus: typeof browserStatus; + browserStop: typeof browserStop; + imageResultFromFile: typeof imageResultFromFile; + loadConfig: typeof loadConfig; + listNodes: typeof listNodes; + callGatewayTool: typeof callGatewayTool; + trackSessionBrowserTab: typeof trackSessionBrowserTab; + untrackSessionBrowserTab: typeof untrackSessionBrowserTab; + }> | null, + ) { + browserToolDeps.browserAct = overrides?.browserAct ?? browserAct; + browserToolDeps.browserArmDialog = overrides?.browserArmDialog ?? browserArmDialog; + browserToolDeps.browserArmFileChooser = + overrides?.browserArmFileChooser ?? browserArmFileChooser; + browserToolDeps.browserCloseTab = overrides?.browserCloseTab ?? browserCloseTab; + browserToolDeps.browserFocusTab = overrides?.browserFocusTab ?? browserFocusTab; + browserToolDeps.browserNavigate = overrides?.browserNavigate ?? browserNavigate; + browserToolDeps.browserOpenTab = overrides?.browserOpenTab ?? browserOpenTab; + browserToolDeps.browserPdfSave = overrides?.browserPdfSave ?? browserPdfSave; + browserToolDeps.browserProfiles = overrides?.browserProfiles ?? browserProfiles; + browserToolDeps.browserScreenshotAction = + overrides?.browserScreenshotAction ?? browserScreenshotAction; + browserToolDeps.browserStart = overrides?.browserStart ?? browserStart; + browserToolDeps.browserStatus = overrides?.browserStatus ?? browserStatus; + browserToolDeps.browserStop = overrides?.browserStop ?? browserStop; + browserToolDeps.imageResultFromFile = overrides?.imageResultFromFile ?? imageResultFromFile; + browserToolDeps.loadConfig = overrides?.loadConfig ?? loadConfig; + browserToolDeps.listNodes = overrides?.listNodes ?? listNodes; + browserToolDeps.callGatewayTool = overrides?.callGatewayTool ?? callGatewayTool; + browserToolDeps.trackSessionBrowserTab = + overrides?.trackSessionBrowserTab ?? trackSessionBrowserTab; + browserToolDeps.untrackSessionBrowserTab = + overrides?.untrackSessionBrowserTab ?? untrackSessionBrowserTab; + }, +}; + +function readOptionalTargetAndTimeout(params: Record) { + const targetId = typeof params.targetId === "string" ? params.targetId.trim() : undefined; + const timeoutMs = + typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs) + ? params.timeoutMs + : undefined; + return { targetId, timeoutMs }; +} + +function readTargetUrlParam(params: Record) { + return ( + readStringParam(params, "targetUrl") ?? + readStringParam(params, "url", { required: true, label: "targetUrl" }) + ); +} + +const LEGACY_BROWSER_ACT_REQUEST_KEYS = [ + "targetId", + "ref", + "doubleClick", + "button", + "modifiers", + "text", + "submit", + "slowly", + "key", + "delayMs", + "startRef", + "endRef", + "values", + "fields", + "width", + "height", + "timeMs", + "textGone", + "selector", + "url", + "loadState", + "fn", + "timeoutMs", +] as const; + +function readActRequestParam(params: Record) { + const requestParam = params.request; + if (requestParam && typeof requestParam === "object") { + return requestParam as Parameters[1]; + } + + const kind = readStringParam(params, "kind"); + if (!kind) { + return undefined; + } + + const request: Record = { kind }; + for (const key of LEGACY_BROWSER_ACT_REQUEST_KEYS) { + if (!Object.hasOwn(params, key)) { + continue; + } + request[key] = params[key]; + } + return request as Parameters[1]; +} + +type BrowserProxyFile = { + path: string; + base64: string; + mimeType?: string; +}; + +type BrowserProxyResult = { + result: unknown; + files?: BrowserProxyFile[]; +}; + +const DEFAULT_BROWSER_PROXY_TIMEOUT_MS = 20_000; +const BROWSER_PROXY_GATEWAY_TIMEOUT_SLACK_MS = 5_000; + +type BrowserNodeTarget = { + nodeId: string; + label?: string; +}; + +function isBrowserNode(node: NodeListNode) { + const caps = Array.isArray(node.caps) ? node.caps : []; + const commands = Array.isArray(node.commands) ? node.commands : []; + return caps.includes("browser") || commands.includes("browser.proxy"); +} + +async function resolveBrowserNodeTarget(params: { + requestedNode?: string; + target?: "sandbox" | "host" | "node"; + sandboxBridgeUrl?: string; +}): Promise { + const cfg = browserToolDeps.loadConfig(); + const policy = cfg.gateway?.nodes?.browser; + const mode = policy?.mode ?? "auto"; + if (mode === "off") { + if (params.target === "node" || params.requestedNode) { + throw new Error("Node browser proxy is disabled (gateway.nodes.browser.mode=off)."); + } + return null; + } + if (params.sandboxBridgeUrl?.trim() && params.target !== "node" && !params.requestedNode) { + return null; + } + if (params.target && params.target !== "node") { + return null; + } + if (mode === "manual" && params.target !== "node" && !params.requestedNode) { + return null; + } + + const nodes = await browserToolDeps.listNodes({}); + const browserNodes = nodes.filter((node) => node.connected && isBrowserNode(node)); + if (browserNodes.length === 0) { + if (params.target === "node" || params.requestedNode) { + throw new Error("No connected browser-capable nodes."); + } + return null; + } + + const requested = params.requestedNode?.trim() || policy?.node?.trim(); + if (requested) { + const nodeId = resolveNodeIdFromList(browserNodes, requested, false); + const node = browserNodes.find((entry) => entry.nodeId === nodeId); + return { nodeId, label: node?.displayName ?? node?.remoteIp ?? nodeId }; + } + + const selected = selectDefaultNodeFromList(browserNodes, { + preferLocalMac: false, + fallback: "none", + }); + + if (params.target === "node") { + if (selected) { + return { + nodeId: selected.nodeId, + label: selected.displayName ?? selected.remoteIp ?? selected.nodeId, + }; + } + throw new Error( + `Multiple browser-capable nodes connected (${browserNodes.length}). Set gateway.nodes.browser.node or pass node=.`, + ); + } + + if (mode === "manual") { + return null; + } + + if (selected) { + return { + nodeId: selected.nodeId, + label: selected.displayName ?? selected.remoteIp ?? selected.nodeId, + }; + } + return null; +} + +async function callBrowserProxy(params: { + nodeId: string; + method: string; + path: string; + query?: Record; + body?: unknown; + timeoutMs?: number; + profile?: string; +}): Promise { + const proxyTimeoutMs = + typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs) + ? Math.max(1, Math.floor(params.timeoutMs)) + : DEFAULT_BROWSER_PROXY_TIMEOUT_MS; + const gatewayTimeoutMs = proxyTimeoutMs + BROWSER_PROXY_GATEWAY_TIMEOUT_SLACK_MS; + const payload = await browserToolDeps.callGatewayTool<{ payloadJSON?: string; payload?: string }>( + "node.invoke", + { timeoutMs: gatewayTimeoutMs }, + { + nodeId: params.nodeId, + command: "browser.proxy", + params: { + method: params.method, + path: params.path, + query: params.query, + body: params.body, + timeoutMs: proxyTimeoutMs, + profile: params.profile, + }, + idempotencyKey: crypto.randomUUID(), + }, + ); + const parsed = + payload?.payload ?? + (typeof payload?.payloadJSON === "string" && payload.payloadJSON + ? (JSON.parse(payload.payloadJSON) as BrowserProxyResult) + : null); + if (!parsed || typeof parsed !== "object" || !("result" in parsed)) { + throw new Error("browser proxy failed"); + } + return parsed; +} + +async function persistProxyFiles(files: BrowserProxyFile[] | undefined) { + return await persistBrowserProxyFiles(files); +} + +function applyProxyPaths(result: unknown, mapping: Map) { + applyBrowserProxyPaths(result, mapping); +} + +function resolveBrowserBaseUrl(params: { + target?: "sandbox" | "host"; + sandboxBridgeUrl?: string; + allowHostControl?: boolean; +}): string | undefined { + const cfg = loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + const normalizedSandbox = params.sandboxBridgeUrl?.trim() ?? ""; + const target = params.target ?? (normalizedSandbox ? "sandbox" : "host"); + + if (target === "sandbox") { + if (!normalizedSandbox) { + throw new Error( + 'Sandbox browser is unavailable. Enable agents.defaults.sandbox.browser.enabled or use target="host" if allowed.', + ); + } + return normalizedSandbox.replace(/\/$/, ""); + } + + if (params.allowHostControl === false) { + throw new Error("Host browser control is disabled by sandbox policy."); + } + if (!resolved.enabled) { + throw new Error( + "Browser control is disabled. Set browser.enabled=true in ~/.openclaw/openclaw.json.", + ); + } + return undefined; +} + +function shouldPreferHostForProfile(profileName: string | undefined) { + if (!profileName) { + return false; + } + const cfg = browserToolDeps.loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + const profile = resolveProfile(resolved, profileName); + if (!profile) { + return false; + } + const capabilities = getBrowserProfileCapabilities(profile); + return capabilities.usesChromeMcp; +} + +export function createBrowserTool(opts?: { + sandboxBridgeUrl?: string; + allowHostControl?: boolean; + agentSessionKey?: string; +}): AnyAgentTool { + const targetDefault = opts?.sandboxBridgeUrl ? "sandbox" : "host"; + const hostHint = + opts?.allowHostControl === false ? "Host target blocked by policy." : "Host target allowed."; + return { + label: "Browser", + name: "browser", + description: [ + "Control the browser via OpenClaw's browser control server (status/start/stop/profiles/tabs/open/snapshot/screenshot/actions).", + "Browser choice: omit profile by default for the isolated OpenClaw-managed browser (`openclaw`).", + 'For the logged-in user browser on the local host, use profile="user". A supported Chromium-based browser (v144+) must be running. Use only when existing logins/cookies matter and the user is present.', + 'When a node-hosted browser proxy is available, the tool may auto-route to it. Pin a node with node= or target="node".', + "When using refs from snapshot (e.g. e12), keep the same tab: prefer passing targetId from the snapshot response into subsequent actions (act/click/type/etc).", + 'For stable, self-resolving refs across calls, use snapshot with refs="aria" (Playwright aria-ref ids). Default refs="role" are role+name-based.', + "Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.", + `target selects browser location (sandbox|host|node). Default: ${targetDefault}.`, + hostHint, + ].join(" "), + parameters: BrowserToolSchema, + execute: async (_toolCallId, args) => { + const params = args as Record; + const action = readStringParam(params, "action", { required: true }); + const profile = readStringParam(params, "profile"); + const requestedNode = readStringParam(params, "node"); + let target = readStringParam(params, "target") as "sandbox" | "host" | "node" | undefined; + + if (requestedNode && target && target !== "node") { + throw new Error('node is only supported with target="node".'); + } + // User-browser profiles (existing-session) are host-only. + const isUserBrowserProfile = shouldPreferHostForProfile(profile); + if (isUserBrowserProfile) { + if (requestedNode || target === "node") { + throw new Error(`profile="${profile}" only supports the local host browser.`); + } + if (target === "sandbox") { + throw new Error( + `profile="${profile}" cannot use the sandbox browser; use target="host" or omit target.`, + ); + } + if (!target && !requestedNode) { + target = "host"; + } + } + + const nodeTarget = await resolveBrowserNodeTarget({ + requestedNode: requestedNode ?? undefined, + target, + sandboxBridgeUrl: opts?.sandboxBridgeUrl, + }); + + const resolvedTarget = target === "node" ? undefined : target; + const baseUrl = nodeTarget + ? undefined + : resolveBrowserBaseUrl({ + target: resolvedTarget, + sandboxBridgeUrl: opts?.sandboxBridgeUrl, + allowHostControl: opts?.allowHostControl, + }); + + const proxyRequest = nodeTarget + ? async (opts: { + method: string; + path: string; + query?: Record; + body?: unknown; + timeoutMs?: number; + profile?: string; + }) => { + const proxy = await callBrowserProxy({ + nodeId: nodeTarget.nodeId, + method: opts.method, + path: opts.path, + query: opts.query, + body: opts.body, + timeoutMs: opts.timeoutMs, + profile: opts.profile, + }); + const mapping = await persistProxyFiles(proxy.files); + applyProxyPaths(proxy.result, mapping); + return proxy.result; + } + : null; + + switch (action) { + case "status": + if (proxyRequest) { + return jsonResult( + await proxyRequest({ + method: "GET", + path: "/", + profile, + }), + ); + } + return jsonResult(await browserToolDeps.browserStatus(baseUrl, { profile })); + case "start": + if (proxyRequest) { + await proxyRequest({ + method: "POST", + path: "/start", + profile, + }); + return jsonResult( + await proxyRequest({ + method: "GET", + path: "/", + profile, + }), + ); + } + await browserToolDeps.browserStart(baseUrl, { profile }); + return jsonResult(await browserToolDeps.browserStatus(baseUrl, { profile })); + case "stop": + if (proxyRequest) { + await proxyRequest({ + method: "POST", + path: "/stop", + profile, + }); + return jsonResult( + await proxyRequest({ + method: "GET", + path: "/", + profile, + }), + ); + } + await browserToolDeps.browserStop(baseUrl, { profile }); + return jsonResult(await browserToolDeps.browserStatus(baseUrl, { profile })); + case "profiles": + if (proxyRequest) { + const result = await proxyRequest({ + method: "GET", + path: "/profiles", + }); + return jsonResult(result); + } + return jsonResult({ profiles: await browserToolDeps.browserProfiles(baseUrl) }); + case "tabs": + return await executeTabsAction({ baseUrl, profile, proxyRequest }); + case "open": { + const targetUrl = readTargetUrlParam(params); + if (proxyRequest) { + const result = await proxyRequest({ + method: "POST", + path: "/tabs/open", + profile, + body: { url: targetUrl }, + }); + return jsonResult(result); + } + const opened = await browserToolDeps.browserOpenTab(baseUrl, targetUrl, { profile }); + browserToolDeps.trackSessionBrowserTab({ + sessionKey: opts?.agentSessionKey, + targetId: opened.targetId, + baseUrl, + profile, + }); + return jsonResult(opened); + } + case "focus": { + const targetId = readStringParam(params, "targetId", { + required: true, + }); + if (proxyRequest) { + const result = await proxyRequest({ + method: "POST", + path: "/tabs/focus", + profile, + body: { targetId }, + }); + return jsonResult(result); + } + await browserToolDeps.browserFocusTab(baseUrl, targetId, { profile }); + return jsonResult({ ok: true }); + } + case "close": { + const targetId = readStringParam(params, "targetId"); + if (proxyRequest) { + const result = targetId + ? await proxyRequest({ + method: "DELETE", + path: `/tabs/${encodeURIComponent(targetId)}`, + profile, + }) + : await proxyRequest({ + method: "POST", + path: "/act", + profile, + body: { kind: "close" }, + }); + return jsonResult(result); + } + if (targetId) { + await browserToolDeps.browserCloseTab(baseUrl, targetId, { profile }); + browserToolDeps.untrackSessionBrowserTab({ + sessionKey: opts?.agentSessionKey, + targetId, + baseUrl, + profile, + }); + } else { + await browserToolDeps.browserAct(baseUrl, { kind: "close" }, { profile }); + } + return jsonResult({ ok: true }); + } + case "snapshot": + return await executeSnapshotAction({ + input: params, + baseUrl, + profile, + proxyRequest, + }); + case "screenshot": { + const targetId = readStringParam(params, "targetId"); + const fullPage = Boolean(params.fullPage); + const ref = readStringParam(params, "ref"); + const element = readStringParam(params, "element"); + const type = params.type === "jpeg" ? "jpeg" : "png"; + const result = proxyRequest + ? ((await proxyRequest({ + method: "POST", + path: "/screenshot", + profile, + body: { + targetId, + fullPage, + ref, + element, + type, + }, + })) as Awaited>) + : await browserToolDeps.browserScreenshotAction(baseUrl, { + targetId, + fullPage, + ref, + element, + type, + profile, + }); + return await browserToolDeps.imageResultFromFile({ + label: "browser:screenshot", + path: result.path, + details: result, + }); + } + case "navigate": { + const targetUrl = readTargetUrlParam(params); + const targetId = readStringParam(params, "targetId"); + if (proxyRequest) { + const result = await proxyRequest({ + method: "POST", + path: "/navigate", + profile, + body: { + url: targetUrl, + targetId, + }, + }); + return jsonResult(result); + } + return jsonResult( + await browserToolDeps.browserNavigate(baseUrl, { + url: targetUrl, + targetId, + profile, + }), + ); + } + case "console": + return await executeConsoleAction({ + input: params, + baseUrl, + profile, + proxyRequest, + }); + case "pdf": { + const targetId = typeof params.targetId === "string" ? params.targetId.trim() : undefined; + const result = proxyRequest + ? ((await proxyRequest({ + method: "POST", + path: "/pdf", + profile, + body: { targetId }, + })) as Awaited>) + : await browserToolDeps.browserPdfSave(baseUrl, { targetId, profile }); + return { + content: [{ type: "text" as const, text: `FILE:${result.path}` }], + details: result, + }; + } + case "upload": { + const paths = Array.isArray(params.paths) ? params.paths.map((p) => String(p)) : []; + if (paths.length === 0) { + throw new Error("paths required"); + } + const uploadPathsResult = await resolveExistingPathsWithinRoot({ + rootDir: DEFAULT_UPLOAD_DIR, + requestedPaths: paths, + scopeLabel: `uploads directory (${DEFAULT_UPLOAD_DIR})`, + }); + if (!uploadPathsResult.ok) { + throw new Error(uploadPathsResult.error); + } + const normalizedPaths = uploadPathsResult.paths; + const ref = readStringParam(params, "ref"); + const inputRef = readStringParam(params, "inputRef"); + const element = readStringParam(params, "element"); + const { targetId, timeoutMs } = readOptionalTargetAndTimeout(params); + if (proxyRequest) { + const result = await proxyRequest({ + method: "POST", + path: "/hooks/file-chooser", + profile, + body: { + paths: normalizedPaths, + ref, + inputRef, + element, + targetId, + timeoutMs, + }, + }); + return jsonResult(result); + } + return jsonResult( + await browserToolDeps.browserArmFileChooser(baseUrl, { + paths: normalizedPaths, + ref, + inputRef, + element, + targetId, + timeoutMs, + profile, + }), + ); + } + case "dialog": { + const accept = Boolean(params.accept); + const promptText = typeof params.promptText === "string" ? params.promptText : undefined; + const { targetId, timeoutMs } = readOptionalTargetAndTimeout(params); + if (proxyRequest) { + const result = await proxyRequest({ + method: "POST", + path: "/hooks/dialog", + profile, + body: { + accept, + promptText, + targetId, + timeoutMs, + }, + }); + return jsonResult(result); + } + return jsonResult( + await browserToolDeps.browserArmDialog(baseUrl, { + accept, + promptText, + targetId, + timeoutMs, + profile, + }), + ); + } + case "act": { + const request = readActRequestParam(params); + if (!request) { + throw new Error("request required"); + } + return await executeActAction({ + request, + baseUrl, + profile, + proxyRequest, + }); + } + default: + throw new Error(`Unknown action: ${action}`); + } + }, + }; +} diff --git a/extensions/browser/src/cli/browser-cli-actions-input.ts b/extensions/browser/src/cli/browser-cli-actions-input.ts new file mode 100644 index 00000000000..7c852948ed0 --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-actions-input.ts @@ -0,0 +1 @@ +export { registerBrowserActionInputCommands } from "./browser-cli-actions-input/register.js"; diff --git a/extensions/browser/src/cli/browser-cli-actions-input/register.element.ts b/extensions/browser/src/cli/browser-cli-actions-input/register.element.ts new file mode 100644 index 00000000000..6236de6b634 --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-actions-input/register.element.ts @@ -0,0 +1,194 @@ +import type { Command } from "commander"; +import type { BrowserParentOpts } from "../browser-cli-shared.js"; +import { danger, defaultRuntime } from "../core-api.js"; +import { + callBrowserAct, + logBrowserActionResult, + requireRef, + resolveBrowserActionContext, +} from "./shared.js"; + +export function registerBrowserElementCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + const runElementAction = async (params: { + cmd: Command; + body: Record; + successMessage: string | ((result: unknown) => string); + timeoutMs?: number; + }): Promise => { + const { parent, profile } = resolveBrowserActionContext(params.cmd, parentOpts); + try { + const result = await callBrowserAct({ + parent, + profile, + body: params.body, + timeoutMs: params.timeoutMs, + }); + const successMessage = + typeof params.successMessage === "function" + ? params.successMessage(result) + : params.successMessage; + logBrowserActionResult(parent, result, successMessage); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }; + + browser + .command("click") + .description("Click an element by ref from snapshot") + .argument("", "Ref id from snapshot") + .option("--target-id ", "CDP target id (or unique prefix)") + .option("--double", "Double click", false) + .option("--button ", "Mouse button to use") + .option("--modifiers ", "Comma-separated modifiers (Shift,Alt,Meta)") + .action(async (ref: string | undefined, opts, cmd) => { + const refValue = requireRef(ref); + if (!refValue) { + return; + } + const modifiers = opts.modifiers + ? String(opts.modifiers) + .split(",") + .map((v: string) => v.trim()) + .filter(Boolean) + : undefined; + await runElementAction({ + cmd, + body: { + kind: "click", + ref: refValue, + targetId: opts.targetId?.trim() || undefined, + doubleClick: Boolean(opts.double), + button: opts.button?.trim() || undefined, + modifiers, + }, + successMessage: (result) => { + const url = (result as { url?: unknown }).url; + const suffix = typeof url === "string" && url ? ` on ${url}` : ""; + return `clicked ref ${refValue}${suffix}`; + }, + }); + }); + + browser + .command("type") + .description("Type into an element by ref from snapshot") + .argument("", "Ref id from snapshot") + .argument("", "Text to type") + .option("--submit", "Press Enter after typing", false) + .option("--slowly", "Type slowly (human-like)", false) + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (ref: string | undefined, text: string, opts, cmd) => { + const refValue = requireRef(ref); + if (!refValue) { + return; + } + await runElementAction({ + cmd, + body: { + kind: "type", + ref: refValue, + text, + submit: Boolean(opts.submit), + slowly: Boolean(opts.slowly), + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: `typed into ref ${refValue}`, + }); + }); + + browser + .command("press") + .description("Press a key") + .argument("", "Key to press (e.g. Enter)") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (key: string, opts, cmd) => { + await runElementAction({ + cmd, + body: { kind: "press", key, targetId: opts.targetId?.trim() || undefined }, + successMessage: `pressed ${key}`, + }); + }); + + browser + .command("hover") + .description("Hover an element by ai ref") + .argument("", "Ref id from snapshot") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (ref: string, opts, cmd) => { + await runElementAction({ + cmd, + body: { kind: "hover", ref, targetId: opts.targetId?.trim() || undefined }, + successMessage: `hovered ref ${ref}`, + }); + }); + + browser + .command("scrollintoview") + .description("Scroll an element into view by ref from snapshot") + .argument("", "Ref id from snapshot") + .option("--target-id ", "CDP target id (or unique prefix)") + .option("--timeout-ms ", "How long to wait for scroll (default: 20000)", (v: string) => + Number(v), + ) + .action(async (ref: string | undefined, opts, cmd) => { + const refValue = requireRef(ref); + if (!refValue) { + return; + } + const timeoutMs = Number.isFinite(opts.timeoutMs) ? opts.timeoutMs : undefined; + await runElementAction({ + cmd, + body: { + kind: "scrollIntoView", + ref: refValue, + targetId: opts.targetId?.trim() || undefined, + timeoutMs, + }, + timeoutMs, + successMessage: `scrolled into view: ${refValue}`, + }); + }); + + browser + .command("drag") + .description("Drag from one ref to another") + .argument("", "Start ref id") + .argument("", "End ref id") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (startRef: string, endRef: string, opts, cmd) => { + await runElementAction({ + cmd, + body: { + kind: "drag", + startRef, + endRef, + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: `dragged ${startRef} → ${endRef}`, + }); + }); + + browser + .command("select") + .description("Select option(s) in a select element") + .argument("", "Ref id from snapshot") + .argument("", "Option values to select") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (ref: string, values: string[], opts, cmd) => { + await runElementAction({ + cmd, + body: { + kind: "select", + ref, + values, + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: `selected ${values.join(", ")}`, + }); + }); +} diff --git a/extensions/browser/src/cli/browser-cli-actions-input/register.files-downloads.ts b/extensions/browser/src/cli/browser-cli-actions-input/register.files-downloads.ts new file mode 100644 index 00000000000..b570abfb02e --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-actions-input/register.files-downloads.ts @@ -0,0 +1,204 @@ +import type { Command } from "commander"; +import { callBrowserRequest, type BrowserParentOpts } from "../browser-cli-shared.js"; +import { + danger, + DEFAULT_UPLOAD_DIR, + defaultRuntime, + resolveExistingPathsWithinRoot, + shortenHomePath, +} from "../core-api.js"; +import { resolveBrowserActionContext } from "./shared.js"; + +async function normalizeUploadPaths(paths: string[]): Promise { + const result = await resolveExistingPathsWithinRoot({ + rootDir: DEFAULT_UPLOAD_DIR, + requestedPaths: paths, + scopeLabel: `uploads directory (${DEFAULT_UPLOAD_DIR})`, + }); + if (!result.ok) { + throw new Error(result.error); + } + return result.paths; +} + +async function runBrowserPostAction(params: { + parent: BrowserParentOpts; + profile: string | undefined; + path: string; + body: Record; + timeoutMs: number; + describeSuccess: (result: T) => string; +}): Promise { + try { + const result = await callBrowserRequest( + params.parent, + { + method: "POST", + path: params.path, + query: params.profile ? { profile: params.profile } : undefined, + body: params.body, + }, + { timeoutMs: params.timeoutMs }, + ); + if (params.parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(params.describeSuccess(result)); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } +} + +export function registerBrowserFilesAndDownloadsCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + const resolveTimeoutAndTarget = (opts: { timeoutMs?: unknown; targetId?: unknown }) => { + const timeoutMs = Number.isFinite(opts.timeoutMs) ? Number(opts.timeoutMs) : undefined; + const targetId = + typeof opts.targetId === "string" ? opts.targetId.trim() || undefined : undefined; + return { timeoutMs, targetId }; + }; + + const runDownloadCommand = async ( + cmd: Command, + opts: { timeoutMs?: unknown; targetId?: unknown }, + request: { path: string; body: Record }, + ) => { + const { parent, profile } = resolveBrowserActionContext(cmd, parentOpts); + const { timeoutMs, targetId } = resolveTimeoutAndTarget(opts); + await runBrowserPostAction<{ download: { path: string } }>({ + parent, + profile, + path: request.path, + body: { + ...request.body, + targetId, + timeoutMs, + }, + timeoutMs: timeoutMs ?? 20000, + describeSuccess: (result) => `downloaded: ${shortenHomePath(result.download.path)}`, + }); + }; + + browser + .command("upload") + .description("Arm file upload for the next file chooser") + .argument( + "", + "File paths to upload (must be within OpenClaw temp uploads dir, e.g. /tmp/openclaw/uploads/file.pdf)", + ) + .option("--ref ", "Ref id from snapshot to click after arming") + .option("--input-ref ", "Ref id for to set directly") + .option("--element ", "CSS selector for ") + .option("--target-id ", "CDP target id (or unique prefix)") + .option( + "--timeout-ms ", + "How long to wait for the next file chooser (default: 120000)", + (v: string) => Number(v), + ) + .action(async (paths: string[], opts, cmd) => { + const { parent, profile } = resolveBrowserActionContext(cmd, parentOpts); + const normalizedPaths = await normalizeUploadPaths(paths); + const { timeoutMs, targetId } = resolveTimeoutAndTarget(opts); + await runBrowserPostAction({ + parent, + profile, + path: "/hooks/file-chooser", + body: { + paths: normalizedPaths, + ref: opts.ref?.trim() || undefined, + inputRef: opts.inputRef?.trim() || undefined, + element: opts.element?.trim() || undefined, + targetId, + timeoutMs, + }, + timeoutMs: timeoutMs ?? 20000, + describeSuccess: () => `upload armed for ${paths.length} file(s)`, + }); + }); + + browser + .command("waitfordownload") + .description("Wait for the next download (and save it)") + .argument( + "[path]", + "Save path within openclaw temp downloads dir (default: /tmp/openclaw/downloads/...; fallback: os.tmpdir()/openclaw/downloads/...)", + ) + .option("--target-id ", "CDP target id (or unique prefix)") + .option( + "--timeout-ms ", + "How long to wait for the next download (default: 120000)", + (v: string) => Number(v), + ) + .action(async (outPath: string | undefined, opts, cmd) => { + await runDownloadCommand(cmd, opts, { + path: "/wait/download", + body: { + path: outPath?.trim() || undefined, + }, + }); + }); + + browser + .command("download") + .description("Click a ref and save the resulting download") + .argument("", "Ref id from snapshot to click") + .argument( + "", + "Save path within openclaw temp downloads dir (e.g. report.pdf or /tmp/openclaw/downloads/report.pdf)", + ) + .option("--target-id ", "CDP target id (or unique prefix)") + .option( + "--timeout-ms ", + "How long to wait for the download to start (default: 120000)", + (v: string) => Number(v), + ) + .action(async (ref: string, outPath: string, opts, cmd) => { + await runDownloadCommand(cmd, opts, { + path: "/download", + body: { + ref, + path: outPath, + }, + }); + }); + + browser + .command("dialog") + .description("Arm the next modal dialog (alert/confirm/prompt)") + .option("--accept", "Accept the dialog", false) + .option("--dismiss", "Dismiss the dialog", false) + .option("--prompt ", "Prompt response text") + .option("--target-id ", "CDP target id (or unique prefix)") + .option( + "--timeout-ms ", + "How long to wait for the next dialog (default: 120000)", + (v: string) => Number(v), + ) + .action(async (opts, cmd) => { + const { parent, profile } = resolveBrowserActionContext(cmd, parentOpts); + const accept = opts.accept ? true : opts.dismiss ? false : undefined; + if (accept === undefined) { + defaultRuntime.error(danger("Specify --accept or --dismiss")); + defaultRuntime.exit(1); + return; + } + const { timeoutMs, targetId } = resolveTimeoutAndTarget(opts); + await runBrowserPostAction({ + parent, + profile, + path: "/hooks/dialog", + body: { + accept, + promptText: opts.prompt?.trim() || undefined, + targetId, + timeoutMs, + }, + timeoutMs: timeoutMs ?? 20000, + describeSuccess: () => "dialog armed", + }); + }); +} diff --git a/extensions/browser/src/cli/browser-cli-actions-input/register.form-wait-eval.ts b/extensions/browser/src/cli/browser-cli-actions-input/register.form-wait-eval.ts new file mode 100644 index 00000000000..3cd717e9bcd --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-actions-input/register.form-wait-eval.ts @@ -0,0 +1,127 @@ +import type { Command } from "commander"; +import type { BrowserParentOpts } from "../browser-cli-shared.js"; +import { danger, defaultRuntime } from "../core-api.js"; +import { + callBrowserAct, + logBrowserActionResult, + readFields, + resolveBrowserActionContext, +} from "./shared.js"; + +export function registerBrowserFormWaitEvalCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + browser + .command("fill") + .description("Fill a form with JSON field descriptors") + .option("--fields ", "JSON array of field objects") + .option("--fields-file ", "Read JSON array from a file") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + const { parent, profile } = resolveBrowserActionContext(cmd, parentOpts); + try { + const fields = await readFields({ + fields: opts.fields, + fieldsFile: opts.fieldsFile, + }); + const result = await callBrowserAct<{ result?: unknown }>({ + parent, + profile, + body: { + kind: "fill", + fields, + targetId: opts.targetId?.trim() || undefined, + }, + }); + logBrowserActionResult(parent, result, `filled ${fields.length} field(s)`); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + + browser + .command("wait") + .description("Wait for time, selector, URL, load state, or JS conditions") + .argument("[selector]", "CSS selector to wait for (visible)") + .option("--time ", "Wait for N milliseconds", (v: string) => Number(v)) + .option("--text ", "Wait for text to appear") + .option("--text-gone ", "Wait for text to disappear") + .option("--url ", "Wait for URL (supports globs like **/dash)") + .option("--load ", "Wait for load state") + .option("--fn ", "Wait for JS condition (passed to waitForFunction)") + .option( + "--timeout-ms ", + "How long to wait for each condition (default: 20000)", + (v: string) => Number(v), + ) + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (selector: string | undefined, opts, cmd) => { + const { parent, profile } = resolveBrowserActionContext(cmd, parentOpts); + try { + const sel = selector?.trim() || undefined; + const load = + opts.load === "load" || opts.load === "domcontentloaded" || opts.load === "networkidle" + ? (opts.load as "load" | "domcontentloaded" | "networkidle") + : undefined; + const timeoutMs = Number.isFinite(opts.timeoutMs) ? opts.timeoutMs : undefined; + const result = await callBrowserAct<{ result?: unknown }>({ + parent, + profile, + body: { + kind: "wait", + timeMs: Number.isFinite(opts.time) ? opts.time : undefined, + text: opts.text?.trim() || undefined, + textGone: opts.textGone?.trim() || undefined, + selector: sel, + url: opts.url?.trim() || undefined, + loadState: load, + fn: opts.fn?.trim() || undefined, + targetId: opts.targetId?.trim() || undefined, + timeoutMs, + }, + timeoutMs, + }); + logBrowserActionResult(parent, result, "wait complete"); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + + browser + .command("evaluate") + .description("Evaluate a function against the page or a ref") + .option("--fn ", "Function source, e.g. (el) => el.textContent") + .option("--ref ", "Ref from snapshot") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + const { parent, profile } = resolveBrowserActionContext(cmd, parentOpts); + if (!opts.fn) { + defaultRuntime.error(danger("Missing --fn")); + defaultRuntime.exit(1); + return; + } + try { + const result = await callBrowserAct<{ result?: unknown }>({ + parent, + profile, + body: { + kind: "evaluate", + fn: opts.fn, + ref: opts.ref?.trim() || undefined, + targetId: opts.targetId?.trim() || undefined, + }, + }); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.writeJson(result.result ?? null); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); +} diff --git a/extensions/browser/src/cli/browser-cli-actions-input/register.navigation.ts b/extensions/browser/src/cli/browser-cli-actions-input/register.navigation.ts new file mode 100644 index 00000000000..b07358c8247 --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-actions-input/register.navigation.ts @@ -0,0 +1,69 @@ +import type { Command } from "commander"; +import { runBrowserResizeWithOutput } from "../browser-cli-resize.js"; +import { callBrowserRequest, type BrowserParentOpts } from "../browser-cli-shared.js"; +import { danger, defaultRuntime } from "../core-api.js"; +import { requireRef, resolveBrowserActionContext } from "./shared.js"; + +export function registerBrowserNavigationCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + browser + .command("navigate") + .description("Navigate the current tab to a URL") + .argument("", "URL to navigate to") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (url: string, opts, cmd) => { + const { parent, profile } = resolveBrowserActionContext(cmd, parentOpts); + try { + const result = await callBrowserRequest<{ url?: string }>( + parent, + { + method: "POST", + path: "/navigate", + query: profile ? { profile } : undefined, + body: { + url, + targetId: opts.targetId?.trim() || undefined, + }, + }, + { timeoutMs: 20000 }, + ); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(`navigated to ${result.url ?? url}`); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + + browser + .command("resize") + .description("Resize the viewport") + .argument("", "Viewport width", (v: string) => Number(v)) + .argument("", "Viewport height", (v: string) => Number(v)) + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (width: number, height: number, opts, cmd) => { + const { parent, profile } = resolveBrowserActionContext(cmd, parentOpts); + try { + await runBrowserResizeWithOutput({ + parent, + profile, + width, + height, + targetId: opts.targetId, + timeoutMs: 20000, + successMessage: `resized to ${width}x${height}`, + }); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + + // Keep `requireRef` reachable; shared utilities are intended for other modules too. + void requireRef; +} diff --git a/extensions/browser/src/cli/browser-cli-actions-input/register.ts b/extensions/browser/src/cli/browser-cli-actions-input/register.ts new file mode 100644 index 00000000000..973488a220e --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-actions-input/register.ts @@ -0,0 +1,16 @@ +import type { Command } from "commander"; +import type { BrowserParentOpts } from "../browser-cli-shared.js"; +import { registerBrowserElementCommands } from "./register.element.js"; +import { registerBrowserFilesAndDownloadsCommands } from "./register.files-downloads.js"; +import { registerBrowserFormWaitEvalCommands } from "./register.form-wait-eval.js"; +import { registerBrowserNavigationCommands } from "./register.navigation.js"; + +export function registerBrowserActionInputCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + registerBrowserNavigationCommands(browser, parentOpts); + registerBrowserElementCommands(browser, parentOpts); + registerBrowserFilesAndDownloadsCommands(browser, parentOpts); + registerBrowserFormWaitEvalCommands(browser, parentOpts); +} diff --git a/extensions/browser/src/cli/browser-cli-actions-input/shared.ts b/extensions/browser/src/cli/browser-cli-actions-input/shared.ts new file mode 100644 index 00000000000..050f2ed7b9d --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-actions-input/shared.ts @@ -0,0 +1,100 @@ +import type { Command } from "commander"; +import { callBrowserRequest, type BrowserParentOpts } from "../browser-cli-shared.js"; +import { + danger, + defaultRuntime, + normalizeBrowserFormField, + normalizeBrowserFormFieldValue, + type BrowserFormField, +} from "../core-api.js"; + +export type BrowserActionContext = { + parent: BrowserParentOpts; + profile: string | undefined; +}; + +export function resolveBrowserActionContext( + cmd: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +): BrowserActionContext { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + return { parent, profile }; +} + +export async function callBrowserAct(params: { + parent: BrowserParentOpts; + profile?: string; + body: Record; + timeoutMs?: number; +}): Promise { + return await callBrowserRequest( + params.parent, + { + method: "POST", + path: "/act", + query: params.profile ? { profile: params.profile } : undefined, + body: params.body, + }, + { timeoutMs: params.timeoutMs ?? 20000 }, + ); +} + +export function logBrowserActionResult( + parent: BrowserParentOpts, + result: unknown, + successMessage: string, +) { + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(successMessage); +} + +export function requireRef(ref: string | undefined) { + const refValue = typeof ref === "string" ? ref.trim() : ""; + if (!refValue) { + defaultRuntime.error(danger("ref is required")); + defaultRuntime.exit(1); + return null; + } + return refValue; +} + +async function readFile(path: string): Promise { + const fs = await import("node:fs/promises"); + return await fs.readFile(path, "utf8"); +} + +export async function readFields(opts: { + fields?: string; + fieldsFile?: string; +}): Promise { + const payload = opts.fieldsFile ? await readFile(opts.fieldsFile) : (opts.fields ?? ""); + if (!payload.trim()) { + throw new Error("fields are required"); + } + const parsed = JSON.parse(payload) as unknown; + if (!Array.isArray(parsed)) { + throw new Error("fields must be an array"); + } + return parsed.map((entry, index) => { + if (!entry || typeof entry !== "object") { + throw new Error(`fields[${index}] must be an object`); + } + const rec = entry as Record; + const parsedField = normalizeBrowserFormField(rec); + if (!parsedField) { + throw new Error(`fields[${index}] must include ref`); + } + if ( + rec.value === undefined || + rec.value === null || + normalizeBrowserFormFieldValue(rec.value) !== undefined + ) { + return parsedField; + } + throw new Error(`fields[${index}].value must be string, number, boolean, or null`); + }); +} diff --git a/extensions/browser/src/cli/browser-cli-actions-observe.ts b/extensions/browser/src/cli/browser-cli-actions-observe.ts new file mode 100644 index 00000000000..01aced7faf9 --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-actions-observe.ts @@ -0,0 +1,114 @@ +import type { Command } from "commander"; +import { runCommandWithRuntime } from "../core-api.js"; +import { callBrowserRequest, type BrowserParentOpts } from "./browser-cli-shared.js"; +import { danger, defaultRuntime, shortenHomePath } from "./core-api.js"; + +function runBrowserObserve(action: () => Promise) { + return runCommandWithRuntime(defaultRuntime, action, (err) => { + defaultRuntime.error(danger(String(err as unknown))); + defaultRuntime.exit(1); + }); +} + +export function registerBrowserActionObserveCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + browser + .command("console") + .description("Get recent console messages") + .option("--level ", "Filter by level (error, warn, info)") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserObserve(async () => { + const result = await callBrowserRequest<{ messages: unknown[] }>( + parent, + { + method: "GET", + path: "/console", + query: { + level: opts.level?.trim() || undefined, + targetId: opts.targetId?.trim() || undefined, + profile, + }, + }, + { timeoutMs: 20000 }, + ); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.writeJson(result.messages); + }); + }); + + browser + .command("pdf") + .description("Save page as PDF") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserObserve(async () => { + const result = await callBrowserRequest<{ path: string }>( + parent, + { + method: "POST", + path: "/pdf", + query: profile ? { profile } : undefined, + body: { targetId: opts.targetId?.trim() || undefined }, + }, + { timeoutMs: 20000 }, + ); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(`PDF: ${shortenHomePath(result.path)}`); + }); + }); + + browser + .command("responsebody") + .description("Wait for a network response and return its body") + .argument("", "URL (exact, substring, or glob like **/api)") + .option("--target-id ", "CDP target id (or unique prefix)") + .option( + "--timeout-ms ", + "How long to wait for the response (default: 20000)", + (v: string) => Number(v), + ) + .option("--max-chars ", "Max body chars to return (default: 200000)", (v: string) => + Number(v), + ) + .action(async (url: string, opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserObserve(async () => { + const timeoutMs = Number.isFinite(opts.timeoutMs) ? opts.timeoutMs : undefined; + const maxChars = Number.isFinite(opts.maxChars) ? opts.maxChars : undefined; + const result = await callBrowserRequest<{ response: { body: string } }>( + parent, + { + method: "POST", + path: "/response/body", + query: profile ? { profile } : undefined, + body: { + url, + targetId: opts.targetId?.trim() || undefined, + timeoutMs, + maxChars, + }, + }, + { timeoutMs: timeoutMs ?? 20000 }, + ); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(result.response.body); + }); + }); +} diff --git a/extensions/browser/src/cli/browser-cli-debug.ts b/extensions/browser/src/cli/browser-cli-debug.ts new file mode 100644 index 00000000000..984bb7d6a36 --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-debug.ts @@ -0,0 +1,230 @@ +import type { Command } from "commander"; +import { runCommandWithRuntime } from "../core-api.js"; +import { callBrowserRequest, type BrowserParentOpts } from "./browser-cli-shared.js"; +import { danger, defaultRuntime, shortenHomePath } from "./core-api.js"; + +const BROWSER_DEBUG_TIMEOUT_MS = 20000; + +type BrowserRequestParams = Parameters[1]; + +type DebugContext = { + parent: BrowserParentOpts; + profile?: string; +}; + +function runBrowserDebug(action: () => Promise) { + return runCommandWithRuntime(defaultRuntime, action, (err) => { + defaultRuntime.error(danger(String(err as unknown))); + defaultRuntime.exit(1); + }); +} + +async function withDebugContext( + cmd: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, + action: (context: DebugContext) => Promise, +) { + const parent = parentOpts(cmd); + await runBrowserDebug(() => + action({ + parent, + profile: parent.browserProfile, + }), + ); +} + +function printJsonResult(parent: BrowserParentOpts, result: unknown): boolean { + if (!parent.json) { + return false; + } + defaultRuntime.writeJson(result); + return true; +} + +async function callDebugRequest( + parent: BrowserParentOpts, + params: BrowserRequestParams, +): Promise { + return callBrowserRequest(parent, params, { timeoutMs: BROWSER_DEBUG_TIMEOUT_MS }); +} + +function resolveProfileQuery(profile?: string) { + return profile ? { profile } : undefined; +} + +function resolveDebugQuery(params: { + targetId?: unknown; + clear?: unknown; + profile?: string; + filter?: unknown; +}) { + return { + targetId: typeof params.targetId === "string" ? params.targetId.trim() || undefined : undefined, + filter: typeof params.filter === "string" ? params.filter.trim() || undefined : undefined, + clear: Boolean(params.clear), + profile: params.profile, + }; +} + +export function registerBrowserDebugCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + browser + .command("highlight") + .description("Highlight an element by ref") + .argument("", "Ref id from snapshot") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (ref: string, opts, cmd) => { + await withDebugContext(cmd, parentOpts, async ({ parent, profile }) => { + const result = await callDebugRequest(parent, { + method: "POST", + path: "/highlight", + query: resolveProfileQuery(profile), + body: { + ref: ref.trim(), + targetId: opts.targetId?.trim() || undefined, + }, + }); + if (printJsonResult(parent, result)) { + return; + } + defaultRuntime.log(`highlighted ${ref.trim()}`); + }); + }); + + browser + .command("errors") + .description("Get recent page errors") + .option("--clear", "Clear stored errors after reading", false) + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + await withDebugContext(cmd, parentOpts, async ({ parent, profile }) => { + const result = await callDebugRequest<{ + errors: Array<{ timestamp: string; name?: string; message: string }>; + }>(parent, { + method: "GET", + path: "/errors", + query: resolveDebugQuery({ + targetId: opts.targetId, + clear: opts.clear, + profile, + }), + }); + if (printJsonResult(parent, result)) { + return; + } + if (!result.errors.length) { + defaultRuntime.log("No page errors."); + return; + } + defaultRuntime.log( + result.errors + .map((e) => `${e.timestamp} ${e.name ? `${e.name}: ` : ""}${e.message}`) + .join("\n"), + ); + }); + }); + + browser + .command("requests") + .description("Get recent network requests (best-effort)") + .option("--filter ", "Only show URLs that contain this substring") + .option("--clear", "Clear stored requests after reading", false) + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + await withDebugContext(cmd, parentOpts, async ({ parent, profile }) => { + const result = await callDebugRequest<{ + requests: Array<{ + timestamp: string; + method: string; + status?: number; + ok?: boolean; + url: string; + failureText?: string; + }>; + }>(parent, { + method: "GET", + path: "/requests", + query: resolveDebugQuery({ + targetId: opts.targetId, + filter: opts.filter, + clear: opts.clear, + profile, + }), + }); + if (printJsonResult(parent, result)) { + return; + } + if (!result.requests.length) { + defaultRuntime.log("No requests recorded."); + return; + } + defaultRuntime.log( + result.requests + .map((r) => { + const status = typeof r.status === "number" ? ` ${r.status}` : ""; + const ok = r.ok === true ? " ok" : r.ok === false ? " fail" : ""; + const fail = r.failureText ? ` (${r.failureText})` : ""; + return `${r.timestamp} ${r.method}${status}${ok} ${r.url}${fail}`; + }) + .join("\n"), + ); + }); + }); + + const trace = browser.command("trace").description("Record a Playwright trace"); + + trace + .command("start") + .description("Start trace recording") + .option("--target-id ", "CDP target id (or unique prefix)") + .option("--no-screenshots", "Disable screenshots") + .option("--no-snapshots", "Disable snapshots") + .option("--sources", "Include sources (bigger traces)", false) + .action(async (opts, cmd) => { + await withDebugContext(cmd, parentOpts, async ({ parent, profile }) => { + const result = await callDebugRequest(parent, { + method: "POST", + path: "/trace/start", + query: resolveProfileQuery(profile), + body: { + targetId: opts.targetId?.trim() || undefined, + screenshots: Boolean(opts.screenshots), + snapshots: Boolean(opts.snapshots), + sources: Boolean(opts.sources), + }, + }); + if (printJsonResult(parent, result)) { + return; + } + defaultRuntime.log("trace started"); + }); + }); + + trace + .command("stop") + .description("Stop trace recording and write a .zip") + .option( + "--out ", + "Output path within openclaw temp dir (e.g. trace.zip or /tmp/openclaw/trace.zip)", + ) + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + await withDebugContext(cmd, parentOpts, async ({ parent, profile }) => { + const result = await callDebugRequest<{ path: string }>(parent, { + method: "POST", + path: "/trace/stop", + query: resolveProfileQuery(profile), + body: { + targetId: opts.targetId?.trim() || undefined, + path: opts.out?.trim() || undefined, + }, + }); + if (printJsonResult(parent, result)) { + return; + } + defaultRuntime.log(`TRACE:${shortenHomePath(result.path)}`); + }); + }); +} diff --git a/extensions/browser/src/cli/browser-cli-examples.ts b/extensions/browser/src/cli/browser-cli-examples.ts new file mode 100644 index 00000000000..7e6df7cd6db --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-examples.ts @@ -0,0 +1,34 @@ +export const browserCoreExamples = [ + "openclaw browser status", + "openclaw browser start", + "openclaw browser stop", + "openclaw browser tabs", + "openclaw browser open https://example.com", + "openclaw browser focus abcd1234", + "openclaw browser close abcd1234", + "openclaw browser screenshot", + "openclaw browser screenshot --full-page", + "openclaw browser screenshot --ref 12", + "openclaw browser snapshot", + "openclaw browser snapshot --format aria --limit 200", + "openclaw browser snapshot --efficient", + "openclaw browser snapshot --labels", +]; + +export const browserActionExamples = [ + "openclaw browser navigate https://example.com", + "openclaw browser resize 1280 720", + "openclaw browser click 12 --double", + 'openclaw browser type 23 "hello" --submit', + "openclaw browser press Enter", + "openclaw browser hover 44", + "openclaw browser drag 10 11", + "openclaw browser select 9 OptionA OptionB", + "openclaw browser upload /tmp/openclaw/uploads/file.pdf", + 'openclaw browser fill --fields \'[{"ref":"1","value":"Ada"}]\'', + "openclaw browser dialog --accept", + 'openclaw browser wait --text "Done"', + "openclaw browser evaluate --fn '(el) => el.textContent' --ref 7", + "openclaw browser console --level error", + "openclaw browser pdf", +]; diff --git a/extensions/browser/src/cli/browser-cli-inspect.ts b/extensions/browser/src/cli/browser-cli-inspect.ts new file mode 100644 index 00000000000..6b12b752f77 --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-inspect.ts @@ -0,0 +1,156 @@ +import type { Command } from "commander"; +import { callBrowserRequest, type BrowserParentOpts } from "./browser-cli-shared.js"; +import { + danger, + defaultRuntime, + loadConfig, + shortenHomePath, + type SnapshotResult, +} from "./core-api.js"; + +export function registerBrowserInspectCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + browser + .command("screenshot") + .description("Capture a screenshot (MEDIA:)") + .argument("[targetId]", "CDP target id (or unique prefix)") + .option("--full-page", "Capture full scrollable page", false) + .option("--ref ", "ARIA ref from ai snapshot") + .option("--element ", "CSS selector for element screenshot") + .option("--type ", "Output type (default: png)", "png") + .action(async (targetId: string | undefined, opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + try { + const result = await callBrowserRequest<{ path: string }>( + parent, + { + method: "POST", + path: "/screenshot", + query: profile ? { profile } : undefined, + body: { + targetId: targetId?.trim() || undefined, + fullPage: Boolean(opts.fullPage), + ref: opts.ref?.trim() || undefined, + element: opts.element?.trim() || undefined, + type: opts.type === "jpeg" ? "jpeg" : "png", + }, + }, + { timeoutMs: 20000 }, + ); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(`MEDIA:${shortenHomePath(result.path)}`); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + + browser + .command("snapshot") + .description("Capture a snapshot (default: ai; aria is the accessibility tree)") + .option("--format ", "Snapshot format (default: ai)", "ai") + .option("--target-id ", "CDP target id (or unique prefix)") + .option("--limit ", "Max nodes (default: 500/800)", (v: string) => Number(v)) + .option("--mode ", "Snapshot preset (efficient)") + .option("--efficient", "Use the efficient snapshot preset", false) + .option("--interactive", "Role snapshot: interactive elements only", false) + .option("--compact", "Role snapshot: compact output", false) + .option("--depth ", "Role snapshot: max depth", (v: string) => Number(v)) + .option("--selector ", "Role snapshot: scope to CSS selector") + .option("--frame ", "Role snapshot: scope to an iframe selector") + .option("--labels", "Include viewport label overlay screenshot", false) + .option("--out ", "Write snapshot to a file") + .action(async (opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + const format = opts.format === "aria" ? "aria" : "ai"; + const configMode = + format === "ai" && loadConfig().browser?.snapshotDefaults?.mode === "efficient" + ? "efficient" + : undefined; + const mode = opts.efficient === true || opts.mode === "efficient" ? "efficient" : configMode; + try { + const query: Record = { + format, + targetId: opts.targetId?.trim() || undefined, + limit: Number.isFinite(opts.limit) ? opts.limit : undefined, + interactive: opts.interactive ? true : undefined, + compact: opts.compact ? true : undefined, + depth: Number.isFinite(opts.depth) ? opts.depth : undefined, + selector: opts.selector?.trim() || undefined, + frame: opts.frame?.trim() || undefined, + labels: opts.labels ? true : undefined, + mode, + profile, + }; + const result = await callBrowserRequest( + parent, + { + method: "GET", + path: "/snapshot", + query, + }, + { timeoutMs: 20000 }, + ); + + if (opts.out) { + const fs = await import("node:fs/promises"); + if (result.format === "ai") { + await fs.writeFile(opts.out, result.snapshot, "utf8"); + } else { + const payload = JSON.stringify(result, null, 2); + await fs.writeFile(opts.out, payload, "utf8"); + } + if (parent?.json) { + defaultRuntime.writeJson({ + ok: true, + out: opts.out, + ...(result.format === "ai" && result.imagePath + ? { imagePath: result.imagePath } + : {}), + }); + } else { + defaultRuntime.log(shortenHomePath(opts.out)); + if (result.format === "ai" && result.imagePath) { + defaultRuntime.log(`MEDIA:${shortenHomePath(result.imagePath)}`); + } + } + return; + } + + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + + if (result.format === "ai") { + defaultRuntime.log(result.snapshot); + if (result.imagePath) { + defaultRuntime.log(`MEDIA:${shortenHomePath(result.imagePath)}`); + } + return; + } + + const nodes = "nodes" in result ? result.nodes : []; + defaultRuntime.log( + nodes + .map((n) => { + const indent = " ".repeat(Math.min(20, n.depth)); + const name = n.name ? ` "${n.name}"` : ""; + const value = n.value ? ` = "${n.value}"` : ""; + return `${indent}- ${n.role}${name}${value}`; + }) + .join("\n"), + ); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); +} diff --git a/extensions/browser/src/cli/browser-cli-manage.ts b/extensions/browser/src/cli/browser-cli-manage.ts new file mode 100644 index 00000000000..c8550a4b106 --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-manage.ts @@ -0,0 +1,534 @@ +import type { Command } from "commander"; +import { runCommandWithRuntime } from "../core-api.js"; +import { callBrowserRequest, type BrowserParentOpts } from "./browser-cli-shared.js"; +import { + danger, + defaultRuntime, + info, + redactCdpUrl, + shortenHomePath, + type BrowserCreateProfileResult, + type BrowserDeleteProfileResult, + type BrowserResetProfileResult, + type BrowserStatus, + type BrowserTab, + type BrowserTransport, + type ProfileStatus, +} from "./core-api.js"; + +const BROWSER_MANAGE_REQUEST_TIMEOUT_MS = 45_000; + +function resolveProfileQuery(profile?: string) { + return profile ? { profile } : undefined; +} + +function printJsonResult(parent: BrowserParentOpts, payload: unknown): boolean { + if (!parent?.json) { + return false; + } + defaultRuntime.writeJson(payload); + return true; +} + +async function callTabAction( + parent: BrowserParentOpts, + profile: string | undefined, + body: { action: "new" | "select" | "close"; index?: number }, +) { + return callBrowserRequest( + parent, + { + method: "POST", + path: "/tabs/action", + query: resolveProfileQuery(profile), + body, + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); +} + +async function fetchBrowserStatus( + parent: BrowserParentOpts, + profile?: string, +): Promise { + return await callBrowserRequest( + parent, + { + method: "GET", + path: "/", + query: resolveProfileQuery(profile), + }, + { + timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS, + }, + ); +} + +async function runBrowserToggle( + parent: BrowserParentOpts, + params: { profile?: string; path: string }, +) { + await callBrowserRequest(parent, { + method: "POST", + path: params.path, + query: resolveProfileQuery(params.profile), + }); + const status = await fetchBrowserStatus(parent, params.profile); + if (printJsonResult(parent, status)) { + return; + } + const name = status.profile ?? "openclaw"; + defaultRuntime.log(info(`🦞 browser [${name}] running: ${status.running}`)); +} + +function runBrowserCommand(action: () => Promise) { + return runCommandWithRuntime(defaultRuntime, action, (err) => { + defaultRuntime.error(danger(String(err as unknown))); + defaultRuntime.exit(1); + }); +} + +function logBrowserTabs(tabs: BrowserTab[], json?: boolean) { + if (json) { + defaultRuntime.writeJson({ tabs }); + return; + } + if (tabs.length === 0) { + defaultRuntime.log("No tabs (browser closed or no targets)."); + return; + } + defaultRuntime.log( + tabs + .map((t, i) => `${i + 1}. ${t.title || "(untitled)"}\n ${t.url}\n id: ${t.targetId}`) + .join("\n"), + ); +} + +function usesChromeMcpTransport(params: { + transport?: BrowserTransport; + driver?: "openclaw" | "existing-session"; +}): boolean { + return params.transport === "chrome-mcp" || params.driver === "existing-session"; +} + +function formatBrowserConnectionSummary(params: { + transport?: BrowserTransport; + driver?: "openclaw" | "existing-session"; + isRemote?: boolean; + cdpPort?: number | null; + cdpUrl?: string | null; + userDataDir?: string | null; +}): string { + if (usesChromeMcpTransport(params)) { + const userDataDir = params.userDataDir ? shortenHomePath(params.userDataDir) : null; + return userDataDir + ? `transport: chrome-mcp, userDataDir: ${userDataDir}` + : "transport: chrome-mcp"; + } + if (params.isRemote) { + return `cdpUrl: ${params.cdpUrl ?? "(unset)"}`; + } + return `port: ${params.cdpPort ?? "(unset)"}`; +} + +export function registerBrowserManageCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + browser + .command("status") + .description("Show browser status") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + await runBrowserCommand(async () => { + const status = await fetchBrowserStatus(parent, parent?.browserProfile); + if (printJsonResult(parent, status)) { + return; + } + const detectedPath = status.detectedExecutablePath ?? status.executablePath; + const detectedDisplay = detectedPath ? shortenHomePath(detectedPath) : "auto"; + defaultRuntime.log( + [ + `profile: ${status.profile ?? "openclaw"}`, + `enabled: ${status.enabled}`, + `running: ${status.running}`, + `transport: ${ + usesChromeMcpTransport(status) ? "chrome-mcp" : (status.transport ?? "cdp") + }`, + ...(!usesChromeMcpTransport(status) + ? [ + `cdpPort: ${status.cdpPort ?? "(unset)"}`, + `cdpUrl: ${redactCdpUrl(status.cdpUrl ?? `http://127.0.0.1:${status.cdpPort}`)}`, + ] + : status.userDataDir + ? [`userDataDir: ${shortenHomePath(status.userDataDir)}`] + : []), + `browser: ${status.chosenBrowser ?? "unknown"}`, + `detectedBrowser: ${status.detectedBrowser ?? "unknown"}`, + `detectedPath: ${detectedDisplay}`, + `profileColor: ${status.color}`, + ...(status.detectError ? [`detectError: ${status.detectError}`] : []), + ].join("\n"), + ); + }); + }); + + browser + .command("start") + .description("Start the browser (no-op if already running)") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + await runBrowserToggle(parent, { profile, path: "/start" }); + }); + }); + + browser + .command("stop") + .description("Stop the browser (best-effort)") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + await runBrowserToggle(parent, { profile, path: "/stop" }); + }); + }); + + browser + .command("reset-profile") + .description("Reset browser profile (moves it to Trash)") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + const result = await callBrowserRequest( + parent, + { + method: "POST", + path: "/reset-profile", + query: resolveProfileQuery(profile), + }, + { timeoutMs: 20000 }, + ); + if (printJsonResult(parent, result)) { + return; + } + if (!result.moved) { + defaultRuntime.log(info(`🦞 browser profile already missing.`)); + return; + } + const dest = result.to ?? result.from; + defaultRuntime.log(info(`🦞 browser profile moved to Trash (${dest})`)); + }); + }); + + browser + .command("tabs") + .description("List open tabs") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + const result = await callBrowserRequest<{ running: boolean; tabs: BrowserTab[] }>( + parent, + { + method: "GET", + path: "/tabs", + query: resolveProfileQuery(profile), + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + const tabs = result.tabs ?? []; + logBrowserTabs(tabs, parent?.json); + }); + }); + + const tab = browser + .command("tab") + .description("Tab shortcuts (index-based)") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + const result = await callBrowserRequest<{ ok: true; tabs: BrowserTab[] }>( + parent, + { + method: "POST", + path: "/tabs/action", + query: resolveProfileQuery(profile), + body: { + action: "list", + }, + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + const tabs = result.tabs ?? []; + logBrowserTabs(tabs, parent?.json); + }); + }); + + tab + .command("new") + .description("Open a new tab (about:blank)") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + const result = await callTabAction(parent, profile, { action: "new" }); + if (printJsonResult(parent, result)) { + return; + } + defaultRuntime.log("opened new tab"); + }); + }); + + tab + .command("select") + .description("Focus tab by index (1-based)") + .argument("", "Tab index (1-based)", (v: string) => Number(v)) + .action(async (index: number, _opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + if (!Number.isFinite(index) || index < 1) { + defaultRuntime.error(danger("index must be a positive number")); + defaultRuntime.exit(1); + return; + } + await runBrowserCommand(async () => { + const result = await callTabAction(parent, profile, { + action: "select", + index: Math.floor(index) - 1, + }); + if (printJsonResult(parent, result)) { + return; + } + defaultRuntime.log(`selected tab ${Math.floor(index)}`); + }); + }); + + tab + .command("close") + .description("Close tab by index (1-based); default: first tab") + .argument("[index]", "Tab index (1-based)", (v: string) => Number(v)) + .action(async (index: number | undefined, _opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + const idx = + typeof index === "number" && Number.isFinite(index) ? Math.floor(index) - 1 : undefined; + if (typeof idx === "number" && idx < 0) { + defaultRuntime.error(danger("index must be >= 1")); + defaultRuntime.exit(1); + return; + } + await runBrowserCommand(async () => { + const result = await callTabAction(parent, profile, { action: "close", index: idx }); + if (printJsonResult(parent, result)) { + return; + } + defaultRuntime.log("closed tab"); + }); + }); + + browser + .command("open") + .description("Open a URL in a new tab") + .argument("", "URL to open") + .action(async (url: string, _opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + const tab = await callBrowserRequest( + parent, + { + method: "POST", + path: "/tabs/open", + query: resolveProfileQuery(profile), + body: { url }, + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + if (printJsonResult(parent, tab)) { + return; + } + defaultRuntime.log(`opened: ${tab.url}\nid: ${tab.targetId}`); + }); + }); + + browser + .command("focus") + .description("Focus a tab by target id (or unique prefix)") + .argument("", "Target id or unique prefix") + .action(async (targetId: string, _opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + await callBrowserRequest( + parent, + { + method: "POST", + path: "/tabs/focus", + query: resolveProfileQuery(profile), + body: { targetId }, + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + if (printJsonResult(parent, { ok: true })) { + return; + } + defaultRuntime.log(`focused tab ${targetId}`); + }); + }); + + browser + .command("close") + .description("Close a tab (target id optional)") + .argument("[targetId]", "Target id or unique prefix (optional)") + .action(async (targetId: string | undefined, _opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + if (targetId?.trim()) { + await callBrowserRequest( + parent, + { + method: "DELETE", + path: `/tabs/${encodeURIComponent(targetId.trim())}`, + query: resolveProfileQuery(profile), + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + } else { + await callBrowserRequest( + parent, + { + method: "POST", + path: "/act", + query: resolveProfileQuery(profile), + body: { kind: "close" }, + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + } + if (printJsonResult(parent, { ok: true })) { + return; + } + defaultRuntime.log("closed tab"); + }); + }); + + // Profile management commands + browser + .command("profiles") + .description("List all browser profiles") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + await runBrowserCommand(async () => { + const result = await callBrowserRequest<{ profiles: ProfileStatus[] }>( + parent, + { + method: "GET", + path: "/profiles", + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + const profiles = result.profiles ?? []; + if (printJsonResult(parent, { profiles })) { + return; + } + if (profiles.length === 0) { + defaultRuntime.log("No profiles configured."); + return; + } + defaultRuntime.log( + profiles + .map((p) => { + const status = p.running ? "running" : "stopped"; + const tabs = p.running ? ` (${p.tabCount} tabs)` : ""; + const def = p.isDefault ? " [default]" : ""; + const loc = formatBrowserConnectionSummary(p); + const remote = p.isRemote ? " [remote]" : ""; + const driver = p.driver !== "openclaw" ? ` [${p.driver}]` : ""; + return `${p.name}: ${status}${tabs}${def}${remote}${driver}\n ${loc}, color: ${p.color}`; + }) + .join("\n"), + ); + }); + }); + + browser + .command("create-profile") + .description("Create a new browser profile") + .requiredOption("--name ", "Profile name (lowercase, numbers, hyphens)") + .option("--color ", "Profile color (hex format, e.g. #0066CC)") + .option("--cdp-url ", "CDP URL for remote Chrome (http/https)") + .option("--user-data-dir ", "User data dir for existing-session Chromium attach") + .option("--driver ", "Profile driver (openclaw|existing-session). Default: openclaw") + .action( + async ( + opts: { + name: string; + color?: string; + cdpUrl?: string; + userDataDir?: string; + driver?: string; + }, + cmd, + ) => { + const parent = parentOpts(cmd); + await runBrowserCommand(async () => { + const result = await callBrowserRequest( + parent, + { + method: "POST", + path: "/profiles/create", + body: { + name: opts.name, + color: opts.color, + cdpUrl: opts.cdpUrl, + userDataDir: opts.userDataDir, + driver: opts.driver === "existing-session" ? "existing-session" : undefined, + }, + }, + { timeoutMs: 10_000 }, + ); + if (printJsonResult(parent, result)) { + return; + } + const loc = ` ${formatBrowserConnectionSummary(result)}`; + defaultRuntime.log( + info( + `🦞 Created profile "${result.profile}"\n${loc}\n color: ${result.color}${ + result.userDataDir ? `\n userDataDir: ${shortenHomePath(result.userDataDir)}` : "" + }${opts.driver === "existing-session" ? "\n driver: existing-session" : ""}`, + ), + ); + }); + }, + ); + + browser + .command("delete-profile") + .description("Delete a browser profile") + .requiredOption("--name ", "Profile name to delete") + .action(async (opts: { name: string }, cmd) => { + const parent = parentOpts(cmd); + await runBrowserCommand(async () => { + const result = await callBrowserRequest( + parent, + { + method: "DELETE", + path: `/profiles/${encodeURIComponent(opts.name)}`, + }, + { timeoutMs: 20_000 }, + ); + if (printJsonResult(parent, result)) { + return; + } + const msg = result.deleted + ? `🦞 Deleted profile "${result.profile}" (user data removed)` + : `🦞 Deleted profile "${result.profile}" (no user data found)`; + defaultRuntime.log(info(msg)); + }); + }); +} diff --git a/extensions/browser/src/cli/browser-cli-resize.ts b/extensions/browser/src/cli/browser-cli-resize.ts new file mode 100644 index 00000000000..ef07772c182 --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-resize.ts @@ -0,0 +1,36 @@ +import { callBrowserResize, type BrowserParentOpts } from "./browser-cli-shared.js"; +import { danger, defaultRuntime } from "./core-api.js"; + +export async function runBrowserResizeWithOutput(params: { + parent: BrowserParentOpts; + profile?: string; + width: number; + height: number; + targetId?: string; + timeoutMs?: number; + successMessage: string; +}): Promise { + const { width, height } = params; + if (!Number.isFinite(width) || !Number.isFinite(height)) { + defaultRuntime.error(danger("width and height must be numbers")); + defaultRuntime.exit(1); + return; + } + + const result = await callBrowserResize( + params.parent, + { + profile: params.profile, + width, + height, + targetId: params.targetId, + }, + { timeoutMs: params.timeoutMs ?? 20000 }, + ); + + if (params.parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(params.successMessage); +} diff --git a/extensions/browser/src/cli/browser-cli-shared.ts b/extensions/browser/src/cli/browser-cli-shared.ts new file mode 100644 index 00000000000..566334e03ff --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-shared.ts @@ -0,0 +1,83 @@ +import { callGatewayFromCli, type GatewayRpcOpts } from "./core-api.js"; + +export type BrowserParentOpts = GatewayRpcOpts & { + json?: boolean; + browserProfile?: string; +}; + +type BrowserRequestParams = { + method: "GET" | "POST" | "DELETE"; + path: string; + query?: Record; + body?: unknown; +}; + +function normalizeQuery(query: BrowserRequestParams["query"]): Record | undefined { + if (!query) { + return undefined; + } + const out: Record = {}; + for (const [key, value] of Object.entries(query)) { + if (value === undefined) { + continue; + } + out[key] = String(value); + } + return Object.keys(out).length ? out : undefined; +} + +export async function callBrowserRequest( + opts: BrowserParentOpts, + params: BrowserRequestParams, + extra?: { timeoutMs?: number; progress?: boolean }, +): Promise { + const resolvedTimeoutMs = + typeof extra?.timeoutMs === "number" && Number.isFinite(extra.timeoutMs) + ? Math.max(1, Math.floor(extra.timeoutMs)) + : typeof opts.timeout === "string" + ? Number.parseInt(opts.timeout, 10) + : undefined; + const resolvedTimeout = + typeof resolvedTimeoutMs === "number" && Number.isFinite(resolvedTimeoutMs) + ? resolvedTimeoutMs + : undefined; + const timeout = typeof resolvedTimeout === "number" ? String(resolvedTimeout) : opts.timeout; + const payload = await callGatewayFromCli( + "browser.request", + { ...opts, timeout }, + { + method: params.method, + path: params.path, + query: normalizeQuery(params.query), + body: params.body, + timeoutMs: resolvedTimeout, + }, + { progress: extra?.progress }, + ); + if (payload === undefined) { + throw new Error("Unexpected browser.request response"); + } + return payload as T; +} + +export async function callBrowserResize( + opts: BrowserParentOpts, + params: { profile?: string; width: number; height: number; targetId?: string }, + extra?: { timeoutMs?: number }, +): Promise { + return callBrowserRequest( + opts, + { + method: "POST", + path: "/act", + query: params.profile ? { profile: params.profile } : undefined, + body: { + kind: "resize", + width: params.width, + height: params.height, + targetId: params.targetId?.trim() || undefined, + }, + }, + extra, + ); +} diff --git a/extensions/browser/src/cli/browser-cli-state.cookies-storage.ts b/extensions/browser/src/cli/browser-cli-state.cookies-storage.ts new file mode 100644 index 00000000000..7f04ebfd07e --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-state.cookies-storage.ts @@ -0,0 +1,227 @@ +import type { Command } from "commander"; +import { callBrowserRequest, type BrowserParentOpts } from "./browser-cli-shared.js"; +import { danger, defaultRuntime, inheritOptionFromParent } from "./core-api.js"; + +function resolveUrl(opts: { url?: string }, command: Command): string | undefined { + if (typeof opts.url === "string" && opts.url.trim()) { + return opts.url.trim(); + } + const inherited = inheritOptionFromParent(command, "url"); + if (typeof inherited === "string" && inherited.trim()) { + return inherited.trim(); + } + return undefined; +} + +function resolveTargetId(rawTargetId: unknown, command: Command): string | undefined { + const local = typeof rawTargetId === "string" ? rawTargetId.trim() : ""; + if (local) { + return local; + } + const inherited = inheritOptionFromParent(command, "targetId"); + if (typeof inherited !== "string") { + return undefined; + } + const trimmed = inherited.trim(); + return trimmed ? trimmed : undefined; +} + +async function runMutationRequest(params: { + parent: BrowserParentOpts; + request: Parameters[1]; + successMessage: string; +}) { + try { + const result = await callBrowserRequest(params.parent, params.request, { timeoutMs: 20000 }); + if (params.parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(params.successMessage); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } +} + +export function registerBrowserCookiesAndStorageCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + const cookies = browser.command("cookies").description("Read/write cookies"); + + cookies + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + const targetId = resolveTargetId(opts.targetId, cmd); + try { + const result = await callBrowserRequest<{ cookies?: unknown[] }>( + parent, + { + method: "GET", + path: "/cookies", + query: { + targetId, + profile, + }, + }, + { timeoutMs: 20000 }, + ); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.writeJson(result.cookies ?? []); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + + cookies + .command("set") + .description("Set a cookie (requires --url or domain+path)") + .argument("", "Cookie name") + .argument("", "Cookie value") + .option("--url ", "Cookie URL scope (recommended)") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (name: string, value: string, opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + const targetId = resolveTargetId(opts.targetId, cmd); + const url = resolveUrl(opts, cmd); + if (!url) { + defaultRuntime.error(danger("Missing required --url option for cookies set")); + defaultRuntime.exit(1); + return; + } + await runMutationRequest({ + parent, + request: { + method: "POST", + path: "/cookies/set", + query: profile ? { profile } : undefined, + body: { + targetId, + cookie: { name, value, url }, + }, + }, + successMessage: `cookie set: ${name}`, + }); + }); + + cookies + .command("clear") + .description("Clear all cookies") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + const targetId = resolveTargetId(opts.targetId, cmd); + await runMutationRequest({ + parent, + request: { + method: "POST", + path: "/cookies/clear", + query: profile ? { profile } : undefined, + body: { + targetId, + }, + }, + successMessage: "cookies cleared", + }); + }); + + const storage = browser.command("storage").description("Read/write localStorage/sessionStorage"); + + function registerStorageKind(kind: "local" | "session") { + const cmd = storage.command(kind).description(`${kind}Storage commands`); + + cmd + .command("get") + .description(`Get ${kind}Storage (all keys or one key)`) + .argument("[key]", "Key (optional)") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (key: string | undefined, opts, cmd2) => { + const parent = parentOpts(cmd2); + const profile = parent?.browserProfile; + const targetId = resolveTargetId(opts.targetId, cmd2); + try { + const result = await callBrowserRequest<{ values?: Record }>( + parent, + { + method: "GET", + path: `/storage/${kind}`, + query: { + key: key?.trim() || undefined, + targetId, + profile, + }, + }, + { timeoutMs: 20000 }, + ); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.writeJson(result.values ?? {}); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + + cmd + .command("set") + .description(`Set a ${kind}Storage key`) + .argument("", "Key") + .argument("", "Value") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (key: string, value: string, opts, cmd2) => { + const parent = parentOpts(cmd2); + const profile = parent?.browserProfile; + const targetId = resolveTargetId(opts.targetId, cmd2); + await runMutationRequest({ + parent, + request: { + method: "POST", + path: `/storage/${kind}/set`, + query: profile ? { profile } : undefined, + body: { + key, + value, + targetId, + }, + }, + successMessage: `${kind}Storage set: ${key}`, + }); + }); + + cmd + .command("clear") + .description(`Clear all ${kind}Storage keys`) + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (opts, cmd2) => { + const parent = parentOpts(cmd2); + const profile = parent?.browserProfile; + const targetId = resolveTargetId(opts.targetId, cmd2); + await runMutationRequest({ + parent, + request: { + method: "POST", + path: `/storage/${kind}/clear`, + query: profile ? { profile } : undefined, + body: { + targetId, + }, + }, + successMessage: `${kind}Storage cleared`, + }); + }); + } + + registerStorageKind("local"); + registerStorageKind("session"); +} diff --git a/extensions/browser/src/cli/browser-cli-state.ts b/extensions/browser/src/cli/browser-cli-state.ts new file mode 100644 index 00000000000..8814d88f54c --- /dev/null +++ b/extensions/browser/src/cli/browser-cli-state.ts @@ -0,0 +1,274 @@ +import type { Command } from "commander"; +import { runCommandWithRuntime } from "../core-api.js"; +import { runBrowserResizeWithOutput } from "./browser-cli-resize.js"; +import { callBrowserRequest, type BrowserParentOpts } from "./browser-cli-shared.js"; +import { registerBrowserCookiesAndStorageCommands } from "./browser-cli-state.cookies-storage.js"; +import { danger, defaultRuntime, parseBooleanValue } from "./core-api.js"; + +function parseOnOff(raw: string): boolean | null { + const parsed = parseBooleanValue(raw); + return parsed === undefined ? null : parsed; +} + +function runBrowserCommand(action: () => Promise) { + return runCommandWithRuntime(defaultRuntime, action, (err) => { + defaultRuntime.error(danger(String(err as unknown))); + defaultRuntime.exit(1); + }); +} + +async function runBrowserSetRequest(params: { + parent: BrowserParentOpts; + path: string; + body: Record; + successMessage: string; +}) { + await runBrowserCommand(async () => { + const profile = params.parent?.browserProfile; + const result = await callBrowserRequest( + params.parent, + { + method: "POST", + path: params.path, + query: profile ? { profile } : undefined, + body: params.body, + }, + { timeoutMs: 20000 }, + ); + if (params.parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(params.successMessage); + }); +} + +export function registerBrowserStateCommands( + browser: Command, + parentOpts: (cmd: Command) => BrowserParentOpts, +) { + registerBrowserCookiesAndStorageCommands(browser, parentOpts); + + const set = browser.command("set").description("Browser environment settings"); + + set + .command("viewport") + .description("Set viewport size (alias for resize)") + .argument("", "Viewport width", (v: string) => Number(v)) + .argument("", "Viewport height", (v: string) => Number(v)) + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (width: number, height: number, opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + await runBrowserResizeWithOutput({ + parent, + profile, + width, + height, + targetId: opts.targetId, + timeoutMs: 20000, + successMessage: `viewport set: ${width}x${height}`, + }); + }); + }); + + set + .command("offline") + .description("Toggle offline mode") + .argument("", "on/off") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (value: string, opts, cmd) => { + const parent = parentOpts(cmd); + const offline = parseOnOff(value); + if (offline === null) { + defaultRuntime.error(danger("Expected on|off")); + defaultRuntime.exit(1); + return; + } + await runBrowserSetRequest({ + parent, + path: "/set/offline", + body: { + offline, + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: `offline: ${offline}`, + }); + }); + + set + .command("headers") + .description("Set extra HTTP headers (JSON object)") + .argument("[headersJson]", "JSON object of headers (alternative to --headers-json)") + .option("--headers-json ", "JSON object of headers") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (headersJson: string | undefined, opts, cmd) => { + const parent = parentOpts(cmd); + await runBrowserCommand(async () => { + const headersJsonValue = + (typeof opts.headersJson === "string" && opts.headersJson.trim()) || + (headersJson?.trim() ? headersJson.trim() : undefined); + if (!headersJsonValue) { + throw new Error("Missing headers JSON (pass --headers-json or positional JSON argument)"); + } + const parsed = JSON.parse(String(headersJsonValue)) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("Headers JSON must be a JSON object"); + } + const headers: Record = {}; + for (const [k, v] of Object.entries(parsed as Record)) { + if (typeof v === "string") { + headers[k] = v; + } + } + const profile = parent?.browserProfile; + const result = await callBrowserRequest( + parent, + { + method: "POST", + path: "/set/headers", + query: profile ? { profile } : undefined, + body: { + headers, + targetId: opts.targetId?.trim() || undefined, + }, + }, + { timeoutMs: 20000 }, + ); + if (parent?.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log("headers set"); + }); + }); + + set + .command("credentials") + .description("Set HTTP basic auth credentials") + .option("--clear", "Clear credentials", false) + .argument("[username]", "Username") + .argument("[password]", "Password") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (username: string | undefined, password: string | undefined, opts, cmd) => { + const parent = parentOpts(cmd); + await runBrowserSetRequest({ + parent, + path: "/set/credentials", + body: { + username: username?.trim() || undefined, + password, + clear: Boolean(opts.clear), + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: opts.clear ? "credentials cleared" : "credentials set", + }); + }); + + set + .command("geo") + .description("Set geolocation (and grant permission)") + .option("--clear", "Clear geolocation + permissions", false) + .argument("[latitude]", "Latitude", (v: string) => Number(v)) + .argument("[longitude]", "Longitude", (v: string) => Number(v)) + .option("--accuracy ", "Accuracy in meters", (v: string) => Number(v)) + .option("--origin ", "Origin to grant permissions for") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (latitude: number | undefined, longitude: number | undefined, opts, cmd) => { + const parent = parentOpts(cmd); + await runBrowserSetRequest({ + parent, + path: "/set/geolocation", + body: { + latitude: Number.isFinite(latitude) ? latitude : undefined, + longitude: Number.isFinite(longitude) ? longitude : undefined, + accuracy: Number.isFinite(opts.accuracy) ? opts.accuracy : undefined, + origin: opts.origin?.trim() || undefined, + clear: Boolean(opts.clear), + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: opts.clear ? "geolocation cleared" : "geolocation set", + }); + }); + + set + .command("media") + .description("Emulate prefers-color-scheme") + .argument("", "dark/light/none") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (value: string, opts, cmd) => { + const parent = parentOpts(cmd); + const v = value.trim().toLowerCase(); + const colorScheme = + v === "dark" ? "dark" : v === "light" ? "light" : v === "none" ? "none" : null; + if (!colorScheme) { + defaultRuntime.error(danger("Expected dark|light|none")); + defaultRuntime.exit(1); + return; + } + await runBrowserSetRequest({ + parent, + path: "/set/media", + body: { + colorScheme, + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: `media colorScheme: ${colorScheme}`, + }); + }); + + set + .command("timezone") + .description("Override timezone (CDP)") + .argument("", "Timezone ID (e.g. America/New_York)") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (timezoneId: string, opts, cmd) => { + const parent = parentOpts(cmd); + await runBrowserSetRequest({ + parent, + path: "/set/timezone", + body: { + timezoneId, + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: `timezone: ${timezoneId}`, + }); + }); + + set + .command("locale") + .description("Override locale (CDP)") + .argument("", "Locale (e.g. en-US)") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (locale: string, opts, cmd) => { + const parent = parentOpts(cmd); + await runBrowserSetRequest({ + parent, + path: "/set/locale", + body: { + locale, + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: `locale: ${locale}`, + }); + }); + + set + .command("device") + .description('Apply a Playwright device descriptor (e.g. "iPhone 14")') + .argument("", "Device name (Playwright devices)") + .option("--target-id ", "CDP target id (or unique prefix)") + .action(async (name: string, opts, cmd) => { + const parent = parentOpts(cmd); + await runBrowserSetRequest({ + parent, + path: "/set/device", + body: { + name, + targetId: opts.targetId?.trim() || undefined, + }, + successMessage: `device: ${name}`, + }); + }); +} diff --git a/extensions/browser/src/cli/browser-cli.ts b/extensions/browser/src/cli/browser-cli.ts new file mode 100644 index 00000000000..25e989fb6ff --- /dev/null +++ b/extensions/browser/src/cli/browser-cli.ts @@ -0,0 +1,55 @@ +import type { Command } from "commander"; +import { registerBrowserActionInputCommands } from "./browser-cli-actions-input.js"; +import { registerBrowserActionObserveCommands } from "./browser-cli-actions-observe.js"; +import { registerBrowserDebugCommands } from "./browser-cli-debug.js"; +import { browserActionExamples, browserCoreExamples } from "./browser-cli-examples.js"; +import { registerBrowserInspectCommands } from "./browser-cli-inspect.js"; +import { registerBrowserManageCommands } from "./browser-cli-manage.js"; +import type { BrowserParentOpts } from "./browser-cli-shared.js"; +import { registerBrowserStateCommands } from "./browser-cli-state.js"; +import { + addGatewayClientOptions, + danger, + defaultRuntime, + formatCliCommand, + formatDocsLink, + formatHelpExamples, + theme, +} from "./core-api.js"; + +export function registerBrowserCli(program: Command) { + const browser = program + .command("browser") + .description("Manage OpenClaw's dedicated browser (Chrome/Chromium)") + .option("--browser-profile ", "Browser profile name (default from config)") + .option("--json", "Output machine-readable JSON", false) + .addHelpText( + "after", + () => + `\n${theme.heading("Examples:")}\n${formatHelpExamples( + [...browserCoreExamples, ...browserActionExamples].map((cmd) => [cmd, ""]), + true, + )}\n\n${theme.muted("Docs:")} ${formatDocsLink( + "/cli/browser", + "docs.openclaw.ai/cli/browser", + )}\n`, + ) + .action(() => { + browser.outputHelp(); + defaultRuntime.error( + danger(`Missing subcommand. Try: "${formatCliCommand("openclaw browser status")}"`), + ); + defaultRuntime.exit(1); + }); + + addGatewayClientOptions(browser); + + const parentOpts = (cmd: Command) => cmd.parent?.opts?.() as BrowserParentOpts; + + registerBrowserManageCommands(browser, parentOpts); + registerBrowserInspectCommands(browser, parentOpts); + registerBrowserActionInputCommands(browser, parentOpts); + registerBrowserActionObserveCommands(browser, parentOpts); + registerBrowserDebugCommands(browser, parentOpts); + registerBrowserStateCommands(browser, parentOpts); +} diff --git a/extensions/browser/src/cli/command-format.ts b/extensions/browser/src/cli/command-format.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/cli/command-format.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/cli/core-api.ts b/extensions/browser/src/cli/core-api.ts new file mode 100644 index 00000000000..5f56b004d49 --- /dev/null +++ b/extensions/browser/src/cli/core-api.ts @@ -0,0 +1 @@ +export * from "../core-api.js"; diff --git a/extensions/browser/src/control-service.ts b/extensions/browser/src/control-service.ts new file mode 100644 index 00000000000..5da98cade41 --- /dev/null +++ b/extensions/browser/src/control-service.ts @@ -0,0 +1,68 @@ +import { createSubsystemLogger, loadConfig } from "openclaw/plugin-sdk/browser-support"; +import { resolveBrowserConfig } from "./browser/config.js"; +import { ensureBrowserControlAuth } from "./browser/control-auth.js"; +import { createBrowserRuntimeState, stopBrowserRuntime } from "./browser/runtime-lifecycle.js"; +import { type BrowserServerState, createBrowserRouteContext } from "./browser/server-context.js"; +import { isDefaultBrowserPluginEnabled } from "./plugin-enabled.js"; + +let state: BrowserServerState | null = null; +const log = createSubsystemLogger("browser"); +const logService = log.child("service"); + +export function getBrowserControlState(): BrowserServerState | null { + return state; +} + +export function createBrowserControlContext() { + return createBrowserRouteContext({ + getState: () => state, + refreshConfigFromDisk: true, + }); +} + +export async function startBrowserControlServiceFromConfig(): Promise { + if (state) { + return state; + } + + const cfg = loadConfig(); + if (!isDefaultBrowserPluginEnabled(cfg)) { + return null; + } + const resolved = resolveBrowserConfig(cfg.browser, cfg); + if (!resolved.enabled) { + return null; + } + try { + const ensured = await ensureBrowserControlAuth({ cfg }); + if (ensured.generatedToken) { + logService.info("No browser auth configured; generated gateway.auth.token automatically."); + } + } catch (err) { + logService.warn(`failed to auto-configure browser auth: ${String(err)}`); + } + + state = await createBrowserRuntimeState({ + server: null, + port: resolved.controlPort, + resolved, + onWarn: (message) => logService.warn(message), + }); + + logService.info( + `Browser control service ready (profiles=${Object.keys(resolved.profiles).length})`, + ); + return state; +} + +export async function stopBrowserControlService(): Promise { + const current = state; + await stopBrowserRuntime({ + current, + getState: () => state, + clearState: () => { + state = null; + }, + onWarn: (message) => logService.warn(message), + }); +} diff --git a/extensions/browser/src/core-api.ts b/extensions/browser/src/core-api.ts new file mode 100644 index 00000000000..6dc5dd6b157 --- /dev/null +++ b/extensions/browser/src/core-api.ts @@ -0,0 +1,111 @@ +export { + DEFAULT_AI_SNAPSHOT_MAX_CHARS, + DEFAULT_UPLOAD_DIR, + applyBrowserProxyPaths, + browserAct, + browserArmDialog, + browserArmFileChooser, + browserCloseTab, + browserCreateProfile, + browserConsoleMessages, + browserDeleteProfile, + browserFocusTab, + browserNavigate, + browserOpenTab, + browserPdfSave, + browserProfiles, + browserResetProfile, + browserScreenshotAction, + browserSnapshot, + browserStart, + browserStatus, + browserStop, + browserTabAction, + browserTabs, + createBrowserControlContext, + createBrowserRouteDispatcher, + createBrowserRuntimeState, + createBrowserRouteContext, + ensureBrowserControlAuth, + getBrowserControlState, + getBrowserProfileCapabilities, + isPersistentBrowserProfileMutation, + installBrowserAuthMiddleware, + installBrowserCommonMiddleware, + normalizeBrowserFormField, + normalizeBrowserFormFieldValue, + normalizeBrowserRequestPath, + persistBrowserProxyFiles, + redactCdpUrl, + registerBrowserRoutes, + resolveBrowserConfig, + resolveBrowserControlAuth, + resolveExistingPathsWithinRoot, + resolveProfile, + resolveRequestedBrowserProfile, + startBrowserControlServiceFromConfig, + stopBrowserControlService, + stopBrowserRuntime, + trackSessionBrowserTab, + untrackSessionBrowserTab, +} from "./browser-runtime.js"; +export type { + BrowserCreateProfileResult, + BrowserDeleteProfileResult, + BrowserFormField, + BrowserResetProfileResult, + BrowserRouteRegistrar, + BrowserServerState, + BrowserStatus, + BrowserTab, + BrowserTransport, + ProfileStatus, + SnapshotResult, +} from "./browser-runtime.js"; +export { + callGatewayTool, + createSubsystemLogger, + danger, + defaultRuntime, + detectMime, + ErrorCodes, + errorShape, + formatCliCommand, + formatDocsLink, + formatHelpExamples, + addGatewayClientOptions, + callGatewayFromCli, + inheritOptionFromParent, + info, + imageResultFromFile, + isNodeCommandAllowed, + jsonResult, + listNodes, + loadConfig, + normalizePluginsConfig, + optionalStringEnum, + parseBooleanValue, + readStringParam, + respondUnavailableOnNodeInvokeError, + resolveEffectiveEnableState, + resolveNodeIdFromList, + resolveNodeCommandAllowlist, + runCommandWithRuntime, + selectDefaultNodeFromList, + safeParseJson, + shortenHomePath, + startBrowserControlServerIfEnabled, + stringEnum, + theme, + withTimeout, + wrapExternalContent, +} from "openclaw/plugin-sdk/browser-support"; +export type { + AnyAgentTool, + GatewayRequestHandlers, + GatewayRpcOpts, + NodeListNode, + NodeSession, + OpenClawConfig, + OpenClawPluginService, +} from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/gateway/auth.ts b/extensions/browser/src/gateway/auth.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/gateway/auth.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/gateway/browser-request.ts b/extensions/browser/src/gateway/browser-request.ts new file mode 100644 index 00000000000..639bacda32e --- /dev/null +++ b/extensions/browser/src/gateway/browser-request.ts @@ -0,0 +1,270 @@ +import crypto from "node:crypto"; +import { + ErrorCodes, + applyBrowserProxyPaths, + createBrowserControlContext, + createBrowserRouteDispatcher, + errorShape, + isNodeCommandAllowed, + isPersistentBrowserProfileMutation, + loadConfig, + persistBrowserProxyFiles, + resolveNodeCommandAllowlist, + resolveRequestedBrowserProfile, + respondUnavailableOnNodeInvokeError, + safeParseJson, + startBrowserControlServiceFromConfig, + type GatewayRequestHandlers, + type NodeSession, +} from "../core-api.js"; + +type BrowserRequestParams = { + method?: string; + path?: string; + query?: Record; + body?: unknown; + timeoutMs?: number; +}; + +type BrowserProxyFile = { + path: string; + base64: string; + mimeType?: string; +}; + +type BrowserProxyResult = { + result: unknown; + files?: BrowserProxyFile[]; +}; + +function isBrowserNode(node: NodeSession) { + const caps = Array.isArray(node.caps) ? node.caps : []; + const commands = Array.isArray(node.commands) ? node.commands : []; + return caps.includes("browser") || commands.includes("browser.proxy"); +} + +function normalizeNodeKey(value: string) { + return value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, ""); +} + +function resolveBrowserNode(nodes: NodeSession[], query: string): NodeSession | null { + const q = query.trim(); + if (!q) { + return null; + } + const qNorm = normalizeNodeKey(q); + const matches = nodes.filter((node) => { + if (node.nodeId === q) { + return true; + } + if (typeof node.remoteIp === "string" && node.remoteIp === q) { + return true; + } + const name = typeof node.displayName === "string" ? node.displayName : ""; + if (name && normalizeNodeKey(name) === qNorm) { + return true; + } + if (q.length >= 6 && node.nodeId.startsWith(q)) { + return true; + } + return false; + }); + if (matches.length === 1) { + return matches[0] ?? null; + } + if (matches.length === 0) { + return null; + } + throw new Error( + `ambiguous node: ${q} (matches: ${matches + .map((node) => node.displayName || node.remoteIp || node.nodeId) + .join(", ")})`, + ); +} + +function resolveBrowserNodeTarget(params: { + cfg: ReturnType; + nodes: NodeSession[]; +}): NodeSession | null { + const policy = params.cfg.gateway?.nodes?.browser; + const mode = policy?.mode ?? "auto"; + if (mode === "off") { + return null; + } + const browserNodes = params.nodes.filter((node) => isBrowserNode(node)); + if (browserNodes.length === 0) { + if (policy?.node?.trim()) { + throw new Error("No connected browser-capable nodes."); + } + return null; + } + const requested = policy?.node?.trim() || ""; + if (requested) { + const resolved = resolveBrowserNode(browserNodes, requested); + if (!resolved) { + throw new Error(`Configured browser node not connected: ${requested}`); + } + return resolved; + } + if (mode === "manual") { + return null; + } + if (browserNodes.length === 1) { + return browserNodes[0] ?? null; + } + return null; +} + +async function persistProxyFiles(files: BrowserProxyFile[] | undefined) { + return await persistBrowserProxyFiles(files); +} + +function applyProxyPaths(result: unknown, mapping: Map) { + applyBrowserProxyPaths(result, mapping); +} + +export async function handleBrowserGatewayRequest({ + params, + respond, + context, +}: Parameters[0]) { + const typed = params as BrowserRequestParams; + const methodRaw = typeof typed.method === "string" ? typed.method.trim().toUpperCase() : ""; + const path = typeof typed.path === "string" ? typed.path.trim() : ""; + const query = typed.query && typeof typed.query === "object" ? typed.query : undefined; + const body = typed.body; + const timeoutMs = + typeof typed.timeoutMs === "number" && Number.isFinite(typed.timeoutMs) + ? Math.max(1, Math.floor(typed.timeoutMs)) + : undefined; + + if (!methodRaw || !path) { + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, "method and path are required"), + ); + return; + } + if (methodRaw !== "GET" && methodRaw !== "POST" && methodRaw !== "DELETE") { + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, "method must be GET, POST, or DELETE"), + ); + return; + } + if (isPersistentBrowserProfileMutation(methodRaw, path)) { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + "browser.request cannot mutate persistent browser profiles", + ), + ); + return; + } + + const cfg = loadConfig(); + let nodeTarget: NodeSession | null = null; + try { + nodeTarget = resolveBrowserNodeTarget({ + cfg, + nodes: context.nodeRegistry.listConnected(), + }); + } catch (err) { + respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, String(err))); + return; + } + + if (nodeTarget) { + const allowlist = resolveNodeCommandAllowlist(cfg, nodeTarget); + const allowed = isNodeCommandAllowed({ + command: "browser.proxy", + declaredCommands: nodeTarget.commands, + allowlist, + }); + if (!allowed.ok) { + const platform = nodeTarget.platform ?? "unknown"; + const hint = `node command not allowed: ${allowed.reason} (platform: ${platform}, command: browser.proxy)`; + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, hint, { + details: { reason: allowed.reason, command: "browser.proxy" }, + }), + ); + return; + } + + const proxyParams = { + method: methodRaw, + path, + query, + body, + timeoutMs, + profile: resolveRequestedBrowserProfile({ query, body }), + }; + const res = await context.nodeRegistry.invoke({ + nodeId: nodeTarget.nodeId, + command: "browser.proxy", + params: proxyParams, + timeoutMs, + idempotencyKey: crypto.randomUUID(), + }); + if (!respondUnavailableOnNodeInvokeError(respond, res)) { + return; + } + const payload = res.payloadJSON ? safeParseJson(res.payloadJSON) : res.payload; + const proxy = payload && typeof payload === "object" ? (payload as BrowserProxyResult) : null; + if (!proxy || !("result" in proxy)) { + respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "browser proxy failed")); + return; + } + const mapping = await persistProxyFiles(proxy.files); + applyProxyPaths(proxy.result, mapping); + respond(true, proxy.result); + return; + } + + const ready = await startBrowserControlServiceFromConfig(); + if (!ready) { + respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "browser control is disabled")); + return; + } + + let dispatcher; + try { + dispatcher = createBrowserRouteDispatcher(createBrowserControlContext()); + } catch (err) { + respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, String(err))); + return; + } + + const result = await dispatcher.dispatch({ + method: methodRaw, + path, + query, + body, + }); + + if (result.status >= 400) { + const message = + result.body && typeof result.body === "object" && "error" in result.body + ? String((result.body as { error?: unknown }).error) + : `browser request failed (${result.status})`; + const code = result.status >= 500 ? ErrorCodes.UNAVAILABLE : ErrorCodes.INVALID_REQUEST; + respond(false, undefined, errorShape(code, message, { details: result.body })); + return; + } + + respond(true, result.body); +} + +export const browserHandlers: GatewayRequestHandlers = { + "browser.request": handleBrowserGatewayRequest, +}; diff --git a/extensions/browser/src/gateway/net.ts b/extensions/browser/src/gateway/net.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/gateway/net.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/gateway/startup-auth.ts b/extensions/browser/src/gateway/startup-auth.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/gateway/startup-auth.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/errors.ts b/extensions/browser/src/infra/errors.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/errors.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/fs-safe.ts b/extensions/browser/src/infra/fs-safe.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/fs-safe.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/net/proxy-env.ts b/extensions/browser/src/infra/net/proxy-env.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/net/proxy-env.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/net/ssrf.ts b/extensions/browser/src/infra/net/ssrf.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/net/ssrf.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/path-guards.ts b/extensions/browser/src/infra/path-guards.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/path-guards.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/ports.ts b/extensions/browser/src/infra/ports.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/ports.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/secure-random.ts b/extensions/browser/src/infra/secure-random.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/secure-random.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/tmp-openclaw-dir.ts b/extensions/browser/src/infra/tmp-openclaw-dir.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/tmp-openclaw-dir.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/infra/ws.ts b/extensions/browser/src/infra/ws.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/infra/ws.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/logging/redact.ts b/extensions/browser/src/logging/redact.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/logging/redact.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/logging/subsystem.ts b/extensions/browser/src/logging/subsystem.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/logging/subsystem.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/media/image-ops.ts b/extensions/browser/src/media/image-ops.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/media/image-ops.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/media/store.ts b/extensions/browser/src/media/store.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/media/store.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/node-host/invoke-browser.ts b/extensions/browser/src/node-host/invoke-browser.ts new file mode 100644 index 00000000000..3b4a1d2eb30 --- /dev/null +++ b/extensions/browser/src/node-host/invoke-browser.ts @@ -0,0 +1,351 @@ +import fsPromises from "node:fs/promises"; +import { + createBrowserControlContext, + createBrowserRouteDispatcher, + detectMime, + isPersistentBrowserProfileMutation, + loadConfig, + normalizeBrowserRequestPath, + redactCdpUrl, + resolveBrowserConfig, + resolveRequestedBrowserProfile, + startBrowserControlServiceFromConfig, + withTimeout, +} from "../core-api.js"; + +type BrowserProxyParams = { + method?: string; + path?: string; + query?: Record; + body?: unknown; + timeoutMs?: number; + profile?: string; +}; + +type BrowserProxyFile = { + path: string; + base64: string; + mimeType?: string; +}; + +type BrowserProxyResult = { + result: unknown; + files?: BrowserProxyFile[]; +}; + +const BROWSER_PROXY_MAX_FILE_BYTES = 10 * 1024 * 1024; +const DEFAULT_BROWSER_PROXY_TIMEOUT_MS = 20_000; +const BROWSER_PROXY_STATUS_TIMEOUT_MS = 750; + +function normalizeProfileAllowlist(raw?: string[]): string[] { + return Array.isArray(raw) ? raw.map((entry) => entry.trim()).filter(Boolean) : []; +} + +function resolveBrowserProxyConfig() { + const cfg = loadConfig(); + const proxy = cfg.nodeHost?.browserProxy; + const allowProfiles = normalizeProfileAllowlist(proxy?.allowProfiles); + const enabled = proxy?.enabled !== false; + return { enabled, allowProfiles }; +} + +let browserControlReady: Promise | null = null; + +async function ensureBrowserControlService(): Promise { + if (browserControlReady) { + return browserControlReady; + } + browserControlReady = (async () => { + const cfg = loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + if (!resolved.enabled) { + throw new Error("browser control disabled"); + } + const started = await startBrowserControlServiceFromConfig(); + if (!started) { + throw new Error("browser control disabled"); + } + })(); + return browserControlReady; +} + +function isProfileAllowed(params: { allowProfiles: string[]; profile?: string | null }) { + const { allowProfiles, profile } = params; + if (!allowProfiles.length) { + return true; + } + if (!profile) { + return false; + } + return allowProfiles.includes(profile.trim()); +} + +function collectBrowserProxyPaths(payload: unknown): string[] { + const paths = new Set(); + const obj = + typeof payload === "object" && payload !== null ? (payload as Record) : null; + if (!obj) { + return []; + } + if (typeof obj.path === "string" && obj.path.trim()) { + paths.add(obj.path.trim()); + } + if (typeof obj.imagePath === "string" && obj.imagePath.trim()) { + paths.add(obj.imagePath.trim()); + } + const download = obj.download; + if (download && typeof download === "object") { + const dlPath = (download as Record).path; + if (typeof dlPath === "string" && dlPath.trim()) { + paths.add(dlPath.trim()); + } + } + return [...paths]; +} + +async function readBrowserProxyFile(filePath: string): Promise { + const stat = await fsPromises.stat(filePath).catch(() => null); + if (!stat || !stat.isFile()) { + return null; + } + if (stat.size > BROWSER_PROXY_MAX_FILE_BYTES) { + throw new Error( + `browser proxy file exceeds ${Math.round(BROWSER_PROXY_MAX_FILE_BYTES / (1024 * 1024))}MB`, + ); + } + const buffer = await fsPromises.readFile(filePath); + const mimeType = await detectMime({ buffer, filePath }); + return { path: filePath, base64: buffer.toString("base64"), mimeType }; +} + +function decodeParams(raw?: string | null): T { + if (!raw) { + throw new Error("INVALID_REQUEST: paramsJSON required"); + } + return JSON.parse(raw) as T; +} + +function resolveBrowserProxyTimeout(timeoutMs?: number): number { + return typeof timeoutMs === "number" && Number.isFinite(timeoutMs) + ? Math.max(1, Math.floor(timeoutMs)) + : DEFAULT_BROWSER_PROXY_TIMEOUT_MS; +} + +function isBrowserProxyTimeoutError(err: unknown): boolean { + return String(err).includes("browser proxy request timed out"); +} + +function isWsBackedBrowserProxyPath(path: string): boolean { + return ( + path === "/act" || + path === "/navigate" || + path === "/pdf" || + path === "/screenshot" || + path === "/snapshot" + ); +} + +async function readBrowserProxyStatus(params: { + dispatcher: ReturnType; + profile?: string; +}): Promise | null> { + const query = params.profile ? { profile: params.profile } : {}; + try { + const response = await withTimeout( + (signal) => + params.dispatcher.dispatch({ + method: "GET", + path: "/", + query, + signal, + }), + BROWSER_PROXY_STATUS_TIMEOUT_MS, + "browser proxy status", + ); + if (response.status >= 400 || !response.body || typeof response.body !== "object") { + return null; + } + const body = response.body as Record; + return { + running: body.running, + transport: body.transport, + cdpHttp: body.cdpHttp, + cdpReady: body.cdpReady, + cdpUrl: body.cdpUrl, + }; + } catch { + return null; + } +} + +function formatBrowserProxyTimeoutMessage(params: { + method: string; + path: string; + profile?: string; + timeoutMs: number; + wsBacked: boolean; + status: Record | null; +}): string { + const parts = [ + `browser proxy timed out for ${params.method} ${params.path} after ${params.timeoutMs}ms`, + params.wsBacked ? "ws-backed browser action" : "browser action", + ]; + if (params.profile) { + parts.push(`profile=${params.profile}`); + } + if (params.status) { + const statusParts = [ + `running=${String(params.status.running)}`, + `cdpHttp=${String(params.status.cdpHttp)}`, + `cdpReady=${String(params.status.cdpReady)}`, + ]; + if (typeof params.status.transport === "string" && params.status.transport.trim()) { + statusParts.push(`transport=${params.status.transport}`); + } + if (typeof params.status.cdpUrl === "string" && params.status.cdpUrl.trim()) { + statusParts.push(`cdpUrl=${redactCdpUrl(params.status.cdpUrl)}`); + } + parts.push(`status(${statusParts.join(", ")})`); + } + return parts.join("; "); +} + +export async function runBrowserProxyCommand(paramsJSON?: string | null): Promise { + const params = decodeParams(paramsJSON); + const pathValue = typeof params.path === "string" ? params.path.trim() : ""; + if (!pathValue) { + throw new Error("INVALID_REQUEST: path required"); + } + const proxyConfig = resolveBrowserProxyConfig(); + if (!proxyConfig.enabled) { + throw new Error("UNAVAILABLE: node browser proxy disabled"); + } + + await ensureBrowserControlService(); + const cfg = loadConfig(); + const resolved = resolveBrowserConfig(cfg.browser, cfg); + const method = typeof params.method === "string" ? params.method.toUpperCase() : "GET"; + const path = normalizeBrowserRequestPath(pathValue); + const body = params.body; + const requestedProfile = + resolveRequestedBrowserProfile({ + query: params.query, + body, + profile: params.profile, + }) ?? ""; + const allowedProfiles = proxyConfig.allowProfiles; + if (allowedProfiles.length > 0) { + if (isPersistentBrowserProfileMutation(method, path)) { + throw new Error( + "INVALID_REQUEST: browser.proxy cannot mutate persistent browser profiles when allowProfiles is configured", + ); + } + if (path !== "/profiles") { + const profileToCheck = requestedProfile || resolved.defaultProfile; + if (!isProfileAllowed({ allowProfiles: allowedProfiles, profile: profileToCheck })) { + throw new Error("INVALID_REQUEST: browser profile not allowed"); + } + } else if (requestedProfile) { + if (!isProfileAllowed({ allowProfiles: allowedProfiles, profile: requestedProfile })) { + throw new Error("INVALID_REQUEST: browser profile not allowed"); + } + } + } + + const timeoutMs = resolveBrowserProxyTimeout(params.timeoutMs); + const query: Record = {}; + const rawQuery = params.query ?? {}; + for (const [key, value] of Object.entries(rawQuery)) { + if (value === undefined || value === null) { + continue; + } + query[key] = typeof value === "string" ? value : String(value); + } + if (requestedProfile) { + query.profile = requestedProfile; + } + + const dispatcher = createBrowserRouteDispatcher(createBrowserControlContext()); + let response; + try { + response = await withTimeout( + (signal) => + dispatcher.dispatch({ + method: method === "DELETE" ? "DELETE" : method === "POST" ? "POST" : "GET", + path, + query, + body, + signal, + }), + timeoutMs, + "browser proxy request", + ); + } catch (err) { + if (!isBrowserProxyTimeoutError(err)) { + throw err; + } + const profileForStatus = requestedProfile || resolved.defaultProfile; + const status = await readBrowserProxyStatus({ + dispatcher, + profile: path === "/profiles" ? undefined : profileForStatus, + }); + throw new Error( + formatBrowserProxyTimeoutMessage({ + method, + path, + profile: path === "/profiles" ? undefined : profileForStatus || undefined, + timeoutMs, + wsBacked: isWsBackedBrowserProxyPath(path), + status, + }), + { cause: err }, + ); + } + if (response.status >= 400) { + const message = + response.body && typeof response.body === "object" && "error" in response.body + ? String((response.body as { error?: unknown }).error) + : `HTTP ${response.status}`; + throw new Error(message); + } + + const result = response.body; + if (allowedProfiles.length > 0 && path === "/profiles") { + const obj = + typeof result === "object" && result !== null ? (result as Record) : {}; + const profiles = Array.isArray(obj.profiles) ? obj.profiles : []; + obj.profiles = profiles.filter((entry) => { + if (!entry || typeof entry !== "object") { + return false; + } + const name = (entry as Record).name; + return typeof name === "string" && allowedProfiles.includes(name); + }); + } + + let files: BrowserProxyFile[] | undefined; + const paths = collectBrowserProxyPaths(result); + if (paths.length > 0) { + const loaded = await Promise.all( + paths.map(async (p) => { + try { + const file = await readBrowserProxyFile(p); + if (!file) { + throw new Error("file not found"); + } + return file; + } catch (err) { + throw new Error(`browser proxy file read failed for ${p}: ${String(err)}`, { + cause: err, + }); + } + }), + ); + if (loaded.length > 0) { + files = loaded; + } + } + + const payload: BrowserProxyResult = files ? { result, files } : { result }; + return JSON.stringify(payload); +} diff --git a/extensions/browser/src/plugin-enabled.ts b/extensions/browser/src/plugin-enabled.ts new file mode 100644 index 00000000000..2b7db20e980 --- /dev/null +++ b/extensions/browser/src/plugin-enabled.ts @@ -0,0 +1,15 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/browser-support"; +import { + normalizePluginsConfig, + resolveEffectiveEnableState, +} from "openclaw/plugin-sdk/browser-support"; + +export function isDefaultBrowserPluginEnabled(cfg: OpenClawConfig): boolean { + return resolveEffectiveEnableState({ + id: "browser", + origin: "bundled", + config: normalizePluginsConfig(cfg.plugins), + rootConfig: cfg, + enabledByDefault: true, + }).enabled; +} diff --git a/extensions/browser/src/plugin-service.ts b/extensions/browser/src/plugin-service.ts new file mode 100644 index 00000000000..8b4b0d51f67 --- /dev/null +++ b/extensions/browser/src/plugin-service.ts @@ -0,0 +1,28 @@ +import { + startBrowserControlServerIfEnabled, + type OpenClawPluginService, +} from "openclaw/plugin-sdk/browser-support"; + +type BrowserControlHandle = Awaited>; + +export function createBrowserPluginService(): OpenClawPluginService { + let handle: BrowserControlHandle = null; + + return { + id: "browser-control", + start: async () => { + if (handle) { + return; + } + handle = await startBrowserControlServerIfEnabled(); + }, + stop: async () => { + const current = handle; + handle = null; + if (!current) { + return; + } + await current.stop().catch(() => {}); + }, + }; +} diff --git a/extensions/browser/src/process/exec.ts b/extensions/browser/src/process/exec.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/process/exec.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/security/secret-equal.ts b/extensions/browser/src/security/secret-equal.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/security/secret-equal.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/server.ts b/extensions/browser/src/server.ts new file mode 100644 index 00000000000..341c438b57c --- /dev/null +++ b/extensions/browser/src/server.ts @@ -0,0 +1,102 @@ +import type { Server } from "node:http"; +import express from "express"; +import { createSubsystemLogger, loadConfig } from "openclaw/plugin-sdk/browser-support"; +import { resolveBrowserConfig } from "./browser/config.js"; +import { ensureBrowserControlAuth, resolveBrowserControlAuth } from "./browser/control-auth.js"; +import { registerBrowserRoutes } from "./browser/routes/index.js"; +import type { BrowserRouteRegistrar } from "./browser/routes/types.js"; +import { createBrowserRuntimeState, stopBrowserRuntime } from "./browser/runtime-lifecycle.js"; +import { type BrowserServerState, createBrowserRouteContext } from "./browser/server-context.js"; +import { + installBrowserAuthMiddleware, + installBrowserCommonMiddleware, +} from "./browser/server-middleware.js"; +import { isDefaultBrowserPluginEnabled } from "./plugin-enabled.js"; + +let state: BrowserServerState | null = null; +const log = createSubsystemLogger("browser"); +const logServer = log.child("server"); + +export async function startBrowserControlServerFromConfig(): Promise { + if (state) { + return state; + } + + const cfg = loadConfig(); + if (!isDefaultBrowserPluginEnabled(cfg)) { + return null; + } + const resolved = resolveBrowserConfig(cfg.browser, cfg); + if (!resolved.enabled) { + return null; + } + + let browserAuth = resolveBrowserControlAuth(cfg); + let browserAuthBootstrapFailed = false; + try { + const ensured = await ensureBrowserControlAuth({ cfg }); + browserAuth = ensured.auth; + if (ensured.generatedToken) { + logServer.info("No browser auth configured; generated gateway.auth.token automatically."); + } + } catch (err) { + logServer.warn(`failed to auto-configure browser auth: ${String(err)}`); + browserAuthBootstrapFailed = true; + } + + // Fail closed: if auth bootstrap failed and no explicit auth is available, + // do not start the browser control HTTP server. + if (browserAuthBootstrapFailed && !browserAuth.token && !browserAuth.password) { + logServer.error( + "browser control startup aborted: authentication bootstrap failed and no fallback auth is configured.", + ); + return null; + } + + const app = express(); + installBrowserCommonMiddleware(app); + installBrowserAuthMiddleware(app, browserAuth); + + const ctx = createBrowserRouteContext({ + getState: () => state, + refreshConfigFromDisk: true, + }); + registerBrowserRoutes(app as unknown as BrowserRouteRegistrar, ctx); + + const port = resolved.controlPort; + const server = await new Promise((resolve, reject) => { + const s = app.listen(port, "127.0.0.1", () => resolve(s)); + s.once("error", reject); + }).catch((err) => { + logServer.error(`openclaw browser server failed to bind 127.0.0.1:${port}: ${String(err)}`); + return null; + }); + + if (!server) { + return null; + } + + state = await createBrowserRuntimeState({ + server, + port, + resolved, + onWarn: (message) => logServer.warn(message), + }); + + const authMode = browserAuth.token ? "token" : browserAuth.password ? "password" : "off"; + logServer.info(`Browser control listening on http://127.0.0.1:${port}/ (auth=${authMode})`); + return state; +} + +export async function stopBrowserControlServer(): Promise { + const current = state; + await stopBrowserRuntime({ + current, + getState: () => state, + clearState: () => { + state = null; + }, + closeServer: true, + onWarn: (message) => logServer.warn(message), + }); +} diff --git a/extensions/browser/src/test-utils/fetch-mock.ts b/extensions/browser/src/test-utils/fetch-mock.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/test-utils/fetch-mock.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/test-utils/vitest-mock-fn.ts b/extensions/browser/src/test-utils/vitest-mock-fn.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/test-utils/vitest-mock-fn.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/utils.ts b/extensions/browser/src/utils.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/utils.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/extensions/browser/src/utils/boolean.ts b/extensions/browser/src/utils/boolean.ts new file mode 100644 index 00000000000..700c049c4f3 --- /dev/null +++ b/extensions/browser/src/utils/boolean.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/browser-support"; diff --git a/package.json b/package.json index 006808c3926..63f46efc14a 100644 --- a/package.json +++ b/package.json @@ -245,6 +245,18 @@ "types": "./dist/plugin-sdk/bluebubbles.d.ts", "default": "./dist/plugin-sdk/bluebubbles.js" }, + "./plugin-sdk/browser": { + "types": "./dist/plugin-sdk/browser.d.ts", + "default": "./dist/plugin-sdk/browser.js" + }, + "./plugin-sdk/browser-runtime": { + "types": "./dist/plugin-sdk/browser-runtime.d.ts", + "default": "./dist/plugin-sdk/browser-runtime.js" + }, + "./plugin-sdk/browser-support": { + "types": "./dist/plugin-sdk/browser-support.d.ts", + "default": "./dist/plugin-sdk/browser-support.js" + }, "./plugin-sdk/boolean-param": { "types": "./dist/plugin-sdk/boolean-param.d.ts", "default": "./dist/plugin-sdk/boolean-param.js" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a0f449ffab..10a77f0c561 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -261,6 +261,8 @@ importers: extensions/brave: {} + extensions/browser: {} + extensions/byteplus: {} extensions/chutes: {} diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index f3de2a2ab96..aa595bde40f 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -51,6 +51,9 @@ "allow-from", "allowlist-config-edit", "bluebubbles", + "browser", + "browser-runtime", + "browser-support", "boolean-param", "command-auth", "device-bootstrap", diff --git a/src/plugin-sdk/browser-runtime.ts b/src/plugin-sdk/browser-runtime.ts new file mode 100644 index 00000000000..f41b3d8d207 --- /dev/null +++ b/src/plugin-sdk/browser-runtime.ts @@ -0,0 +1 @@ +export * from "../../extensions/browser/src/browser-runtime.js"; diff --git a/src/plugin-sdk/browser-support.ts b/src/plugin-sdk/browser-support.ts new file mode 100644 index 00000000000..8377259651d --- /dev/null +++ b/src/plugin-sdk/browser-support.ts @@ -0,0 +1,89 @@ +export { loadConfig } from "../config/config.js"; +export { + createConfigIO, + getRuntimeConfigSnapshot, + writeConfigFile, + type BrowserConfig, + type BrowserProfileConfig, +} from "../config/config.js"; +export { resolveGatewayPort } from "../config/paths.js"; +export { + DEFAULT_BROWSER_CONTROL_PORT, + deriveDefaultBrowserCdpPortRange, + deriveDefaultBrowserControlPort, +} from "../config/port-defaults.js"; +export { createSubsystemLogger } from "../logging/subsystem.js"; +export { redactSensitiveText } from "../logging/redact.js"; +export { detectMime } from "../media/mime.js"; +export { + IMAGE_REDUCE_QUALITY_STEPS, + buildImageResizeSideGrid, + getImageMetadata, + resizeToJpeg, +} from "../media/image-ops.js"; +export { ensureMediaDir, saveMediaBuffer } from "../media/store.js"; +export { normalizePluginsConfig, resolveEffectiveEnableState } from "../plugins/config-state.js"; +export type { OpenClawPluginService } from "../plugins/types.js"; +export { startBrowserControlServerIfEnabled } from "../gateway/server-browser.js"; +export { resolveGatewayAuth } from "../gateway/auth.js"; +export { isLoopbackHost } from "../gateway/net.js"; +export { ensureGatewayStartupAuth } from "../gateway/startup-auth.js"; +export type { AnyAgentTool } from "../agents/tools/common.js"; +export { imageResultFromFile, jsonResult, readStringParam } from "../agents/tools/common.js"; +export { callGatewayTool } from "../agents/tools/gateway.js"; +export type { NodeListNode } from "../agents/tools/nodes-utils.js"; +export { + listNodes, + resolveNodeIdFromList, + selectDefaultNodeFromList, +} from "../agents/tools/nodes-utils.js"; +export { danger, info } from "../globals.js"; +export { defaultRuntime } from "../runtime.js"; +export { wrapExternalContent } from "../security/external-content.js"; +export { safeEqualSecret } from "../security/secret-equal.js"; +export { optionalStringEnum, stringEnum } from "../agents/schema/typebox.js"; +export { formatDocsLink } from "../terminal/links.js"; +export { theme } from "../terminal/theme.js"; +export { CONFIG_DIR, escapeRegExp, resolveUserPath, shortenHomePath } from "../utils.js"; +export { parseBooleanValue } from "../utils/boolean.js"; +export { formatCliCommand } from "../cli/command-format.js"; +export { runCommandWithRuntime } from "../cli/cli-utils.js"; +export { inheritOptionFromParent } from "../cli/command-options.js"; +export { addGatewayClientOptions, callGatewayFromCli } from "../cli/gateway-rpc.js"; +export type { GatewayRpcOpts } from "../cli/gateway-rpc.js"; +export { formatHelpExamples } from "../cli/help-format.js"; +export { withTimeout } from "../node-host/with-timeout.js"; +export { + isNodeCommandAllowed, + resolveNodeCommandAllowlist, +} from "../gateway/node-command-policy.js"; +export type { NodeSession } from "../gateway/node-registry.js"; +export { ErrorCodes, errorShape } from "../gateway/protocol/index.js"; +export { + respondUnavailableOnNodeInvokeError, + safeParseJson, +} from "../gateway/server-methods/nodes.helpers.js"; +export type { GatewayRequestHandlers } from "../gateway/server-methods/types.js"; +export type { OpenClawConfig } from "../config/config.js"; +export { extractErrorCode, formatErrorMessage } from "../infra/errors.js"; +export { + SafeOpenError, + openFileWithinRoot, + writeFileFromPathWithinRoot, +} from "../infra/fs-safe.js"; +export { hasProxyEnvConfigured } from "../infra/net/proxy-env.js"; +export { + SsrFBlockedError, + isPrivateNetworkAllowedByPolicy, + resolvePinnedHostnameWithPolicy, + type LookupFn, + type SsrFPolicy, +} from "../infra/net/ssrf.js"; +export { isNotFoundPathError, isPathInside } from "../infra/path-guards.js"; +export { ensurePortAvailable } from "../infra/ports.js"; +export { generateSecureToken } from "../infra/secure-random.js"; +export { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; +export { rawDataToString } from "../infra/ws.js"; +export { runExec } from "../process/exec.js"; +export { withFetchPreconnect } from "../test-utils/fetch-mock.js"; +export type { MockFn } from "../test-utils/vitest-mock-fn.js"; diff --git a/src/plugin-sdk/browser.ts b/src/plugin-sdk/browser.ts new file mode 100644 index 00000000000..561aaefd745 --- /dev/null +++ b/src/plugin-sdk/browser.ts @@ -0,0 +1,4 @@ +export { createBrowserTool } from "../../extensions/browser/src/browser-tool.js"; +export { registerBrowserCli } from "../../extensions/browser/src/cli/browser-cli.js"; +export { createBrowserPluginService } from "../../extensions/browser/src/plugin-service.js"; +export { handleBrowserGatewayRequest } from "../../extensions/browser/src/gateway/browser-request.js"; diff --git a/src/plugin-sdk/plugin-entry.ts b/src/plugin-sdk/plugin-entry.ts index 3493281cf00..5d264d89c3e 100644 --- a/src/plugin-sdk/plugin-entry.ts +++ b/src/plugin-sdk/plugin-entry.ts @@ -11,6 +11,8 @@ export type { AnyAgentTool, MediaUnderstandingProviderPlugin, OpenClawPluginApi, + OpenClawPluginToolContext, + OpenClawPluginToolFactory, PluginCommandContext, OpenClawPluginConfigSchema, ProviderDiscoveryContext, diff --git a/src/plugins/registry-empty.ts b/src/plugins/registry-empty.ts index 3b54f66d0fd..b29ac2cee17 100644 --- a/src/plugins/registry-empty.ts +++ b/src/plugins/registry-empty.ts @@ -15,6 +15,7 @@ export function createEmptyPluginRegistry(): PluginRegistry { imageGenerationProviders: [], webSearchProviders: [], gatewayHandlers: {}, + gatewayMethodScopes: {}, httpRoutes: [], cliRegistrars: [], services: [], diff --git a/src/plugins/registry.ts b/src/plugins/registry.ts index d6aa648e982..19a85f728d4 100644 --- a/src/plugins/registry.ts +++ b/src/plugins/registry.ts @@ -2,6 +2,7 @@ import path from "node:path"; import type { AnyAgentTool } from "../agents/tools/common.js"; import type { ChannelPlugin } from "../channels/plugins/types.js"; import { registerContextEngineForOwner } from "../context-engine/registry.js"; +import type { OperatorScope } from "../gateway/method-scopes.js"; import type { GatewayRequestHandler, GatewayRequestHandlers, @@ -220,6 +221,7 @@ export type PluginRegistry = { imageGenerationProviders: PluginImageGenerationProviderRegistration[]; webSearchProviders: PluginWebSearchProviderRegistration[]; gatewayHandlers: GatewayRequestHandlers; + gatewayMethodScopes?: Partial>; httpRoutes: PluginHttpRouteRegistration[]; cliRegistrars: PluginCliRegistration[]; services: PluginServiceRegistration[]; @@ -378,6 +380,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { record: PluginRecord, method: string, handler: GatewayRequestHandler, + opts?: { scope?: OperatorScope }, ) => { const trimmed = method.trim(); if (!trimmed) { @@ -393,6 +396,10 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { return; } registry.gatewayHandlers[trimmed] = handler; + if (opts?.scope) { + registry.gatewayMethodScopes ??= {}; + registry.gatewayMethodScopes[trimmed] = opts.scope; + } record.gatewayMethods.push(trimmed); }; @@ -984,7 +991,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { : () => {}, registerGatewayMethod: registrationMode === "full" - ? (method, handler) => registerGatewayMethod(record, method, handler) + ? (method, handler, opts) => registerGatewayMethod(record, method, handler, opts) : () => {}, registerCli: registrationMode === "full" diff --git a/src/plugins/types.ts b/src/plugins/types.ts index 254a4c9fa47..7ef2a8053f6 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -22,6 +22,7 @@ import type { } from "../channels/plugins/types.js"; import type { OpenClawConfig } from "../config/config.js"; import type { CliBackendConfig, ModelProviderConfig } from "../config/types.js"; +import type { OperatorScope } from "../gateway/method-scopes.js"; import type { GatewayRequestHandler } from "../gateway/server-methods/types.js"; import type { InternalHookHandler } from "../hooks/internal-hooks.js"; import type { HookEntry } from "../hooks/types.js"; @@ -108,6 +109,10 @@ export type OpenClawPluginToolContext = { sessionKey?: string; /** Ephemeral session UUID — regenerated on /new and /reset. Use for per-conversation isolation. */ sessionId?: string; + browser?: { + sandboxBridgeUrl?: string; + allowHostControl?: boolean; + }; messageChannel?: string; agentAccountId?: string; /** Trusted sender id from inbound context (runtime-provided, not tool args). */ @@ -1369,7 +1374,11 @@ export type OpenClawPluginApi = { registerHttpRoute: (params: OpenClawPluginHttpRouteParams) => void; /** Register a native messaging channel plugin (channel capability). */ registerChannel: (registration: OpenClawPluginChannelRegistration | ChannelPlugin) => void; - registerGatewayMethod: (method: string, handler: GatewayRequestHandler) => void; + registerGatewayMethod: ( + method: string, + handler: GatewayRequestHandler, + opts?: { scope?: OperatorScope }, + ) => void; registerCli: ( registrar: OpenClawPluginCliRegistrar, opts?: {