mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-24 10:11:14 +00:00
f81f9d8570
* ci: enforce max-lines suppression ratchet * chore: prune stale max-lines suppression * fix: close max-lines ratchet enforcement gaps * fix: harden max-lines ratchet checks * fix(ci): satisfy max-lines ratchet checks * style: format max-lines declarations * fix(ci): match oxlint suppression grammar * test: isolate max-lines git fixtures * chore: prune resolved max-lines debt * test: skip newline path fixture on Windows * fix: harden max-lines suppression ratchet * chore: refresh max-lines baseline * fix: close max-lines ratchet bypasses * fix: derive ratchet base from PR merge tree * fix: support older Git in staged ratchet * fix: align max-lines declarations and baseline * chore: refresh max-lines baseline for current main * fix: exclude generated wizard locales from max-lines * chore: prune resolved max-lines debt
1017 lines
34 KiB
TypeScript
1017 lines
34 KiB
TypeScript
/**
|
|
* Browser agent tool registration.
|
|
*
|
|
* Builds the model-facing browser tool, chooses sandbox/host/node routing, and
|
|
* maps high-level actions onto browser control client calls.
|
|
*/
|
|
import { createBrowserNodeProxyRequest } from "./browser-node-proxy.js";
|
|
import { describeBrowserTool } from "./browser-tool-description.js";
|
|
import {
|
|
executeActAction,
|
|
executeConsoleAction,
|
|
executeDownloadAction,
|
|
executeSnapshotAction,
|
|
executeTabsAction,
|
|
} from "./browser-tool.actions.js";
|
|
import {
|
|
type AnyAgentTool,
|
|
type NodeListNode,
|
|
BrowserToolSchema,
|
|
browserAct,
|
|
browserArmDialog,
|
|
browserArmFileChooser,
|
|
browserCloseTab,
|
|
browserDoctor,
|
|
browserFocusTab,
|
|
browserImportProfile,
|
|
browserNavigate,
|
|
browserOpenTab,
|
|
browserPdfSave,
|
|
browserProfiles,
|
|
browserSystemProfiles,
|
|
browserScreenshotAction,
|
|
browserStart,
|
|
browserStatus,
|
|
browserStop,
|
|
describeImageFile,
|
|
getRuntimeConfig,
|
|
getBrowserProfileCapabilities,
|
|
imageResultFromFile,
|
|
jsonResult,
|
|
listNodes,
|
|
normalizeOptionalString,
|
|
readPositiveIntegerParam,
|
|
readStringParam,
|
|
readStringValue,
|
|
resolveBrowserConfig,
|
|
resolveExistingUploadPaths,
|
|
resolveRuntimeImageSanitization,
|
|
resolveNodeIdFromList,
|
|
resolveProfile,
|
|
saveMediaBuffer,
|
|
selectDefaultNodeFromList,
|
|
stageBrowserScreenshotForSharing,
|
|
touchSessionBrowserTab,
|
|
trackSessionBrowserTab,
|
|
untrackSessionBrowserTab,
|
|
} from "./browser-tool.runtime.js";
|
|
import { DEFAULT_BROWSER_SCREENSHOT_TIMEOUT_MS } from "./browser/constants.js";
|
|
import { parseBrowserNavigationUrl } from "./browser/navigation-guard.js";
|
|
import { normalizeBrowserScreenshot } from "./browser/screenshot.js";
|
|
import { parseSystemProfileDomains } from "./browser/system-profile-domains.js";
|
|
import { describeBrowserScreenshot, neutralizeMediaDirectives } from "./browser/vision.js";
|
|
import { wrapExternalContent } from "./sdk-security-runtime.js";
|
|
|
|
const browserToolDeps = {
|
|
browserAct,
|
|
browserArmDialog,
|
|
browserArmFileChooser,
|
|
browserCloseTab,
|
|
browserDoctor,
|
|
browserFocusTab,
|
|
browserImportProfile,
|
|
browserNavigate,
|
|
browserOpenTab,
|
|
browserPdfSave,
|
|
browserProfiles,
|
|
browserSystemProfiles,
|
|
browserScreenshotAction,
|
|
browserStart,
|
|
browserStatus,
|
|
browserStop,
|
|
describeImageFile,
|
|
getRuntimeConfig,
|
|
imageResultFromFile,
|
|
listNodes,
|
|
normalizeBrowserScreenshot,
|
|
saveMediaBuffer,
|
|
stageBrowserScreenshotForSharing,
|
|
touchSessionBrowserTab,
|
|
trackSessionBrowserTab,
|
|
untrackSessionBrowserTab,
|
|
};
|
|
|
|
function readOptionalTargetAndTimeout(params: Record<string, unknown>) {
|
|
const targetId = normalizeOptionalString(params.targetId);
|
|
const timeoutMs = readPositiveIntegerParam(params, "timeoutMs", {
|
|
message: "timeoutMs must be a positive integer.",
|
|
});
|
|
return { targetId, timeoutMs };
|
|
}
|
|
|
|
function readTargetUrlParam(params: Record<string, unknown>) {
|
|
const targetUrl =
|
|
readStringParam(params, "targetUrl") ??
|
|
readStringParam(params, "url", { required: true, label: "targetUrl" });
|
|
parseBrowserNavigationUrl(targetUrl);
|
|
return targetUrl;
|
|
}
|
|
|
|
function formatScreenshotShareHint(filePath: string): string {
|
|
return `[Screenshot saved to ${JSON.stringify(filePath)}. Use this path with the message tool to share the screenshot explicitly.]`;
|
|
}
|
|
|
|
const SCREENSHOT_SHARE_UNAVAILABLE =
|
|
"[Screenshot sharing is unavailable because an outbound copy could not be prepared.]";
|
|
|
|
const LEGACY_BROWSER_ACT_REQUEST_KEYS = [
|
|
"kind",
|
|
"actions",
|
|
"stopOnError",
|
|
"targetId",
|
|
"ref",
|
|
"doubleClick",
|
|
"button",
|
|
"modifiers",
|
|
"x",
|
|
"y",
|
|
"text",
|
|
"submit",
|
|
"slowly",
|
|
"key",
|
|
"delayMs",
|
|
"startRef",
|
|
"endRef",
|
|
"values",
|
|
"fields",
|
|
"width",
|
|
"height",
|
|
"timeMs",
|
|
"textGone",
|
|
"selector",
|
|
"url",
|
|
"loadState",
|
|
"fn",
|
|
"timeoutMs",
|
|
] as const;
|
|
|
|
const LEGACY_BROWSER_ACT_SHARED_REQUEST_KEYS = new Set<
|
|
(typeof LEGACY_BROWSER_ACT_REQUEST_KEYS)[number]
|
|
>(["targetId"]);
|
|
|
|
function readActRequestParam(params: Record<string, unknown>) {
|
|
const requestParam = params.request;
|
|
if (requestParam && typeof requestParam === "object") {
|
|
const request = { ...(requestParam as Record<string, unknown>) };
|
|
const hasMismatchedKind =
|
|
typeof request.kind === "string" &&
|
|
typeof params.kind === "string" &&
|
|
request.kind !== params.kind;
|
|
for (const key of LEGACY_BROWSER_ACT_REQUEST_KEYS) {
|
|
if (Object.hasOwn(request, key) || !Object.hasOwn(params, key)) {
|
|
continue;
|
|
}
|
|
// Flattened act fields are legacy shape repair. Only the tab scope is
|
|
// safe across kind mismatches; action-specific fields can corrupt the
|
|
// explicit nested request.
|
|
if (hasMismatchedKind && !LEGACY_BROWSER_ACT_SHARED_REQUEST_KEYS.has(key)) {
|
|
continue;
|
|
}
|
|
request[key] = params[key];
|
|
}
|
|
return request as Parameters<typeof browserAct>[1];
|
|
}
|
|
|
|
const kind = readStringParam(params, "kind");
|
|
if (!kind) {
|
|
return undefined;
|
|
}
|
|
|
|
const request: Record<string, unknown> = {};
|
|
for (const key of LEGACY_BROWSER_ACT_REQUEST_KEYS) {
|
|
if (!Object.hasOwn(params, key)) {
|
|
continue;
|
|
}
|
|
request[key] = params[key];
|
|
}
|
|
return request as Parameters<typeof browserAct>[1];
|
|
}
|
|
|
|
type BrowserNodeTarget = {
|
|
nodeId: string;
|
|
label?: string;
|
|
};
|
|
|
|
function isBrowserNode(node: NodeListNode) {
|
|
const caps = Array.isArray(node.caps) ? node.caps : [];
|
|
const commands = Array.isArray(node.commands) ? node.commands : [];
|
|
return caps.includes("browser") || commands.includes("browser.proxy");
|
|
}
|
|
|
|
async function resolveBrowserNodeTarget(params: {
|
|
requestedNode?: string;
|
|
target?: "sandbox" | "host" | "node";
|
|
sandboxBridgeUrl?: string;
|
|
allowHostControl?: boolean;
|
|
}): Promise<BrowserNodeTarget | null> {
|
|
if (params.allowHostControl === false) {
|
|
if (params.target === "node" || params.requestedNode) {
|
|
throw new Error("Node browser control is disabled by sandbox policy.");
|
|
}
|
|
return null;
|
|
}
|
|
|
|
const cfg = browserToolDeps.getRuntimeConfig();
|
|
const policy = cfg.gateway?.nodes?.browser;
|
|
const mode = policy?.mode ?? "auto";
|
|
if (mode === "off") {
|
|
if (params.target === "node" || params.requestedNode) {
|
|
throw new Error("Node browser proxy is disabled (gateway.nodes.browser.mode=off).");
|
|
}
|
|
return null;
|
|
}
|
|
if (params.sandboxBridgeUrl?.trim() && params.target !== "node" && !params.requestedNode) {
|
|
return null;
|
|
}
|
|
if (params.target && params.target !== "node") {
|
|
return null;
|
|
}
|
|
if (mode === "manual" && params.target !== "node" && !params.requestedNode) {
|
|
return null;
|
|
}
|
|
|
|
const nodes = await browserToolDeps.listNodes({});
|
|
const browserNodes = nodes.filter((node) => node.connected && isBrowserNode(node));
|
|
if (browserNodes.length === 0) {
|
|
if (params.target === "node" || params.requestedNode) {
|
|
throw new Error("No connected browser-capable nodes.");
|
|
}
|
|
return null;
|
|
}
|
|
|
|
const requested = params.requestedNode?.trim() || policy?.node?.trim();
|
|
if (requested) {
|
|
const nodeId = resolveNodeIdFromList(browserNodes, requested, false, {
|
|
allowCompactDisplayName: true,
|
|
});
|
|
const node = browserNodes.find((entry) => entry.nodeId === nodeId);
|
|
return { nodeId, label: node?.displayName ?? node?.remoteIp ?? nodeId };
|
|
}
|
|
|
|
const selected = selectDefaultNodeFromList(browserNodes, {
|
|
preferLocalMac: false,
|
|
fallback: "none",
|
|
});
|
|
|
|
if (params.target === "node") {
|
|
if (selected) {
|
|
return {
|
|
nodeId: selected.nodeId,
|
|
label: selected.displayName ?? selected.remoteIp ?? selected.nodeId,
|
|
};
|
|
}
|
|
throw new Error(
|
|
`Multiple browser-capable nodes connected (${browserNodes.length}). Set gateway.nodes.browser.node or pass node=<id>.`,
|
|
);
|
|
}
|
|
|
|
if (mode === "manual") {
|
|
return null;
|
|
}
|
|
|
|
if (selected) {
|
|
return {
|
|
nodeId: selected.nodeId,
|
|
label: selected.displayName ?? selected.remoteIp ?? selected.nodeId,
|
|
};
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function resolveBrowserBaseUrl(params: {
|
|
target?: "sandbox" | "host";
|
|
sandboxBridgeUrl?: string;
|
|
allowHostControl?: boolean;
|
|
}): string | undefined {
|
|
const cfg = getRuntimeConfig();
|
|
const resolved = resolveBrowserConfig(cfg.browser, cfg);
|
|
const normalizedSandbox = params.sandboxBridgeUrl?.trim() ?? "";
|
|
const target = params.target ?? (normalizedSandbox ? "sandbox" : "host");
|
|
|
|
if (target === "sandbox") {
|
|
if (!normalizedSandbox) {
|
|
throw new Error(
|
|
'Sandbox browser is unavailable. Enable agents.defaults.sandbox.browser.enabled or use target="host" if allowed.',
|
|
);
|
|
}
|
|
return normalizedSandbox.replace(/\/$/, "");
|
|
}
|
|
|
|
if (params.allowHostControl === false) {
|
|
throw new Error("Host browser control is disabled by sandbox policy.");
|
|
}
|
|
if (!resolved.enabled) {
|
|
throw new Error(
|
|
"Browser control is disabled. Set browser.enabled=true in ~/.openclaw/openclaw.json.",
|
|
);
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
/**
|
|
* Read importable system profiles from the host control server. Discovery must
|
|
* match where import runs (host-local), so it never uses a node proxy or the
|
|
* sandbox base URL. Returns [] when host control is unavailable.
|
|
*/
|
|
async function readHostSystemProfiles(params: {
|
|
allowHostControl?: boolean;
|
|
sandboxBridgeUrl?: string;
|
|
timeoutMs?: number;
|
|
}) {
|
|
if (params.allowHostControl === false) {
|
|
return [];
|
|
}
|
|
let hostBaseUrl: string | undefined;
|
|
try {
|
|
hostBaseUrl = resolveBrowserBaseUrl({
|
|
target: "host",
|
|
sandboxBridgeUrl: params.sandboxBridgeUrl,
|
|
allowHostControl: params.allowHostControl,
|
|
});
|
|
} catch {
|
|
return [];
|
|
}
|
|
return await browserToolDeps
|
|
.browserSystemProfiles(hostBaseUrl, { timeoutMs: params.timeoutMs })
|
|
.catch(() => []);
|
|
}
|
|
|
|
function shouldPreferHostForProfile(profileName: string | undefined) {
|
|
if (!profileName) {
|
|
return false;
|
|
}
|
|
const cfg = browserToolDeps.getRuntimeConfig();
|
|
const resolved = resolveBrowserConfig(cfg.browser, cfg);
|
|
const profile = resolveProfile(resolved, profileName);
|
|
if (!profile) {
|
|
return false;
|
|
}
|
|
const capabilities = getBrowserProfileCapabilities(profile);
|
|
return capabilities.usesChromeMcp;
|
|
}
|
|
|
|
const DEFAULT_EXISTING_SESSION_MANAGE_TIMEOUT_MS = 45_000;
|
|
const EXISTING_SESSION_MANAGE_ACTIONS = new Set([
|
|
"status",
|
|
"start",
|
|
"stop",
|
|
"profiles",
|
|
"tabs",
|
|
"open",
|
|
"focus",
|
|
"close",
|
|
]);
|
|
|
|
function usesExistingSessionManageFlow(params: { action: string; profileName?: string }) {
|
|
if (!EXISTING_SESSION_MANAGE_ACTIONS.has(params.action)) {
|
|
return false;
|
|
}
|
|
const cfg = browserToolDeps.getRuntimeConfig();
|
|
const resolved = resolveBrowserConfig(cfg.browser, cfg);
|
|
const profile = resolveProfile(resolved, params.profileName ?? resolved.defaultProfile);
|
|
if (profile && getBrowserProfileCapabilities(profile).usesChromeMcp) {
|
|
return true;
|
|
}
|
|
if (params.action !== "profiles") {
|
|
return false;
|
|
}
|
|
return Object.keys(resolved.profiles).some((name) => {
|
|
const candidate = resolveProfile(resolved, name);
|
|
return candidate ? getBrowserProfileCapabilities(candidate).usesChromeMcp : false;
|
|
});
|
|
}
|
|
|
|
function readToolTimeoutMs(params: Record<string, unknown>) {
|
|
return readPositiveIntegerParam(params, "timeoutMs", {
|
|
message: "timeoutMs must be a positive integer.",
|
|
});
|
|
}
|
|
|
|
/** Create the Browser tool exposed to agents. */
|
|
export function createBrowserTool(opts?: {
|
|
sandboxBridgeUrl?: string;
|
|
allowHostControl?: boolean;
|
|
agentSessionKey?: string;
|
|
agentDir?: string;
|
|
workspaceDir?: string;
|
|
activeModel?: {
|
|
provider?: string;
|
|
model?: string;
|
|
};
|
|
mediaScope?: {
|
|
sessionKey?: string;
|
|
channel?: string;
|
|
chatType?: string;
|
|
};
|
|
}): AnyAgentTool {
|
|
const targetDefault = opts?.sandboxBridgeUrl ? "sandbox" : "host";
|
|
const hostHint =
|
|
opts?.allowHostControl === false ? "Host target blocked by policy." : "Host target allowed.";
|
|
return {
|
|
label: "Browser",
|
|
name: "browser",
|
|
description: describeBrowserTool({ targetDefault, hostHint }),
|
|
parameters: BrowserToolSchema,
|
|
execute: async (_toolCallId, args) => {
|
|
const params = args as Record<string, unknown>;
|
|
const action = readStringParam(params, "action", { required: true });
|
|
const profile = readStringParam(params, "profile");
|
|
const requestedNode = readStringParam(params, "node");
|
|
const requestedTimeoutMs = readToolTimeoutMs(params);
|
|
let target = readStringParam(params, "target") as "sandbox" | "host" | "node" | undefined;
|
|
const configuredNode = browserToolDeps
|
|
.getRuntimeConfig()
|
|
.gateway?.nodes?.browser?.node?.trim();
|
|
|
|
if (requestedNode && target && target !== "node") {
|
|
throw new Error('node is only supported with target="node".');
|
|
}
|
|
|
|
// System-profile import reads the local macOS Keychain and Chrome profile,
|
|
// so it can only run on the host. Pin it before target/node resolution so a
|
|
// sandbox default or auto-selected browser node never receives the request.
|
|
if (action === "importprofile") {
|
|
if (target === "sandbox" || target === "node" || requestedNode) {
|
|
throw new Error(
|
|
'system profile import must run on the host; omit target or use target="host".',
|
|
);
|
|
}
|
|
target = "host";
|
|
}
|
|
// existing-session profiles can attach through the selected host or browser node,
|
|
// but they must never fall back into the sandbox browser.
|
|
const isUserBrowserProfile = shouldPreferHostForProfile(profile);
|
|
if (isUserBrowserProfile) {
|
|
if (target === "sandbox") {
|
|
throw new Error(
|
|
`profile="${profile}" cannot use the sandbox browser; use target="host" or omit target.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
let nodeTarget: BrowserNodeTarget | null = null;
|
|
try {
|
|
nodeTarget = await resolveBrowserNodeTarget({
|
|
requestedNode: requestedNode ?? undefined,
|
|
target,
|
|
sandboxBridgeUrl: opts?.sandboxBridgeUrl,
|
|
allowHostControl: opts?.allowHostControl,
|
|
});
|
|
} catch (error) {
|
|
// Keep the logged-in user browser usable on the host when auto-discovery
|
|
// of browser nodes fails transiently. Explicit node requests still fail.
|
|
if (!(isUserBrowserProfile && !target && !requestedNode && !configuredNode)) {
|
|
throw error;
|
|
}
|
|
}
|
|
if (isUserBrowserProfile && !target && !requestedNode && !nodeTarget) {
|
|
target = "host";
|
|
}
|
|
|
|
const resolvedTarget = target === "node" ? undefined : target;
|
|
const baseUrl = nodeTarget
|
|
? undefined
|
|
: resolveBrowserBaseUrl({
|
|
target: resolvedTarget,
|
|
sandboxBridgeUrl: opts?.sandboxBridgeUrl,
|
|
allowHostControl: opts?.allowHostControl,
|
|
});
|
|
|
|
const allowAutomaticHostFallback = Boolean(
|
|
nodeTarget &&
|
|
!target &&
|
|
!requestedNode &&
|
|
!configuredNode &&
|
|
opts?.allowHostControl !== false,
|
|
);
|
|
const proxyRequest = nodeTarget
|
|
? createBrowserNodeProxyRequest({ nodeTarget, allowAutomaticHostFallback })
|
|
: null;
|
|
const toolTimeoutMs =
|
|
requestedTimeoutMs ??
|
|
(usesExistingSessionManageFlow({ action, profileName: profile })
|
|
? DEFAULT_EXISTING_SESSION_MANAGE_TIMEOUT_MS
|
|
: undefined);
|
|
const touchTrackedTab = (targetId: string | undefined) => {
|
|
if (!targetId || (proxyRequest && !proxyRequest.isHostFallbackActive())) {
|
|
return;
|
|
}
|
|
browserToolDeps.touchSessionBrowserTab({
|
|
sessionKey: opts?.agentSessionKey,
|
|
targetId,
|
|
baseUrl,
|
|
profile,
|
|
});
|
|
};
|
|
|
|
switch (action) {
|
|
case "doctor":
|
|
if (proxyRequest) {
|
|
return jsonResult(
|
|
await proxyRequest({
|
|
method: "GET",
|
|
path: "/doctor",
|
|
profile,
|
|
}),
|
|
);
|
|
}
|
|
return jsonResult(await browserToolDeps.browserDoctor(baseUrl, { profile }));
|
|
case "status":
|
|
if (proxyRequest) {
|
|
return jsonResult(
|
|
await proxyRequest({
|
|
method: "GET",
|
|
path: "/",
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
}),
|
|
);
|
|
}
|
|
return jsonResult(
|
|
await browserToolDeps.browserStatus(baseUrl, { profile, timeoutMs: toolTimeoutMs }),
|
|
);
|
|
case "start":
|
|
if (proxyRequest) {
|
|
await proxyRequest({
|
|
method: "POST",
|
|
path: "/start",
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
return jsonResult(
|
|
await proxyRequest({
|
|
method: "GET",
|
|
path: "/",
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
}),
|
|
);
|
|
}
|
|
await browserToolDeps.browserStart(baseUrl, { profile, timeoutMs: toolTimeoutMs });
|
|
return jsonResult(
|
|
await browserToolDeps.browserStatus(baseUrl, { profile, timeoutMs: toolTimeoutMs }),
|
|
);
|
|
case "stop":
|
|
if (proxyRequest) {
|
|
await proxyRequest({
|
|
method: "POST",
|
|
path: "/stop",
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
return jsonResult(
|
|
await proxyRequest({
|
|
method: "GET",
|
|
path: "/",
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
}),
|
|
);
|
|
}
|
|
await browserToolDeps.browserStop(baseUrl, { profile, timeoutMs: toolTimeoutMs });
|
|
return jsonResult(
|
|
await browserToolDeps.browserStatus(baseUrl, { profile, timeoutMs: toolTimeoutMs }),
|
|
);
|
|
case "profiles": {
|
|
// Importable system profiles are host-local (import runs on the host),
|
|
// so read them from the host regardless of the profiles action target;
|
|
// never let a node proxy or sandbox describe the wrong Chrome profiles.
|
|
const systemProfiles = await readHostSystemProfiles({
|
|
allowHostControl: opts?.allowHostControl,
|
|
sandboxBridgeUrl: opts?.sandboxBridgeUrl,
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
if (proxyRequest) {
|
|
const result = await proxyRequest({
|
|
method: "GET",
|
|
path: "/profiles",
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
return jsonResult({
|
|
...(result && typeof result === "object" ? result : { profiles: result }),
|
|
systemProfiles,
|
|
});
|
|
}
|
|
return jsonResult({
|
|
profiles: await browserToolDeps.browserProfiles(baseUrl, { timeoutMs: toolTimeoutMs }),
|
|
systemProfiles,
|
|
});
|
|
}
|
|
case "importprofile": {
|
|
if (proxyRequest) {
|
|
throw new Error("system profile import must run on the browser host");
|
|
}
|
|
const domains = parseSystemProfileDomains(params.domains);
|
|
return jsonResult(
|
|
await browserToolDeps.browserImportProfile(baseUrl, {
|
|
browser: normalizeOptionalString(params.browser) ?? "chrome",
|
|
systemProfile: normalizeOptionalString(params.systemProfile) ?? "Default",
|
|
into: normalizeOptionalString(params.into) ?? "imported",
|
|
domains,
|
|
}),
|
|
);
|
|
}
|
|
case "tabs":
|
|
return await executeTabsAction({
|
|
baseUrl,
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
proxyRequest,
|
|
});
|
|
case "open": {
|
|
const targetUrl = readTargetUrlParam(params);
|
|
const label = normalizeOptionalString(params.label);
|
|
if (proxyRequest) {
|
|
const result = await proxyRequest({
|
|
method: "POST",
|
|
path: "/tabs/open",
|
|
profile,
|
|
body: { url: targetUrl, ...(label ? { label } : {}) },
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
if (proxyRequest.isHostFallbackActive()) {
|
|
browserToolDeps.trackSessionBrowserTab({
|
|
sessionKey: opts?.agentSessionKey,
|
|
targetId: readStringValue((result as { targetId?: unknown }).targetId),
|
|
baseUrl,
|
|
profile,
|
|
});
|
|
}
|
|
return jsonResult(result);
|
|
}
|
|
const opened = await browserToolDeps.browserOpenTab(baseUrl, targetUrl, {
|
|
profile,
|
|
label,
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
browserToolDeps.trackSessionBrowserTab({
|
|
sessionKey: opts?.agentSessionKey,
|
|
targetId: opened.targetId,
|
|
baseUrl,
|
|
profile,
|
|
});
|
|
return jsonResult(opened);
|
|
}
|
|
case "focus": {
|
|
const targetId = readStringParam(params, "targetId", {
|
|
required: true,
|
|
});
|
|
if (proxyRequest) {
|
|
const result = await proxyRequest({
|
|
method: "POST",
|
|
path: "/tabs/focus",
|
|
profile,
|
|
body: { targetId },
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
touchTrackedTab(targetId);
|
|
return jsonResult(result);
|
|
}
|
|
await browserToolDeps.browserFocusTab(baseUrl, targetId, {
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
touchTrackedTab(targetId);
|
|
return jsonResult({ ok: true });
|
|
}
|
|
case "close": {
|
|
const targetId = readStringParam(params, "targetId");
|
|
if (proxyRequest) {
|
|
const result = targetId
|
|
? await proxyRequest({
|
|
method: "DELETE",
|
|
path: `/tabs/${encodeURIComponent(targetId)}`,
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
})
|
|
: await proxyRequest({
|
|
method: "POST",
|
|
path: "/act",
|
|
profile,
|
|
body: { kind: "close" },
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
if (targetId && proxyRequest.isHostFallbackActive()) {
|
|
browserToolDeps.untrackSessionBrowserTab({
|
|
sessionKey: opts?.agentSessionKey,
|
|
targetId,
|
|
baseUrl,
|
|
profile,
|
|
});
|
|
}
|
|
return jsonResult(result);
|
|
}
|
|
if (targetId) {
|
|
await browserToolDeps.browserCloseTab(baseUrl, targetId, {
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
});
|
|
browserToolDeps.untrackSessionBrowserTab({
|
|
sessionKey: opts?.agentSessionKey,
|
|
targetId,
|
|
baseUrl,
|
|
profile,
|
|
});
|
|
} else {
|
|
await browserToolDeps.browserAct(
|
|
baseUrl,
|
|
{ kind: "close" },
|
|
{
|
|
profile,
|
|
timeoutMs: toolTimeoutMs,
|
|
},
|
|
);
|
|
}
|
|
return jsonResult({ ok: true });
|
|
}
|
|
case "snapshot":
|
|
return await executeSnapshotAction({
|
|
input: params,
|
|
baseUrl,
|
|
profile,
|
|
proxyRequest,
|
|
onTabActivity: touchTrackedTab,
|
|
});
|
|
case "screenshot": {
|
|
const targetId = readStringParam(params, "targetId");
|
|
const fullPage = Boolean(params.fullPage);
|
|
const ref = readStringParam(params, "ref");
|
|
const element = readStringParam(params, "element");
|
|
const labels = typeof params.labels === "boolean" ? params.labels : undefined;
|
|
const type = params.type === "jpeg" ? "jpeg" : "png";
|
|
const effectiveTimeoutMs = requestedTimeoutMs ?? DEFAULT_BROWSER_SCREENSHOT_TIMEOUT_MS;
|
|
const result = proxyRequest
|
|
? ((await proxyRequest({
|
|
method: "POST",
|
|
path: "/screenshot",
|
|
profile,
|
|
timeoutMs: effectiveTimeoutMs,
|
|
body: {
|
|
targetId,
|
|
fullPage,
|
|
ref,
|
|
element,
|
|
type,
|
|
labels,
|
|
timeoutMs: effectiveTimeoutMs,
|
|
},
|
|
})) as Awaited<ReturnType<typeof browserScreenshotAction>>)
|
|
: await browserToolDeps.browserScreenshotAction(baseUrl, {
|
|
targetId,
|
|
fullPage,
|
|
ref,
|
|
element,
|
|
type,
|
|
labels,
|
|
timeoutMs: effectiveTimeoutMs,
|
|
profile,
|
|
});
|
|
touchTrackedTab(readStringValue(result.targetId) ?? targetId);
|
|
const screenshotPath = result.path;
|
|
const screenshotCfg = browserToolDeps.getRuntimeConfig();
|
|
const imageSanitization = resolveRuntimeImageSanitization();
|
|
let shareHint = SCREENSHOT_SHARE_UNAVAILABLE;
|
|
try {
|
|
// The original result remains private. Only this bounded outbound
|
|
// copy may cross the sandbox boundary after an explicit message call.
|
|
const sharePath = await browserToolDeps.stageBrowserScreenshotForSharing(
|
|
screenshotPath,
|
|
imageSanitization?.maxDimensionPx,
|
|
);
|
|
shareHint = formatScreenshotShareHint(sharePath);
|
|
} catch {
|
|
// Screenshot viewing remains useful when optional outbound staging fails.
|
|
}
|
|
// Screenshots stay in the tool result for agent vision, but channel
|
|
// delivery must remain an explicit message-tool action.
|
|
const screenshotDetails = {
|
|
...(result as Record<string, unknown>),
|
|
media: { outbound: false },
|
|
};
|
|
try {
|
|
const described = await describeBrowserScreenshot(
|
|
{
|
|
cfg: screenshotCfg,
|
|
filePath: screenshotPath,
|
|
agentDir: opts?.agentDir,
|
|
workspaceDir: opts?.workspaceDir,
|
|
activeModel: opts?.activeModel,
|
|
mediaScope: opts?.mediaScope,
|
|
imageSanitization,
|
|
},
|
|
{
|
|
describeImageFile: browserToolDeps.describeImageFile,
|
|
normalizeBrowserScreenshot: browserToolDeps.normalizeBrowserScreenshot,
|
|
saveMediaBuffer: browserToolDeps.saveMediaBuffer,
|
|
},
|
|
);
|
|
if (described) {
|
|
const analyzedBy =
|
|
described.provider && described.model
|
|
? `${described.provider}/${described.model}`
|
|
: "media image understanding";
|
|
const headerLines = [`[analyzed by ${analyzedBy}]`];
|
|
// Vision model descriptions contain web page content which is
|
|
// untrusted external input — wrap it the same way snapshot and
|
|
// tabs results are wrapped to mitigate prompt injection.
|
|
const wrappedDescription = wrapExternalContent(
|
|
neutralizeMediaDirectives(described.text.trim()),
|
|
{
|
|
source: "browser",
|
|
includeWarning: true,
|
|
},
|
|
);
|
|
const text = `${headerLines.join("\n")}\n${wrappedDescription}\n${shareHint}`;
|
|
return {
|
|
content: [{ type: "text", text }],
|
|
details: {
|
|
...(result as Record<string, unknown>),
|
|
// Do NOT include details.media here — the vision path returns
|
|
// a text description as the deliverable output. Exposing the raw
|
|
// screenshot as media would cause channel delivery to auto-send
|
|
// potentially sensitive page content. The text block carries the
|
|
// staged outbound-copy path for an explicit message-tool send.
|
|
vision: {
|
|
provider: described.provider,
|
|
model: described.model,
|
|
decision: described.decision,
|
|
},
|
|
},
|
|
};
|
|
}
|
|
} catch (err) {
|
|
// Fall back to returning the raw image block so the agent loop can
|
|
// still recover. Provider/runtime error messages are untrusted
|
|
// input too, so defang line-start final-reply media directives.
|
|
const rawReason = err instanceof Error ? err.message : String(err);
|
|
const reason = neutralizeMediaDirectives(rawReason);
|
|
const extraText = `[browser screenshot vision failed: ${reason}]\n${shareHint}`;
|
|
return await browserToolDeps.imageResultFromFile({
|
|
label: "browser:screenshot",
|
|
path: screenshotPath,
|
|
extraText,
|
|
details: screenshotDetails,
|
|
imageSanitization,
|
|
});
|
|
}
|
|
return await browserToolDeps.imageResultFromFile({
|
|
label: "browser:screenshot",
|
|
path: screenshotPath,
|
|
extraText: shareHint,
|
|
details: screenshotDetails,
|
|
imageSanitization,
|
|
});
|
|
}
|
|
case "navigate": {
|
|
const targetUrl = readTargetUrlParam(params);
|
|
const targetId = readStringParam(params, "targetId");
|
|
if (proxyRequest) {
|
|
const result = await proxyRequest({
|
|
method: "POST",
|
|
path: "/navigate",
|
|
profile,
|
|
body: {
|
|
url: targetUrl,
|
|
targetId,
|
|
},
|
|
});
|
|
touchTrackedTab(
|
|
readStringValue((result as { targetId?: unknown }).targetId) ?? targetId,
|
|
);
|
|
return jsonResult(result);
|
|
}
|
|
const result = await browserToolDeps.browserNavigate(baseUrl, {
|
|
url: targetUrl,
|
|
targetId,
|
|
profile,
|
|
});
|
|
touchTrackedTab(readStringValue(result.targetId) ?? targetId);
|
|
return jsonResult(result);
|
|
}
|
|
case "console":
|
|
return await executeConsoleAction({
|
|
input: params,
|
|
baseUrl,
|
|
profile,
|
|
proxyRequest,
|
|
});
|
|
case "pdf": {
|
|
const targetId = normalizeOptionalString(params.targetId);
|
|
const result = proxyRequest
|
|
? ((await proxyRequest({
|
|
method: "POST",
|
|
path: "/pdf",
|
|
profile,
|
|
body: { targetId },
|
|
})) as Awaited<ReturnType<typeof browserPdfSave>>)
|
|
: await browserToolDeps.browserPdfSave(baseUrl, { targetId, profile });
|
|
touchTrackedTab(readStringValue(result.targetId) ?? targetId);
|
|
return {
|
|
content: [{ type: "text" as const, text: `FILE:${result.path}` }],
|
|
details: result,
|
|
};
|
|
}
|
|
case "download":
|
|
case "waitfordownload":
|
|
return await executeDownloadAction({
|
|
action,
|
|
input: params,
|
|
baseUrl,
|
|
profile,
|
|
proxyRequest,
|
|
onTabActivity: touchTrackedTab,
|
|
});
|
|
case "upload": {
|
|
const paths = Array.isArray(params.paths) ? params.paths.map((p) => String(p)) : [];
|
|
if (paths.length === 0) {
|
|
throw new Error("paths required");
|
|
}
|
|
const resolvedResult = await resolveExistingUploadPaths({ requestedPaths: paths });
|
|
if (!resolvedResult.ok) {
|
|
throw new Error(resolvedResult.error);
|
|
}
|
|
const normalizedPaths = resolvedResult.paths;
|
|
const ref = readStringParam(params, "ref");
|
|
const inputRef = readStringParam(params, "inputRef");
|
|
const element = readStringParam(params, "element");
|
|
const { targetId, timeoutMs } = readOptionalTargetAndTimeout(params);
|
|
if (proxyRequest) {
|
|
const result = await proxyRequest({
|
|
method: "POST",
|
|
path: "/hooks/file-chooser",
|
|
profile,
|
|
body: {
|
|
paths: normalizedPaths,
|
|
ref,
|
|
inputRef,
|
|
element,
|
|
targetId,
|
|
timeoutMs,
|
|
},
|
|
});
|
|
return jsonResult(result);
|
|
}
|
|
const result = await browserToolDeps.browserArmFileChooser(baseUrl, {
|
|
paths: normalizedPaths,
|
|
ref,
|
|
inputRef,
|
|
element,
|
|
targetId,
|
|
timeoutMs,
|
|
profile,
|
|
});
|
|
touchTrackedTab(readStringValue((result as { targetId?: unknown }).targetId) ?? targetId);
|
|
return jsonResult(result);
|
|
}
|
|
case "dialog": {
|
|
const accept = Boolean(params.accept);
|
|
const promptText = readStringValue(params.promptText);
|
|
const dialogId = readStringValue(params.dialogId);
|
|
const { targetId, timeoutMs } = readOptionalTargetAndTimeout(params);
|
|
if (proxyRequest) {
|
|
const result = await proxyRequest({
|
|
method: "POST",
|
|
path: "/hooks/dialog",
|
|
profile,
|
|
body: {
|
|
accept,
|
|
promptText,
|
|
dialogId,
|
|
targetId,
|
|
timeoutMs,
|
|
},
|
|
});
|
|
return jsonResult(result);
|
|
}
|
|
const result = await browserToolDeps.browserArmDialog(baseUrl, {
|
|
accept,
|
|
promptText,
|
|
dialogId,
|
|
targetId,
|
|
timeoutMs,
|
|
profile,
|
|
});
|
|
touchTrackedTab(readStringValue((result as { targetId?: unknown }).targetId) ?? targetId);
|
|
return jsonResult(result);
|
|
}
|
|
case "act": {
|
|
const request = readActRequestParam(params);
|
|
if (!request) {
|
|
throw new Error("request required");
|
|
}
|
|
return await executeActAction({
|
|
request,
|
|
baseUrl,
|
|
profile,
|
|
proxyRequest,
|
|
onTabActivity: touchTrackedTab,
|
|
});
|
|
}
|
|
default:
|
|
throw new Error(`Unknown action: ${action}`);
|
|
}
|
|
},
|
|
};
|
|
}
|
|
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|