From 7132ca5766afc3641a1fb10da36083e672cf7835 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Apr 2026 08:22:54 +0100 Subject: [PATCH] feat(browser): include safe tab urls in agent responses --- CHANGELOG.md | 1 + .../browser/src/browser-tool.actions.ts | 2 + .../src/browser/client-actions-observe.ts | 9 ++- .../browser/src/browser/routes/agent.act.ts | 79 +++++++++++-------- .../browser/src/browser/routes/agent.debug.ts | 24 +++--- .../src/browser/routes/agent.shared.test.ts | 66 +++++++++++++++- .../src/browser/routes/agent.shared.ts | 45 ++++++++++- .../routes/existing-session.test-support.ts | 5 ++ 8 files changed, 183 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4afd7431448..50b97363832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Docs: https://docs.openclaw.ai - Gateway/nodes: add disabled-by-default `gateway.nodes.pairing.autoApproveCidrs` for first-time node pairing from explicit trusted CIDRs, while keeping operator/browser pairing and all upgrade flows manual. Fixes #60800. Thanks @sahilsatralkar. - Browser: add viewport coordinate clicks for managed and existing-session automation, plus `openclaw browser click-coords` for CLI use. (#54452) Thanks @dluttz. - Browser: add `browser.actionTimeoutMs` and use a 60s default action budget so healthy long browser waits do not fail at the client transport boundary. (#62589) Thanks @andyylin. +- Browser: include policy-safe current page URLs in agent-facing browser action and debug responses, omitting blocked URLs instead of leaking private targets. Thanks @zeroaltitude. - Browser/config: support per-profile `browser.profiles..headless` overrides for locally launched browser profiles, so one profile can run headless without forcing all browser profiles headless. Thanks @nakamotoliu. - Plugins/PDF: move local PDF extraction into a bundled `document-extract` plugin so core no longer owns `pdfjs-dist` or PDF image-rendering dependencies. Thanks @vincentkoc. - Dependencies/memory: stop installing `node-llama-cpp` by default; local embeddings now load it only when operators install the optional runtime package. Thanks @vincentkoc. diff --git a/extensions/browser/src/browser-tool.actions.ts b/extensions/browser/src/browser-tool.actions.ts index 1bc1565158b..687f8f11373 100644 --- a/extensions/browser/src/browser-tool.actions.ts +++ b/extensions/browser/src/browser-tool.actions.ts @@ -223,6 +223,7 @@ function formatTabsToolResult(tabs: unknown[]): AgentToolResult { function formatConsoleToolResult(result: { targetId?: string; + url?: string; messages?: unknown[]; }): AgentToolResult { const wrapped = wrapBrowserExternalJson({ @@ -235,6 +236,7 @@ function formatConsoleToolResult(result: { details: { ...wrapped.safeDetails, targetId: readStringValue(result.targetId), + url: readStringValue(result.url), messageCount: Array.isArray(result.messages) ? result.messages.length : undefined, }, }; diff --git a/extensions/browser/src/browser/client-actions-observe.ts b/extensions/browser/src/browser/client-actions-observe.ts index 7f7d8cd6926..f0c78be3d51 100644 --- a/extensions/browser/src/browser/client-actions-observe.ts +++ b/extensions/browser/src/browser/client-actions-observe.ts @@ -25,7 +25,7 @@ function buildQuerySuffix(params: Array<[string, string | boolean | undefined]>) export async function browserConsoleMessages( baseUrl: string | undefined, opts: { level?: string; targetId?: string; profile?: string } = {}, -): Promise<{ ok: true; messages: BrowserConsoleMessage[]; targetId: string }> { +): Promise<{ ok: true; messages: BrowserConsoleMessage[]; targetId: string; url?: string }> { const suffix = buildQuerySuffix([ ["level", opts.level], ["targetId", opts.targetId], @@ -35,6 +35,7 @@ export async function browserConsoleMessages( ok: true; messages: BrowserConsoleMessage[]; targetId: string; + url?: string; }>(withBaseUrl(baseUrl, `/console${suffix}`), { timeoutMs: 20000 }); } @@ -54,7 +55,7 @@ export async function browserPdfSave( export async function browserPageErrors( baseUrl: string | undefined, opts: { targetId?: string; clear?: boolean; profile?: string } = {}, -): Promise<{ ok: true; targetId: string; errors: BrowserPageError[] }> { +): Promise<{ ok: true; targetId: string; url?: string; errors: BrowserPageError[] }> { const suffix = buildQuerySuffix([ ["targetId", opts.targetId], ["clear", typeof opts.clear === "boolean" ? opts.clear : undefined], @@ -63,6 +64,7 @@ export async function browserPageErrors( return await fetchBrowserJson<{ ok: true; targetId: string; + url?: string; errors: BrowserPageError[]; }>(withBaseUrl(baseUrl, `/errors${suffix}`), { timeoutMs: 20000 }); } @@ -75,7 +77,7 @@ export async function browserRequests( clear?: boolean; profile?: string; } = {}, -): Promise<{ ok: true; targetId: string; requests: BrowserNetworkRequest[] }> { +): Promise<{ ok: true; targetId: string; url?: string; requests: BrowserNetworkRequest[] }> { const suffix = buildQuerySuffix([ ["targetId", opts.targetId], ["filter", opts.filter], @@ -85,6 +87,7 @@ export async function browserRequests( return await fetchBrowserJson<{ ok: true; targetId: string; + url?: string; requests: BrowserNetworkRequest[]; }>(withBaseUrl(baseUrl, `/requests${suffix}`), { timeoutMs: 20000 }); } diff --git a/extensions/browser/src/browser/routes/agent.act.ts b/extensions/browser/src/browser/routes/agent.act.ts index c75dafb0b40..28d28131081 100644 --- a/extensions/browser/src/browser/routes/agent.act.ts +++ b/extensions/browser/src/browser/routes/agent.act.ts @@ -378,9 +378,18 @@ export function registerBrowserAgentActRoutes( res, ctx, targetId, - run: async ({ profileCtx, cdpUrl, tab }) => { + run: async ({ profileCtx, cdpUrl, tab, resolveTabUrl }) => { const evaluateEnabled = ctx.state().resolved.evaluateEnabled; const ssrfPolicy = ctx.state().resolved.ssrfPolicy; + const jsonOk = async (extra?: Record) => { + const url = await resolveTabUrl(tab.url); + return res.json({ + ok: true, + targetId: tab.targetId, + ...(url ? { url } : {}), + ...extra, + }); + }; if (action.targetId && action.targetId !== tab.targetId) { return jsonActError( res, @@ -427,7 +436,7 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); + return await jsonOk(); case "clickCoords": await runExistingSessionActionWithNavigationGuard({ execute: () => @@ -443,7 +452,7 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); + return await jsonOk(); case "type": await runExistingSessionActionWithNavigationGuard({ execute: async () => { @@ -465,7 +474,7 @@ export function registerBrowserAgentActRoutes( }, guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "press": await runExistingSessionActionWithNavigationGuard({ execute: () => @@ -477,7 +486,7 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "hover": await runExistingSessionActionWithNavigationGuard({ execute: () => @@ -489,7 +498,7 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "scrollIntoView": await runExistingSessionActionWithNavigationGuard({ execute: () => @@ -502,7 +511,7 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "drag": await runExistingSessionActionWithNavigationGuard({ execute: () => @@ -515,7 +524,7 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "select": await runExistingSessionActionWithNavigationGuard({ execute: () => @@ -528,7 +537,7 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "fill": await runExistingSessionActionWithNavigationGuard({ execute: () => @@ -543,7 +552,7 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "resize": await resizeChromeMcpPage({ profileName, @@ -552,7 +561,7 @@ export function registerBrowserAgentActRoutes( width: action.width, height: action.height, }); - return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); + return await jsonOk(); case "wait": await waitForExistingSessionCondition({ profileName, @@ -567,7 +576,7 @@ export function registerBrowserAgentActRoutes( fn: action.fn, timeoutMs: action.timeoutMs, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "evaluate": { const result = await runExistingSessionActionWithNavigationGuard({ execute: () => @@ -580,16 +589,11 @@ export function registerBrowserAgentActRoutes( }), guard: existingSessionNavigationGuard, }); - return res.json({ - ok: true, - targetId: tab.targetId, - url: tab.url, - result, - }); + return await jsonOk({ result }); } case "close": await closeChromeMcpTab(profileName, tab.targetId, profileCtx.profile.userDataDir); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); case "batch": return jsonActError( res, @@ -620,20 +624,15 @@ export function registerBrowserAgentActRoutes( }); switch (action.kind) { case "batch": - return res.json({ ok: true, targetId: tab.targetId, results: result.results ?? [] }); + return await jsonOk({ results: result.results ?? [] }); case "evaluate": - return res.json({ - ok: true, - targetId: tab.targetId, - url: tab.url, - result: result.result, - }); + return await jsonOk({ result: result.result }); case "click": case "clickCoords": case "resize": - return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); + return await jsonOk(); default: - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); } }, }); @@ -660,7 +659,7 @@ export function registerBrowserAgentActRoutes( res, ctx, targetId, - run: async ({ profileCtx, cdpUrl, tab }) => { + run: async ({ profileCtx, cdpUrl, tab, resolveTabUrl }) => { if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { return jsonError(res, 501, EXISTING_SESSION_LIMITS.responseBody); } @@ -675,7 +674,13 @@ export function registerBrowserAgentActRoutes( timeoutMs: timeoutMs ?? undefined, maxChars: maxChars ?? undefined, }); - res.json({ ok: true, targetId: tab.targetId, response: result }); + const currentUrl = await resolveTabUrl(tab.url); + res.json({ + ok: true, + targetId: tab.targetId, + ...(currentUrl ? { url: currentUrl } : {}), + response: result, + }); }, }); }), @@ -696,7 +701,15 @@ export function registerBrowserAgentActRoutes( res, ctx, targetId, - run: async ({ profileCtx, cdpUrl, tab }) => { + run: async ({ profileCtx, cdpUrl, tab, resolveTabUrl }) => { + const jsonOk = async () => { + const currentUrl = await resolveTabUrl(tab.url); + return res.json({ + ok: true, + targetId: tab.targetId, + ...(currentUrl ? { url: currentUrl } : {}), + }); + }; if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { await evaluateChromeMcpScript({ profileName: profileCtx.profile.name, @@ -719,7 +732,7 @@ export function registerBrowserAgentActRoutes( return true; }`, }); - return res.json({ ok: true, targetId: tab.targetId }); + return await jsonOk(); } const pw = await requirePwAi(res, "highlight"); if (!pw) { @@ -730,7 +743,7 @@ export function registerBrowserAgentActRoutes( targetId: tab.targetId, ref, }); - res.json({ ok: true, targetId: tab.targetId }); + await jsonOk(); }, }); }), diff --git a/extensions/browser/src/browser/routes/agent.debug.ts b/extensions/browser/src/browser/routes/agent.debug.ts index 4337ad1e060..948f1f03be4 100644 --- a/extensions/browser/src/browser/routes/agent.debug.ts +++ b/extensions/browser/src/browser/routes/agent.debug.ts @@ -29,13 +29,14 @@ export function registerBrowserAgentDebugRoutes( ctx, targetId, feature: "console messages", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => { const messages = await pw.getConsoleMessagesViaPlaywright({ cdpUrl, targetId: tab.targetId, level: normalizeOptionalString(level), }); - res.json({ ok: true, messages, targetId: tab.targetId }); + const url = await resolveTabUrl(tab.url); + res.json({ ok: true, messages, targetId: tab.targetId, ...(url ? { url } : {}) }); }, }); }), @@ -53,13 +54,14 @@ export function registerBrowserAgentDebugRoutes( ctx, targetId, feature: "page errors", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => { const result = await pw.getPageErrorsViaPlaywright({ cdpUrl, targetId: tab.targetId, clear, }); - res.json({ ok: true, targetId: tab.targetId, ...result }); + const url = await resolveTabUrl(tab.url); + res.json({ ok: true, targetId: tab.targetId, ...(url ? { url } : {}), ...result }); }, }); }), @@ -78,14 +80,15 @@ export function registerBrowserAgentDebugRoutes( ctx, targetId, feature: "network requests", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => { const result = await pw.getNetworkRequestsViaPlaywright({ cdpUrl, targetId: tab.targetId, filter: normalizeOptionalString(filter), clear, }); - res.json({ ok: true, targetId: tab.targetId, ...result }); + const url = await resolveTabUrl(tab.url); + res.json({ ok: true, targetId: tab.targetId, ...(url ? { url } : {}), ...result }); }, }); }), @@ -106,7 +109,7 @@ export function registerBrowserAgentDebugRoutes( ctx, targetId, feature: "trace start", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => { await pw.traceStartViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -114,7 +117,8 @@ export function registerBrowserAgentDebugRoutes( snapshots, sources, }); - res.json({ ok: true, targetId: tab.targetId }); + const url = await resolveTabUrl(tab.url); + res.json({ ok: true, targetId: tab.targetId, ...(url ? { url } : {}) }); }, }); }), @@ -133,7 +137,7 @@ export function registerBrowserAgentDebugRoutes( ctx, targetId, feature: "trace stop", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => { const id = crypto.randomUUID(); const tracePath = await resolveWritableOutputPathOrRespond({ res, @@ -151,9 +155,11 @@ export function registerBrowserAgentDebugRoutes( targetId: tab.targetId, path: tracePath, }); + const url = await resolveTabUrl(tab.url); res.json({ ok: true, targetId: tab.targetId, + ...(url ? { url } : {}), path: path.resolve(tracePath), }); }, diff --git a/extensions/browser/src/browser/routes/agent.shared.test.ts b/extensions/browser/src/browser/routes/agent.shared.test.ts index 4a43ba56891..a6247d35724 100644 --- a/extensions/browser/src/browser/routes/agent.shared.test.ts +++ b/extensions/browser/src/browser/routes/agent.shared.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { readBody, resolveTargetIdFromBody, resolveTargetIdFromQuery } from "./agent.shared.js"; +import { + readBody, + resolveSafeRouteTabUrl, + resolveTargetIdFromBody, + resolveTargetIdFromQuery, +} from "./agent.shared.js"; import type { BrowserRequest } from "./types.js"; function requestWithBody(body: unknown): BrowserRequest { @@ -10,6 +15,27 @@ function requestWithBody(body: unknown): BrowserRequest { }; } +function routeContext(ssrfPolicy?: unknown) { + return { + state: () => ({ + resolved: { + extraArgs: [], + ssrfPolicy, + }, + }), + }; +} + +function profileContext(tabs: Array<{ targetId: string; url: string }>) { + return { + profile: { + cdpIsLoopback: true, + driver: "openclaw", + }, + listTabs: async () => tabs, + }; +} + describe("browser route shared helpers", () => { describe("readBody", () => { it("returns object bodies", () => { @@ -36,4 +62,42 @@ describe("browser route shared helpers", () => { expect(resolveTargetIdFromQuery({ targetId: false })).toBeUndefined(); }); }); + + describe("safe route tab URLs", () => { + it("returns the current listed URL for a tab target", async () => { + await expect( + resolveSafeRouteTabUrl({ + ctx: routeContext() as never, + profileCtx: profileContext([ + { targetId: "tab-1", url: "https://example.com/current" }, + ]) as never, + targetId: "tab-1", + fallbackUrl: "https://example.com/stale", + }), + ).resolves.toBe("https://example.com/current"); + }); + + it("falls back to the ensured tab URL when tab listing is stale", async () => { + await expect( + resolveSafeRouteTabUrl({ + ctx: routeContext() as never, + profileCtx: profileContext([]) as never, + targetId: "tab-1", + fallbackUrl: "https://example.com/fallback", + }), + ).resolves.toBe("https://example.com/fallback"); + }); + + it("omits URLs blocked by the browser SSRF policy", async () => { + await expect( + resolveSafeRouteTabUrl({ + ctx: routeContext({ dangerouslyAllowPrivateNetwork: false }) as never, + profileCtx: profileContext([ + { targetId: "tab-1", url: "http://127.0.0.1:9222/" }, + ]) as never, + targetId: "tab-1", + }), + ).resolves.toBeUndefined(); + }); + }); }); diff --git a/extensions/browser/src/browser/routes/agent.shared.ts b/extensions/browser/src/browser/routes/agent.shared.ts index e3495753b6c..9a513ccd0eb 100644 --- a/extensions/browser/src/browser/routes/agent.shared.ts +++ b/extensions/browser/src/browser/routes/agent.shared.ts @@ -1,4 +1,9 @@ +import { resolveBrowserNavigationProxyMode } from "../browser-proxy-mode.js"; import { toBrowserErrorResponse } from "../errors.js"; +import { + assertBrowserNavigationResultAllowed, + withBrowserNavigationPolicy, +} from "../navigation-guard.js"; import type { PwAiModule } from "../pw-ai-module.js"; import { getPwAiModule as getPwAiModuleBase } from "../pw-ai-module.js"; import type { BrowserRouteContext, ProfileContext } from "../server-context.js"; @@ -90,6 +95,7 @@ type RouteTabContext = { profileCtx: ProfileContext; tab: Awaited>; cdpUrl: string; + resolveTabUrl: (fallbackUrl?: string) => Promise; }; type RouteTabPwContext = RouteTabContext & { @@ -117,6 +123,13 @@ export async function withRouteTabContext( profileCtx, tab, cdpUrl: profileCtx.profile.cdpUrl, + resolveTabUrl: (fallbackUrl?: string) => + resolveSafeRouteTabUrl({ + ctx: params.ctx, + profileCtx, + targetId: tab.targetId, + fallbackUrl, + }), }); } catch (err) { handleRouteError(params.ctx, params.res, err); @@ -124,6 +137,34 @@ export async function withRouteTabContext( } } +export async function resolveSafeRouteTabUrl(params: { + ctx: BrowserRouteContext; + profileCtx: ProfileContext; + targetId: string; + fallbackUrl?: string; +}): Promise { + const tabs = await params.profileCtx.listTabs().catch(() => []); + const candidateUrl = + tabs.find((tab) => tab.targetId === params.targetId)?.url ?? params.fallbackUrl; + if (!candidateUrl) { + return undefined; + } + try { + await assertBrowserNavigationResultAllowed({ + url: candidateUrl, + ...withBrowserNavigationPolicy(params.ctx.state().resolved.ssrfPolicy, { + browserProxyMode: resolveBrowserNavigationProxyMode({ + resolved: params.ctx.state().resolved, + profile: params.profileCtx.profile, + }), + }), + }); + return candidateUrl; + } catch { + return undefined; + } +} + type RouteWithPwParams = { req: BrowserRequest; res: BrowserResponse; @@ -141,12 +182,12 @@ export async function withPlaywrightRouteContext( res: params.res, ctx: params.ctx, targetId: params.targetId, - run: async ({ profileCtx, tab, cdpUrl }) => { + run: async ({ profileCtx, tab, cdpUrl, resolveTabUrl }) => { const pw = await requirePwAi(params.res, params.feature); if (!pw) { return undefined as T | undefined; } - return await params.run({ profileCtx, tab, cdpUrl, pw }); + return await params.run({ profileCtx, tab, cdpUrl, resolveTabUrl, pw }); }, }); } diff --git a/extensions/browser/src/browser/routes/existing-session.test-support.ts b/extensions/browser/src/browser/routes/existing-session.test-support.ts index 48e4cdf680e..4d1313fbc19 100644 --- a/extensions/browser/src/browser/routes/existing-session.test-support.ts +++ b/extensions/browser/src/browser/routes/existing-session.test-support.ts @@ -42,7 +42,12 @@ export function createExistingSessionAgentSharedModule() { profileCtx: existingSessionRouteState.profileCtx, cdpUrl: "http://127.0.0.1:18800", tab: existingSessionRouteState.tab, + resolveTabUrl: vi.fn(async (fallbackUrl?: string) => fallbackUrl ?? routeStateUrl()), }); }), }; } + +function routeStateUrl() { + return existingSessionRouteState.tab.url; +}