vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 10:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(browser): include safe tab urls in agent responses

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-25 08:22:54 +01:00
parent e8191e5b8f
commit 7132ca5766
8 changed files with 183 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -30,6 +30,7 @@ Docs: https://docs.openclaw.ai
- Gateway/nodes: add disabled-by-default `gateway.nodes.pairing.autoApproveCidrs` for first-time node pairing from explicit trusted CIDRs, while keeping operator/browser pairing and all upgrade flows manual. Fixes #60800. Thanks @sahilsatralkar.
- Browser: add viewport coordinate clicks for managed and existing-session automation, plus `openclaw browser click-coords` for CLI use. (#54452) Thanks @dluttz.
- Browser: add `browser.actionTimeoutMs` and use a 60s default action budget so healthy long browser waits do not fail at the client transport boundary. (#62589) Thanks @andyylin.
- Browser: include policy-safe current page URLs in agent-facing browser action and debug responses, omitting blocked URLs instead of leaking private targets. Thanks @zeroaltitude.
- Browser/config: support per-profile `browser.profiles.<name>.headless` overrides for locally launched browser profiles, so one profile can run headless without forcing all browser profiles headless. Thanks @nakamotoliu.
- Plugins/PDF: move local PDF extraction into a bundled `document-extract` plugin so core no longer owns `pdfjs-dist` or PDF image-rendering dependencies. Thanks @vincentkoc.
- Dependencies/memory: stop installing `node-llama-cpp` by default; local embeddings now load it only when operators install the optional runtime package. Thanks @vincentkoc.

2
extensions/browser/src/browser-tool.actions.ts
View File

@@ -223,6 +223,7 @@ function formatTabsToolResult(tabs: unknown[]): AgentToolResult<unknown> {
function formatConsoleToolResult(result: {
targetId?: string;
url?: string;
messages?: unknown[];
}): AgentToolResult<unknown> {
const wrapped = wrapBrowserExternalJson({
@@ -235,6 +236,7 @@ function formatConsoleToolResult(result: {
details: {
...wrapped.safeDetails,
targetId: readStringValue(result.targetId),
url: readStringValue(result.url),
messageCount: Array.isArray(result.messages) ? result.messages.length : undefined,
},
};

9
extensions/browser/src/browser/client-actions-observe.ts
View File

@@ -25,7 +25,7 @@ function buildQuerySuffix(params: Array<[string, string | boolean | undefined]>)
export async function browserConsoleMessages(
baseUrl: string | undefined,
opts: { level?: string; targetId?: string; profile?: string } = {},
): Promise<{ ok: true; messages: BrowserConsoleMessage[]; targetId: string }> {
): Promise<{ ok: true; messages: BrowserConsoleMessage[]; targetId: string; url?: string }> {
const suffix = buildQuerySuffix([
["level", opts.level],
["targetId", opts.targetId],
@@ -35,6 +35,7 @@ export async function browserConsoleMessages(
ok: true;
messages: BrowserConsoleMessage[];
targetId: string;
url?: string;
}>(withBaseUrl(baseUrl, `/console${suffix}`), { timeoutMs: 20000 });
}
@@ -54,7 +55,7 @@ export async function browserPdfSave(
export async function browserPageErrors(
baseUrl: string | undefined,
opts: { targetId?: string; clear?: boolean; profile?: string } = {},
): Promise<{ ok: true; targetId: string; errors: BrowserPageError[] }> {
): Promise<{ ok: true; targetId: string; url?: string; errors: BrowserPageError[] }> {
const suffix = buildQuerySuffix([
["targetId", opts.targetId],
["clear", typeof opts.clear === "boolean" ? opts.clear : undefined],
@@ -63,6 +64,7 @@ export async function browserPageErrors(
return await fetchBrowserJson<{
ok: true;
targetId: string;
url?: string;
errors: BrowserPageError[];
}>(withBaseUrl(baseUrl, `/errors${suffix}`), { timeoutMs: 20000 });
}
@@ -75,7 +77,7 @@ export async function browserRequests(
clear?: boolean;
profile?: string;
} = {},
): Promise<{ ok: true; targetId: string; requests: BrowserNetworkRequest[] }> {
): Promise<{ ok: true; targetId: string; url?: string; requests: BrowserNetworkRequest[] }> {
const suffix = buildQuerySuffix([
["targetId", opts.targetId],
["filter", opts.filter],
@@ -85,6 +87,7 @@ export async function browserRequests(
return await fetchBrowserJson<{
ok: true;
targetId: string;
url?: string;
requests: BrowserNetworkRequest[];
}>(withBaseUrl(baseUrl, `/requests${suffix}`), { timeoutMs: 20000 });
}

79
extensions/browser/src/browser/routes/agent.act.ts
View File

@@ -378,9 +378,18 @@ export function registerBrowserAgentActRoutes(
res,
ctx,
targetId,
run: async ({ profileCtx, cdpUrl, tab }) => {
run: async ({ profileCtx, cdpUrl, tab, resolveTabUrl }) => {
const evaluateEnabled = ctx.state().resolved.evaluateEnabled;
const ssrfPolicy = ctx.state().resolved.ssrfPolicy;
const jsonOk = async (extra?: Record<string, unknown>) => {
const url = await resolveTabUrl(tab.url);
return res.json({
ok: true,
targetId: tab.targetId,
...(url ? { url } : {}),
...extra,
});
};
if (action.targetId && action.targetId !== tab.targetId) {
return jsonActError(
res,
@@ -427,7 +436,7 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return await jsonOk();
case "clickCoords":
await runExistingSessionActionWithNavigationGuard({
execute: () =>
@@ -443,7 +452,7 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return await jsonOk();
case "type":
await runExistingSessionActionWithNavigationGuard({
execute: async () => {
@@ -465,7 +474,7 @@ export function registerBrowserAgentActRoutes(
},
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "press":
await runExistingSessionActionWithNavigationGuard({
execute: () =>
@@ -477,7 +486,7 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "hover":
await runExistingSessionActionWithNavigationGuard({
execute: () =>
@@ -489,7 +498,7 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "scrollIntoView":
await runExistingSessionActionWithNavigationGuard({
execute: () =>
@@ -502,7 +511,7 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "drag":
await runExistingSessionActionWithNavigationGuard({
execute: () =>
@@ -515,7 +524,7 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "select":
await runExistingSessionActionWithNavigationGuard({
execute: () =>
@@ -528,7 +537,7 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "fill":
await runExistingSessionActionWithNavigationGuard({
execute: () =>
@@ -543,7 +552,7 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "resize":
await resizeChromeMcpPage({
profileName,
@@ -552,7 +561,7 @@ export function registerBrowserAgentActRoutes(
width: action.width,
height: action.height,
});
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return await jsonOk();
case "wait":
await waitForExistingSessionCondition({
profileName,
@@ -567,7 +576,7 @@ export function registerBrowserAgentActRoutes(
fn: action.fn,
timeoutMs: action.timeoutMs,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "evaluate": {
const result = await runExistingSessionActionWithNavigationGuard({
execute: () =>
@@ -580,16 +589,11 @@ export function registerBrowserAgentActRoutes(
}),
guard: existingSessionNavigationGuard,
});
return res.json({
ok: true,
targetId: tab.targetId,
url: tab.url,
result,
});
return await jsonOk({ result });
}
case "close":
await closeChromeMcpTab(profileName, tab.targetId, profileCtx.profile.userDataDir);
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
case "batch":
return jsonActError(
res,
@@ -620,20 +624,15 @@ export function registerBrowserAgentActRoutes(
});
switch (action.kind) {
case "batch":
return res.json({ ok: true, targetId: tab.targetId, results: result.results ?? [] });
return await jsonOk({ results: result.results ?? [] });
case "evaluate":
return res.json({
ok: true,
targetId: tab.targetId,
url: tab.url,
result: result.result,
});
return await jsonOk({ result: result.result });
case "click":
case "clickCoords":
case "resize":
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return await jsonOk();
default:
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
}
},
});
@@ -660,7 +659,7 @@ export function registerBrowserAgentActRoutes(
res,
ctx,
targetId,
run: async ({ profileCtx, cdpUrl, tab }) => {
run: async ({ profileCtx, cdpUrl, tab, resolveTabUrl }) => {
if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) {
return jsonError(res, 501, EXISTING_SESSION_LIMITS.responseBody);
}
@@ -675,7 +674,13 @@ export function registerBrowserAgentActRoutes(
timeoutMs: timeoutMs ?? undefined,
maxChars: maxChars ?? undefined,
});
res.json({ ok: true, targetId: tab.targetId, response: result });
const currentUrl = await resolveTabUrl(tab.url);
res.json({
ok: true,
targetId: tab.targetId,
...(currentUrl ? { url: currentUrl } : {}),
response: result,
});
},
});
}),
@@ -696,7 +701,15 @@ export function registerBrowserAgentActRoutes(
res,
ctx,
targetId,
run: async ({ profileCtx, cdpUrl, tab }) => {
run: async ({ profileCtx, cdpUrl, tab, resolveTabUrl }) => {
const jsonOk = async () => {
const currentUrl = await resolveTabUrl(tab.url);
return res.json({
ok: true,
targetId: tab.targetId,
...(currentUrl ? { url: currentUrl } : {}),
});
};
if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) {
await evaluateChromeMcpScript({
profileName: profileCtx.profile.name,
@@ -719,7 +732,7 @@ export function registerBrowserAgentActRoutes(
return true;
}`,
});
return res.json({ ok: true, targetId: tab.targetId });
return await jsonOk();
}
const pw = await requirePwAi(res, "highlight");
if (!pw) {
@@ -730,7 +743,7 @@ export function registerBrowserAgentActRoutes(
targetId: tab.targetId,
ref,
});
res.json({ ok: true, targetId: tab.targetId });
await jsonOk();
},
});
}),

24
extensions/browser/src/browser/routes/agent.debug.ts
View File

@@ -29,13 +29,14 @@ export function registerBrowserAgentDebugRoutes(
ctx,
targetId,
feature: "console messages",
run: async ({ cdpUrl, tab, pw }) => {
run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => {
const messages = await pw.getConsoleMessagesViaPlaywright({
cdpUrl,
targetId: tab.targetId,
level: normalizeOptionalString(level),
});
res.json({ ok: true, messages, targetId: tab.targetId });
const url = await resolveTabUrl(tab.url);
res.json({ ok: true, messages, targetId: tab.targetId, ...(url ? { url } : {}) });
},
});
}),
@@ -53,13 +54,14 @@ export function registerBrowserAgentDebugRoutes(
ctx,
targetId,
feature: "page errors",
run: async ({ cdpUrl, tab, pw }) => {
run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => {
const result = await pw.getPageErrorsViaPlaywright({
cdpUrl,
targetId: tab.targetId,
clear,
});
res.json({ ok: true, targetId: tab.targetId, ...result });
const url = await resolveTabUrl(tab.url);
res.json({ ok: true, targetId: tab.targetId, ...(url ? { url } : {}), ...result });
},
});
}),
@@ -78,14 +80,15 @@ export function registerBrowserAgentDebugRoutes(
ctx,
targetId,
feature: "network requests",
run: async ({ cdpUrl, tab, pw }) => {
run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => {
const result = await pw.getNetworkRequestsViaPlaywright({
cdpUrl,
targetId: tab.targetId,
filter: normalizeOptionalString(filter),
clear,
});
res.json({ ok: true, targetId: tab.targetId, ...result });
const url = await resolveTabUrl(tab.url);
res.json({ ok: true, targetId: tab.targetId, ...(url ? { url } : {}), ...result });
},
});
}),
@@ -106,7 +109,7 @@ export function registerBrowserAgentDebugRoutes(
ctx,
targetId,
feature: "trace start",
run: async ({ cdpUrl, tab, pw }) => {
run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => {
await pw.traceStartViaPlaywright({
cdpUrl,
targetId: tab.targetId,
@@ -114,7 +117,8 @@ export function registerBrowserAgentDebugRoutes(
snapshots,
sources,
});
res.json({ ok: true, targetId: tab.targetId });
const url = await resolveTabUrl(tab.url);
res.json({ ok: true, targetId: tab.targetId, ...(url ? { url } : {}) });
},
});
}),
@@ -133,7 +137,7 @@ export function registerBrowserAgentDebugRoutes(
ctx,
targetId,
feature: "trace stop",
run: async ({ cdpUrl, tab, pw }) => {
run: async ({ cdpUrl, tab, pw, resolveTabUrl }) => {
const id = crypto.randomUUID();
const tracePath = await resolveWritableOutputPathOrRespond({
res,
@@ -151,9 +155,11 @@ export function registerBrowserAgentDebugRoutes(
targetId: tab.targetId,
path: tracePath,
});
const url = await resolveTabUrl(tab.url);
res.json({
ok: true,
targetId: tab.targetId,
...(url ? { url } : {}),
path: path.resolve(tracePath),
});
},

66
extensions/browser/src/browser/routes/agent.shared.test.ts
View File

@@ -1,5 +1,10 @@
import { describe, expect, it } from "vitest";
import { readBody, resolveTargetIdFromBody, resolveTargetIdFromQuery } from "./agent.shared.js";
import {
readBody,
resolveSafeRouteTabUrl,
resolveTargetIdFromBody,
resolveTargetIdFromQuery,
} from "./agent.shared.js";
import type { BrowserRequest } from "./types.js";
function requestWithBody(body: unknown): BrowserRequest {
@@ -10,6 +15,27 @@ function requestWithBody(body: unknown): BrowserRequest {
};
}
function routeContext(ssrfPolicy?: unknown) {
return {
state: () => ({
resolved: {
extraArgs: [],
ssrfPolicy,
},
}),
};
}
function profileContext(tabs: Array<{ targetId: string; url: string }>) {
return {
profile: {
cdpIsLoopback: true,
driver: "openclaw",
},
listTabs: async () => tabs,
};
}
describe("browser route shared helpers", () => {
describe("readBody", () => {
it("returns object bodies", () => {
@@ -36,4 +62,42 @@ describe("browser route shared helpers", () => {
expect(resolveTargetIdFromQuery({ targetId: false })).toBeUndefined();
});
});
describe("safe route tab URLs", () => {
it("returns the current listed URL for a tab target", async () => {
await expect(
resolveSafeRouteTabUrl({
ctx: routeContext() as never,
profileCtx: profileContext([
{ targetId: "tab-1", url: "https://example.com/current" },
]) as never,
targetId: "tab-1",
fallbackUrl: "https://example.com/stale",
}),
).resolves.toBe("https://example.com/current");
});
it("falls back to the ensured tab URL when tab listing is stale", async () => {
await expect(
resolveSafeRouteTabUrl({
ctx: routeContext() as never,
profileCtx: profileContext([]) as never,
targetId: "tab-1",
fallbackUrl: "https://example.com/fallback",
}),
).resolves.toBe("https://example.com/fallback");
});
it("omits URLs blocked by the browser SSRF policy", async () => {
await expect(
resolveSafeRouteTabUrl({
ctx: routeContext({ dangerouslyAllowPrivateNetwork: false }) as never,
profileCtx: profileContext([
{ targetId: "tab-1", url: "http://127.0.0.1:9222/" },
]) as never,
targetId: "tab-1",
}),
).resolves.toBeUndefined();
});
});
});

45
extensions/browser/src/browser/routes/agent.shared.ts
View File

@@ -1,4 +1,9 @@
import { resolveBrowserNavigationProxyMode } from "../browser-proxy-mode.js";
import { toBrowserErrorResponse } from "../errors.js";
import {
assertBrowserNavigationResultAllowed,
withBrowserNavigationPolicy,
} from "../navigation-guard.js";
import type { PwAiModule } from "../pw-ai-module.js";
import { getPwAiModule as getPwAiModuleBase } from "../pw-ai-module.js";
import type { BrowserRouteContext, ProfileContext } from "../server-context.js";
@@ -90,6 +95,7 @@ type RouteTabContext = {
profileCtx: ProfileContext;
tab: Awaited<ReturnType<ProfileContext["ensureTabAvailable"]>>;
cdpUrl: string;
resolveTabUrl: (fallbackUrl?: string) => Promise<string | undefined>;
};
type RouteTabPwContext = RouteTabContext & {
@@ -117,6 +123,13 @@ export async function withRouteTabContext<T>(
profileCtx,
tab,
cdpUrl: profileCtx.profile.cdpUrl,
resolveTabUrl: (fallbackUrl?: string) =>
resolveSafeRouteTabUrl({
ctx: params.ctx,
profileCtx,
targetId: tab.targetId,
fallbackUrl,
}),
});
} catch (err) {
handleRouteError(params.ctx, params.res, err);
@@ -124,6 +137,34 @@ export async function withRouteTabContext<T>(
}
}
export async function resolveSafeRouteTabUrl(params: {
ctx: BrowserRouteContext;
profileCtx: ProfileContext;
targetId: string;
fallbackUrl?: string;
}): Promise<string | undefined> {
const tabs = await params.profileCtx.listTabs().catch(() => []);
const candidateUrl =
tabs.find((tab) => tab.targetId === params.targetId)?.url ?? params.fallbackUrl;
if (!candidateUrl) {
return undefined;
}
try {
await assertBrowserNavigationResultAllowed({
url: candidateUrl,
...withBrowserNavigationPolicy(params.ctx.state().resolved.ssrfPolicy, {
browserProxyMode: resolveBrowserNavigationProxyMode({
resolved: params.ctx.state().resolved,
profile: params.profileCtx.profile,
}),
}),
});
return candidateUrl;
} catch {
return undefined;
}
}
type RouteWithPwParams<T> = {
req: BrowserRequest;
res: BrowserResponse;
@@ -141,12 +182,12 @@ export async function withPlaywrightRouteContext<T>(
res: params.res,
ctx: params.ctx,
targetId: params.targetId,
run: async ({ profileCtx, tab, cdpUrl }) => {
run: async ({ profileCtx, tab, cdpUrl, resolveTabUrl }) => {
const pw = await requirePwAi(params.res, params.feature);
if (!pw) {
return undefined as T | undefined;
}
return await params.run({ profileCtx, tab, cdpUrl, pw });
return await params.run({ profileCtx, tab, cdpUrl, resolveTabUrl, pw });
},
});
}

5
extensions/browser/src/browser/routes/existing-session.test-support.ts
View File

@@ -42,7 +42,12 @@ export function createExistingSessionAgentSharedModule() {
profileCtx: existingSessionRouteState.profileCtx,
cdpUrl: "http://127.0.0.1:18800",
tab: existingSessionRouteState.tab,
resolveTabUrl: vi.fn(async (fallbackUrl?: string) => fallbackUrl ?? routeStateUrl()),
});
}),
};
}
function routeStateUrl() {
return existingSessionRouteState.tab.url;
}