From 1d4859dc53129174b43930bd54c997f5af0a3ee0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Apr 2026 00:56:12 +0100 Subject: [PATCH] feat(browser): add doctor and richer inspection helpers --- .../skills/browser-automation/SKILL.md | 5 + .../browser/src/browser-tool.actions.ts | 2 + extensions/browser/src/browser-tool.schema.ts | 1 + extensions/browser/src/browser-tool.ts | 3 + .../src/browser/client-actions-core.ts | 2 + .../src/browser/client-actions-types.ts | 3 + extensions/browser/src/browser/client.ts | 4 + extensions/browser/src/browser/errors.test.ts | 12 ++ extensions/browser/src/browser/errors.ts | 9 +- .../src/browser/pw-tools-core.snapshot.ts | 60 ++++++- .../routes/agent.snapshot.plan.test.ts | 11 ++ .../src/browser/routes/agent.snapshot.plan.ts | 4 + .../src/browser/routes/agent.snapshot.ts | 157 +++++++++++++++++- extensions/browser/src/browser/routes/tabs.ts | 4 +- .../src/browser/server-context.selection.ts | 4 +- .../src/browser/server-context.tab-ops.ts | 4 +- .../src/cli/browser-cli-inspect.test.ts | 17 ++ .../browser/src/cli/browser-cli-inspect.ts | 4 + .../src/cli/browser-cli-manage.test.ts | 51 ++++++ .../browser/src/cli/browser-cli-manage.ts | 114 +++++++++++++ 20 files changed, 451 insertions(+), 20 deletions(-) create mode 100644 extensions/browser/src/browser/errors.test.ts diff --git a/extensions/browser/skills/browser-automation/SKILL.md b/extensions/browser/skills/browser-automation/SKILL.md index b15f4367571..2833f8bbaa5 100644 --- a/extensions/browser/skills/browser-automation/SKILL.md +++ b/extensions/browser/skills/browser-automation/SKILL.md @@ -11,6 +11,7 @@ Use this skill when you need the `browser` tool for anything beyond a single pag ## Operating Loop 1. Check browser state before acting: + - `openclaw browser doctor` or `action="status"` when the browser/plugin setup itself may be broken. - `action="status"` for availability. - `action="profiles"` if login state or profile choice matters. - `action="tabs"` before opening a new tab if retries/timeouts may have left windows behind. @@ -22,6 +23,8 @@ Use this skill when you need the `browser` tool for anything beyond a single pag - Use `action="snapshot"` on the intended `targetId`. - Use the same `targetId` for follow-up actions so refs stay on the same tab. - For durable Playwright refs, request `refs="aria"` when supported. + - Use `urls=true` when link text is ambiguous or a direct navigation target would avoid brittle clicks. + - Use `labels=true` on snapshot or screenshot when visual position matters. 4. Act narrowly: - Prefer `action="act"` with a ref from the latest snapshot. - After navigation, modal changes, or form submission, snapshot again before the next action. @@ -58,6 +61,8 @@ If a retry creates duplicates, close the extras by `tabId`: { "action": "close", "targetId": "t3" } ``` +Do not pass bare numbers like `"2"` as `targetId`. Numeric tab positions are only for the CLI `openclaw browser tab select 2` helper; browser tool calls need a `suggestedTargetId`, label, `tabId`, or raw target id. + ## Stale Ref Recovery If an action fails with a missing or stale ref: diff --git a/extensions/browser/src/browser-tool.actions.ts b/extensions/browser/src/browser-tool.actions.ts index ba01454de46..8338c0df22c 100644 --- a/extensions/browser/src/browser-tool.actions.ts +++ b/extensions/browser/src/browser-tool.actions.ts @@ -244,6 +244,7 @@ export async function executeSnapshotAction(params: { ? "efficient" : undefined; const labels = typeof input.labels === "boolean" ? input.labels : undefined; + const urls = typeof input.urls === "boolean" ? input.urls : undefined; const refs: "aria" | "role" | undefined = input.refs === "aria" || input.refs === "role" ? input.refs : undefined; const hasMaxChars = Object.hasOwn(input, "maxChars"); @@ -282,6 +283,7 @@ export async function executeSnapshotAction(params: { selector, frame, labels, + urls, mode, }; let refsFallback: "role" | undefined; diff --git a/extensions/browser/src/browser-tool.schema.ts b/extensions/browser/src/browser-tool.schema.ts index f1cebf5883b..bdc46ba3482 100644 --- a/extensions/browser/src/browser-tool.schema.ts +++ b/extensions/browser/src/browser-tool.schema.ts @@ -105,6 +105,7 @@ export const BrowserToolSchema = Type.Object({ selector: Type.Optional(Type.String()), frame: Type.Optional(Type.String()), labels: Type.Optional(Type.Boolean()), + urls: Type.Optional(Type.Boolean()), fullPage: Type.Optional(Type.Boolean()), ref: Type.Optional(Type.String()), element: Type.Optional(Type.String()), diff --git a/extensions/browser/src/browser-tool.ts b/extensions/browser/src/browser-tool.ts index cc08742e1f3..06e8a745503 100644 --- a/extensions/browser/src/browser-tool.ts +++ b/extensions/browser/src/browser-tool.ts @@ -604,6 +604,7 @@ export function createBrowserTool(opts?: { const fullPage = Boolean(params.fullPage); const ref = readStringParam(params, "ref"); const element = readStringParam(params, "element"); + const labels = typeof params.labels === "boolean" ? params.labels : undefined; const type = params.type === "jpeg" ? "jpeg" : "png"; const result = proxyRequest ? ((await proxyRequest({ @@ -616,6 +617,7 @@ export function createBrowserTool(opts?: { ref, element, type, + labels, }, })) as Awaited>) : await browserToolDeps.browserScreenshotAction(baseUrl, { @@ -624,6 +626,7 @@ export function createBrowserTool(opts?: { ref, element, type, + labels, profile, }); return await browserToolDeps.imageResultFromFile({ diff --git a/extensions/browser/src/browser/client-actions-core.ts b/extensions/browser/src/browser/client-actions-core.ts index 1c57f820598..c899182a356 100644 --- a/extensions/browser/src/browser/client-actions-core.ts +++ b/extensions/browser/src/browser/client-actions-core.ts @@ -175,6 +175,7 @@ export async function browserScreenshotAction( ref?: string; element?: string; type?: "png" | "jpeg"; + labels?: boolean; profile?: string; }, ): Promise { @@ -188,6 +189,7 @@ export async function browserScreenshotAction( ref: opts.ref, element: opts.element, type: opts.type, + labels: opts.labels, }), timeoutMs: 20000, }); diff --git a/extensions/browser/src/browser/client-actions-types.ts b/extensions/browser/src/browser/client-actions-types.ts index 9ad0d820da2..112dd24f987 100644 --- a/extensions/browser/src/browser/client-actions-types.ts +++ b/extensions/browser/src/browser/client-actions-types.ts @@ -11,6 +11,9 @@ export type BrowserActionPathResult = { path: string; targetId: string; url?: string; + labels?: boolean; + labelsCount?: number; + labelsSkipped?: number; }; export type BrowserActionTargetOk = { ok: true; targetId: string }; diff --git a/extensions/browser/src/browser/client.ts b/extensions/browser/src/browser/client.ts index 091c88fed3f..9bf1d010fe6 100644 --- a/extensions/browser/src/browser/client.ts +++ b/extensions/browser/src/browser/client.ts @@ -271,6 +271,7 @@ export async function browserSnapshot( selector?: string; frame?: string; labels?: boolean; + urls?: boolean; mode?: "efficient"; profile?: string; }, @@ -309,6 +310,9 @@ export async function browserSnapshot( if (opts.labels === true) { q.set("labels", "1"); } + if (opts.urls === true) { + q.set("urls", "1"); + } if (opts.mode) { q.set("mode", opts.mode); } diff --git a/extensions/browser/src/browser/errors.test.ts b/extensions/browser/src/browser/errors.test.ts new file mode 100644 index 00000000000..f776a2dfb4e --- /dev/null +++ b/extensions/browser/src/browser/errors.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest"; +import { BrowserTabNotFoundError } from "./errors.js"; + +describe("BrowserTabNotFoundError", () => { + it("teaches agents that bare numbers are not stable tab targets", () => { + const err = new BrowserTabNotFoundError({ input: "2" }); + + expect(err.message).toContain('browser tab "2" not found'); + expect(err.message).toContain("Numeric values are not tab targets"); + expect(err.message).toContain("openclaw browser tab select 2"); + }); +}); diff --git a/extensions/browser/src/browser/errors.ts b/extensions/browser/src/browser/errors.ts index c27abedb1c3..acc052d5139 100644 --- a/extensions/browser/src/browser/errors.ts +++ b/extensions/browser/src/browser/errors.ts @@ -42,7 +42,14 @@ export class BrowserTargetAmbiguousError extends BrowserError { } export class BrowserTabNotFoundError extends BrowserError { - constructor(message = "tab not found", options?: ErrorOptions) { + constructor(inputOrMessage?: string | { input?: string }, options?: ErrorOptions) { + const input = + typeof inputOrMessage === "object" ? inputOrMessage.input?.trim() : inputOrMessage?.trim(); + const message = input + ? /^\d+$/.test(input) + ? `tab not found: browser tab "${input}" not found. Numeric values are not tab targets; use a stable tab id like "t1", a label, or a raw targetId. For positional selection, use "openclaw browser tab select ${input}".` + : `tab not found: browser tab "${input}" not found. Use action=tabs and pass suggestedTargetId, tabId, label, or raw targetId.` + : "tab not found"; super(message, 404, options); } } diff --git a/extensions/browser/src/browser/pw-tools-core.snapshot.ts b/extensions/browser/src/browser/pw-tools-core.snapshot.ts index 02e0ecc1d74..f4bb5717ff2 100644 --- a/extensions/browser/src/browser/pw-tools-core.snapshot.ts +++ b/extensions/browser/src/browser/pw-tools-core.snapshot.ts @@ -1,4 +1,5 @@ import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; +import type { Page } from "playwright-core"; import type { SsrFPolicy } from "../infra/net/ssrf.js"; import { type AriaSnapshotNode, formatAriaSnapshot, type RawAXNode } from "./cdp.js"; import { assertBrowserNavigationAllowed, withBrowserNavigationPolicy } from "./navigation-guard.js"; @@ -19,6 +20,46 @@ import { } from "./pw-session.js"; import { withPageScopedCdpClient } from "./pw-session.page-cdp.js"; +type SnapshotUrlEntry = { + text: string; + url: string; +}; + +async function collectSnapshotUrls(page: Page): Promise { + const urls = await page + .evaluate(() => { + const seen = new Set(); + const out: SnapshotUrlEntry[] = []; + for (const anchor of Array.from(document.querySelectorAll("a[href]"))) { + const href = anchor instanceof HTMLAnchorElement ? anchor.href : ""; + if (!href || seen.has(href)) { + continue; + } + const text = + (anchor.textContent || anchor.getAttribute("aria-label") || "") + .replace(/\s+/g, " ") + .trim() + .slice(0, 120) || href; + seen.add(href); + out.push({ text, url: href }); + if (out.length >= 100) { + break; + } + } + return out; + }) + .catch(() => []); + return Array.isArray(urls) ? urls : []; +} + +function appendSnapshotUrls(snapshot: string, urls: SnapshotUrlEntry[]): string { + if (urls.length === 0) { + return snapshot; + } + const lines = urls.map((entry, index) => `${index + 1}. ${entry.text} -> ${entry.url}`); + return `${snapshot}\n\nLinks:\n${lines.join("\n")}`; +} + export async function snapshotAriaViaPlaywright(opts: { cdpUrl: string; targetId?: string; @@ -62,6 +103,7 @@ export async function snapshotAiViaPlaywright(opts: { targetId?: string; timeoutMs?: number; maxChars?: number; + urls?: boolean; ssrfPolicy?: SsrFPolicy; }): Promise<{ snapshot: string; truncated?: boolean; refs: RoleRefMap }> { const page = await getPageForTargetId({ @@ -83,6 +125,9 @@ export async function snapshotAiViaPlaywright(opts: { mode: "ai", timeout: Math.max(500, Math.min(60_000, Math.floor(opts.timeoutMs ?? 5000))), }); + if (opts.urls) { + snapshot = appendSnapshotUrls(snapshot, await collectSnapshotUrls(page)); + } const maxChars = opts.maxChars; const limit = typeof maxChars === "number" && Number.isFinite(maxChars) && maxChars > 0 @@ -112,6 +157,7 @@ export async function snapshotRoleViaPlaywright(opts: { frameSelector?: string; refsMode?: "role" | "aria"; options?: RoleSnapshotOptions; + urls?: boolean; ssrfPolicy?: SsrFPolicy; }): Promise<{ snapshot: string; @@ -142,6 +188,9 @@ export async function snapshotRoleViaPlaywright(opts: { timeout: 5000, }); const built = buildRoleSnapshotFromAiSnapshot(snapshot, opts.options); + const snapshotWithUrls = opts.urls + ? appendSnapshotUrls(built.snapshot, await collectSnapshotUrls(page)) + : built.snapshot; storeRoleRefsForTarget({ page, cdpUrl: opts.cdpUrl, @@ -150,9 +199,9 @@ export async function snapshotRoleViaPlaywright(opts: { mode: "aria", }); return { - snapshot: built.snapshot, + snapshot: snapshotWithUrls, refs: built.refs, - stats: getRoleSnapshotStats(built.snapshot, built.refs), + stats: getRoleSnapshotStats(snapshotWithUrls, built.refs), }; } @@ -168,6 +217,9 @@ export async function snapshotRoleViaPlaywright(opts: { const ariaSnapshot = await locator.ariaSnapshot(); const built = buildRoleSnapshotFromAriaSnapshot(ariaSnapshot ?? "", opts.options); + const snapshotWithUrls = opts.urls + ? appendSnapshotUrls(built.snapshot, await collectSnapshotUrls(page)) + : built.snapshot; storeRoleRefsForTarget({ page, cdpUrl: opts.cdpUrl, @@ -177,9 +229,9 @@ export async function snapshotRoleViaPlaywright(opts: { mode: "role", }); return { - snapshot: built.snapshot, + snapshot: snapshotWithUrls, refs: built.refs, - stats: getRoleSnapshotStats(built.snapshot, built.refs), + stats: getRoleSnapshotStats(snapshotWithUrls, built.refs), }; } diff --git a/extensions/browser/src/browser/routes/agent.snapshot.plan.test.ts b/extensions/browser/src/browser/routes/agent.snapshot.plan.test.ts index 36bfd5852ac..a9582e48072 100644 --- a/extensions/browser/src/browser/routes/agent.snapshot.plan.test.ts +++ b/extensions/browser/src/browser/routes/agent.snapshot.plan.test.ts @@ -35,4 +35,15 @@ describe("resolveSnapshotPlan", () => { expect(plan.format).toBe("ai"); }); + + it("treats urls as a role snapshot feature", () => { + const plan = resolveSnapshotPlan({ + profile: profile("openclaw"), + query: { urls: "1" }, + hasPlaywright: true, + }); + + expect(plan.urls).toBe(true); + expect(plan.wantsRoleSnapshot).toBe(true); + }); }); diff --git a/extensions/browser/src/browser/routes/agent.snapshot.plan.ts b/extensions/browser/src/browser/routes/agent.snapshot.plan.ts index 27d8e454527..ccf8009fa63 100644 --- a/extensions/browser/src/browser/routes/agent.snapshot.plan.ts +++ b/extensions/browser/src/browser/routes/agent.snapshot.plan.ts @@ -23,6 +23,7 @@ export type BrowserSnapshotPlan = { format: "ai" | "aria"; mode?: "efficient"; labels?: boolean; + urls?: boolean; limit?: number; resolvedMaxChars?: number; interactive?: boolean; @@ -41,6 +42,7 @@ export function resolveSnapshotPlan(params: { }): BrowserSnapshotPlan { const mode = params.query.mode === "efficient" ? "efficient" : undefined; const labels = toBoolean(params.query.labels) ?? undefined; + const urls = toBoolean(params.query.urls) ?? undefined; const explicitFormat = params.query.format === "aria" ? "aria" : params.query.format === "ai" ? "ai" : undefined; const format = resolveDefaultSnapshotFormat({ @@ -82,6 +84,7 @@ export function resolveSnapshotPlan(params: { format, mode, labels, + urls, limit, resolvedMaxChars, interactive, @@ -92,6 +95,7 @@ export function resolveSnapshotPlan(params: { frameSelectorValue, wantsRoleSnapshot: labels === true || + urls === true || mode === "efficient" || interactive === true || compact === true || diff --git a/extensions/browser/src/browser/routes/agent.snapshot.ts b/extensions/browser/src/browser/routes/agent.snapshot.ts index b39760013cb..5fa2ae00437 100644 --- a/extensions/browser/src/browser/routes/agent.snapshot.ts +++ b/extensions/browser/src/browser/routes/agent.snapshot.ts @@ -44,6 +44,51 @@ import { asyncBrowserRoute, jsonError, toBoolean, toStringOrEmpty } from "./util const CHROME_MCP_OVERLAY_ATTR = "data-openclaw-mcp-overlay"; +async function collectChromeMcpSnapshotUrls(params: { + profileName: string; + userDataDir?: string; + targetId: string; +}): Promise> { + const result = await evaluateChromeMcpScript({ + profileName: params.profileName, + userDataDir: params.userDataDir, + targetId: params.targetId, + fn: `() => { + const seen = new Set(); + const out = []; + for (const anchor of Array.from(document.querySelectorAll("a[href]"))) { + const href = anchor.href || ""; + if (!href || seen.has(href)) continue; + const text = (anchor.innerText || anchor.textContent || anchor.getAttribute("aria-label") || "") + .replace(/\\s+/g, " ") + .trim() + .slice(0, 120) || href; + seen.add(href); + out.push({ text, url: href }); + if (out.length >= 100) break; + } + return out; + }`, + }).catch(() => []); + return Array.isArray(result) + ? result.filter( + (entry): entry is { text: string; url: string } => + entry && + typeof entry === "object" && + typeof (entry as { text?: unknown }).text === "string" && + typeof (entry as { url?: unknown }).url === "string", + ) + : []; +} + +function appendSnapshotUrls(snapshot: string, urls: Array<{ text: string; url: string }>): string { + if (urls.length === 0) { + return snapshot; + } + const lines = urls.map((entry, index) => `${index + 1}. ${entry.text} -> ${entry.url}`); + return `${snapshot}\n\nLinks:\n${lines.join("\n")}`; +} + async function clearChromeMcpOverlay(params: { profileName: string; userDataDir?: string; @@ -135,6 +180,9 @@ async function saveNormalizedScreenshotResponse(params: { type: "png" | "jpeg"; targetId: string; url: string; + labels?: boolean; + labelsCount?: number; + labelsSkipped?: number; }) { const normalized = await normalizeBrowserScreenshot(params.buffer, { maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE, @@ -147,6 +195,9 @@ async function saveNormalizedScreenshotResponse(params: { maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, targetId: params.targetId, url: params.url, + labels: params.labels, + labelsCount: params.labelsCount, + labelsSkipped: params.labelsSkipped, }); } @@ -157,6 +208,9 @@ async function saveBrowserMediaResponse(params: { maxBytes: number; targetId: string; url: string; + labels?: boolean; + labelsCount?: number; + labelsSkipped?: number; }) { await ensureMediaDir(); const saved = await saveMediaBuffer( @@ -170,6 +224,9 @@ async function saveBrowserMediaResponse(params: { path: path.resolve(saved.path), targetId: params.targetId, url: params.url, + ...(params.labels ? { labels: true } : {}), + ...(typeof params.labelsCount === "number" ? { labelsCount: params.labelsCount } : {}), + ...(typeof params.labelsSkipped === "number" ? { labelsSkipped: params.labelsSkipped } : {}), }); } @@ -269,6 +326,7 @@ export function registerBrowserAgentSnapshotRoutes( const fullPage = toBoolean(body.fullPage) ?? false; const ref = toStringOrEmpty(body.ref) || undefined; const element = toStringOrEmpty(body.element) || undefined; + const labels = toBoolean(body.labels) ?? false; const type = body.type === "jpeg" ? "jpeg" : "png"; if (fullPage && (ref || element)) { @@ -292,6 +350,46 @@ export function registerBrowserAgentSnapshotRoutes( ...ssrfPolicyOpts, }); } + if (labels) { + const snapshot = await takeChromeMcpSnapshot({ + profileName: profileCtx.profile.name, + userDataDir: profileCtx.profile.userDataDir, + targetId: tab.targetId, + }); + const built = buildAiSnapshotFromChromeMcpSnapshot({ root: snapshot }); + const labelResult = await renderChromeMcpLabels({ + profileName: profileCtx.profile.name, + userDataDir: profileCtx.profile.userDataDir, + targetId: tab.targetId, + refs: Object.keys(built.refs), + }); + try { + const buffer = await takeChromeMcpScreenshot({ + profileName: profileCtx.profile.name, + userDataDir: profileCtx.profile.userDataDir, + targetId: tab.targetId, + fullPage, + format: type, + }); + await saveNormalizedScreenshotResponse({ + res, + buffer, + type, + targetId: tab.targetId, + url: tab.url, + labels: true, + labelsCount: labelResult.labels, + labelsSkipped: labelResult.skipped, + }); + } finally { + await clearChromeMcpOverlay({ + profileName: profileCtx.profile.name, + userDataDir: profileCtx.profile.userDataDir, + targetId: tab.targetId, + }); + } + return; + } const buffer = await takeChromeMcpScreenshot({ profileName: profileCtx.profile.name, userDataDir: profileCtx.profile.userDataDir, @@ -311,17 +409,43 @@ export function registerBrowserAgentSnapshotRoutes( } let buffer: Buffer; - const shouldUsePlaywright = shouldUsePlaywrightForScreenshot({ - profile: profileCtx.profile, - wsUrl: tab.wsUrl, - ref, - element, - }); + const shouldUsePlaywright = + labels || + shouldUsePlaywrightForScreenshot({ + profile: profileCtx.profile, + wsUrl: tab.wsUrl, + ref, + element, + }); if (shouldUsePlaywright) { const pw = await requirePwAi(res, "screenshot"); if (!pw) { return; } + if (labels) { + const snap = await pw.snapshotRoleViaPlaywright({ + cdpUrl, + targetId: tab.targetId, + ssrfPolicy: ctx.state().resolved.ssrfPolicy, + }); + const labeled = await pw.screenshotWithLabelsViaPlaywright({ + cdpUrl, + targetId: tab.targetId, + refs: snap.refs, + type, + }); + await saveNormalizedScreenshotResponse({ + res, + buffer: labeled.buffer, + type, + targetId: tab.targetId, + url: tab.url, + labels: true, + labelsCount: labeled.labels, + labelsSkipped: labeled.skipped, + }); + return; + } const snap = await pw.takeScreenshotViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -406,8 +530,21 @@ export function registerBrowserAgentSnapshotRoutes( }, maxChars: plan.resolvedMaxChars, }); + const builtWithUrls = plan.urls + ? { + ...built, + snapshot: appendSnapshotUrls( + built.snapshot, + await collectChromeMcpSnapshotUrls({ + profileName: profileCtx.profile.name, + userDataDir: profileCtx.profile.userDataDir, + targetId: tab.targetId, + }), + ), + } + : built; if (plan.labels) { - const refs = Object.keys(built.refs); + const refs = Object.keys(builtWithUrls.refs); const labelResult = await renderChromeMcpLabels({ profileName: profileCtx.profile.name, userDataDir: profileCtx.profile.userDataDir, @@ -442,7 +579,7 @@ export function registerBrowserAgentSnapshotRoutes( labelsSkipped: labelResult.skipped, imagePath: path.resolve(saved.path), imageType: normalized.contentType?.includes("jpeg") ? "jpeg" : "png", - ...built, + ...builtWithUrls, }); } finally { await clearChromeMcpOverlay({ @@ -457,7 +594,7 @@ export function registerBrowserAgentSnapshotRoutes( format: "ai", targetId: tab.targetId, url: tab.url, - ...built, + ...builtWithUrls, }); } if (plan.format === "ai") { @@ -472,6 +609,7 @@ export function registerBrowserAgentSnapshotRoutes( frameSelector: plan.frameSelectorValue, refsMode: plan.refsMode, ssrfPolicy: ctx.state().resolved.ssrfPolicy, + urls: plan.urls, options: { interactive: plan.interactive ?? undefined, compact: plan.compact ?? undefined, @@ -485,6 +623,7 @@ export function registerBrowserAgentSnapshotRoutes( cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, ssrfPolicy: ctx.state().resolved.ssrfPolicy, + urls: plan.urls, ...(typeof plan.resolvedMaxChars === "number" ? { maxChars: plan.resolvedMaxChars } : {}), diff --git a/extensions/browser/src/browser/routes/tabs.ts b/extensions/browser/src/browser/routes/tabs.ts index ce80dc448f6..664641e1bf7 100644 --- a/extensions/browser/src/browser/routes/tabs.ts +++ b/extensions/browser/src/browser/routes/tabs.ts @@ -217,11 +217,11 @@ export function registerBrowserTabRoutes(app: BrowserRouteRegistrar, ctx: Browse if (resolved.reason === "ambiguous") { throw new BrowserTargetAmbiguousError(); } - throw new BrowserTabNotFoundError(); + throw new BrowserTabNotFoundError({ input: id }); } const tab = tabs.find((currentTab) => currentTab.targetId === resolved.targetId); if (!tab) { - throw new BrowserTabNotFoundError(); + throw new BrowserTabNotFoundError({ input: id }); } const ssrfPolicyOpts = withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy); if (ssrfPolicyOpts.ssrfPolicy) { diff --git a/extensions/browser/src/browser/server-context.selection.ts b/extensions/browser/src/browser/server-context.selection.ts index e17294dc046..7b799adf66e 100644 --- a/extensions/browser/src/browser/server-context.selection.ts +++ b/extensions/browser/src/browser/server-context.selection.ts @@ -76,7 +76,7 @@ export function createProfileSelectionOps({ throw new BrowserTargetAmbiguousError(); } if (!chosen) { - throw new BrowserTabNotFoundError(); + throw new BrowserTabNotFoundError(targetId ? { input: targetId } : undefined); } profileState.lastTargetId = chosen.targetId; return chosen; @@ -89,7 +89,7 @@ export function createProfileSelectionOps({ if (resolved.reason === "ambiguous") { throw new BrowserTargetAmbiguousError(); } - throw new BrowserTabNotFoundError(); + throw new BrowserTabNotFoundError({ input: targetId }); } return resolved.targetId; }; diff --git a/extensions/browser/src/browser/server-context.tab-ops.ts b/extensions/browser/src/browser/server-context.tab-ops.ts index b95c1e3ec19..c1ff2c24ac5 100644 --- a/extensions/browser/src/browser/server-context.tab-ops.ts +++ b/extensions/browser/src/browser/server-context.tab-ops.ts @@ -348,11 +348,11 @@ export function createProfileTabOps({ if (resolved.reason === "ambiguous") { throw new BrowserTargetAmbiguousError(); } - throw new BrowserTabNotFoundError(); + throw new BrowserTabNotFoundError({ input: targetId }); } const tab = tabs.find((candidate) => candidate.targetId === resolved.targetId); if (!tab) { - throw new BrowserTabNotFoundError(); + throw new BrowserTabNotFoundError({ input: targetId }); } return assignTabAlias({ profileState: getProfileState(), tab, label: normalizedLabel }); }; diff --git a/extensions/browser/src/cli/browser-cli-inspect.test.ts b/extensions/browser/src/cli/browser-cli-inspect.test.ts index 2276ff95e55..46cc6c12971 100644 --- a/extensions/browser/src/cli/browser-cli-inspect.test.ts +++ b/extensions/browser/src/cli/browser-cli-inspect.test.ts @@ -143,6 +143,14 @@ describe("browser cli snapshot defaults", () => { }); }); + it("passes URL expansion for snapshots", async () => { + const params = await runSnapshot(["--urls"]); + expect(params?.query).toMatchObject({ + format: "ai", + urls: true, + }); + }); + it("sends screenshot request with trimmed target id and jpeg type", async () => { const params = await runBrowserInspect(["screenshot", " tab-1 ", "--type", "jpeg"], true); expect(params?.path).toBe("/screenshot"); @@ -152,4 +160,13 @@ describe("browser cli snapshot defaults", () => { fullPage: false, }); }); + + it("passes screenshot labels", async () => { + const params = await runBrowserInspect(["screenshot", "tab-1", "--labels"], true); + expect(params?.path).toBe("/screenshot"); + expect((params as { body?: Record } | undefined)?.body).toMatchObject({ + targetId: "tab-1", + labels: true, + }); + }); }); diff --git a/extensions/browser/src/cli/browser-cli-inspect.ts b/extensions/browser/src/cli/browser-cli-inspect.ts index 1366a40546f..65a300fe7b5 100644 --- a/extensions/browser/src/cli/browser-cli-inspect.ts +++ b/extensions/browser/src/cli/browser-cli-inspect.ts @@ -21,6 +21,7 @@ export function registerBrowserInspectCommands( .option("--full-page", "Capture full scrollable page", false) .option("--ref ", "ARIA ref from ai snapshot") .option("--element ", "CSS selector for element screenshot") + .option("--labels", "Overlay role refs on the screenshot", false) .option("--type ", "Output type (default: png)", "png") .action(async (targetId: string | undefined, opts, cmd) => { const parent = parentOpts(cmd); @@ -37,6 +38,7 @@ export function registerBrowserInspectCommands( fullPage: Boolean(opts.fullPage), ref: normalizeOptionalString(opts.ref), element: normalizeOptionalString(opts.element), + labels: Boolean(opts.labels), type: opts.type === "jpeg" ? "jpeg" : "png", }, }, @@ -67,6 +69,7 @@ export function registerBrowserInspectCommands( .option("--selector ", "Role snapshot: scope to CSS selector") .option("--frame ", "Role snapshot: scope to an iframe selector") .option("--labels", "Include viewport label overlay screenshot", false) + .option("--urls", "Append discovered link URLs to AI snapshots", false) .option("--out ", "Write snapshot to a file") .action(async (opts, cmd) => { const parent = parentOpts(cmd); @@ -88,6 +91,7 @@ export function registerBrowserInspectCommands( selector: normalizeOptionalString(opts.selector), frame: normalizeOptionalString(opts.frame), labels: opts.labels ? true : undefined, + urls: opts.urls ? true : undefined, mode, profile, }; diff --git a/extensions/browser/src/cli/browser-cli-manage.test.ts b/extensions/browser/src/cli/browser-cli-manage.test.ts index 390b94bb234..e0406fa9d17 100644 --- a/extensions/browser/src/cli/browser-cli-manage.test.ts +++ b/extensions/browser/src/cli/browser-cli-manage.test.ts @@ -179,4 +179,55 @@ describe("browser manage output", () => { expect(output).not.toContain("supersecretpasswordvalue1234"); expect(output).not.toContain("supersecrettokenvalue1234567890"); }); + + it("prints a readable browser doctor report", async () => { + getBrowserManageCallBrowserRequestMock().mockImplementation(async (_opts: unknown, req) => { + if (req.path === "/") { + return { + enabled: true, + profile: "openclaw", + driver: "openclaw", + transport: "cdp", + running: true, + cdpReady: true, + cdpHttp: true, + pid: 4321, + cdpPort: 18792, + cdpUrl: "http://127.0.0.1:18792", + chosenBrowser: "chrome", + userDataDir: null, + color: "#00AA00", + headless: false, + noSandbox: false, + executablePath: null, + attachOnly: false, + }; + } + if (req.path === "/profiles") { + return { profiles: [{ name: "openclaw", running: true }] }; + } + if (req.path === "/tabs") { + return { + running: true, + tabs: [ + { + targetId: "abc", + tabId: "t1", + suggestedTargetId: "t1", + title: "Example", + url: "https://example.com", + }, + ], + }; + } + return {}; + }); + + const program = createBrowserManageProgram(); + await program.parseAsync(["browser", "doctor"], { from: "user" }); + + const output = getBrowserCliRuntime().log.mock.calls.at(-1)?.[0] as string; + expect(output).toContain("OK gateway: browser control endpoint reachable"); + expect(output).toContain("OK tabs: 1 visible, use target t1"); + }); }); diff --git a/extensions/browser/src/cli/browser-cli-manage.ts b/extensions/browser/src/cli/browser-cli-manage.ts index f58031dd78a..8606d3eaf84 100644 --- a/extensions/browser/src/cli/browser-cli-manage.ts +++ b/extensions/browser/src/cli/browser-cli-manage.ts @@ -18,6 +18,12 @@ import { const BROWSER_MANAGE_REQUEST_TIMEOUT_MS = 45_000; +type BrowserDoctorCheck = { + name: string; + ok: boolean; + detail?: string; +}; + function resolveProfileQuery(profile?: string) { return profile ? { profile } : undefined; } @@ -110,6 +116,96 @@ function logBrowserTabs(tabs: BrowserTab[], json?: boolean) { ); } +function formatDoctorLine(check: BrowserDoctorCheck): string { + return `${check.ok ? "OK" : "FAIL"} ${check.name}${check.detail ? `: ${check.detail}` : ""}`; +} + +async function runBrowserDoctor(parent: BrowserParentOpts, profile?: string) { + const checks: BrowserDoctorCheck[] = []; + let status: BrowserStatus | null = null; + + try { + status = await fetchBrowserStatus(parent, profile); + checks.push({ + name: "gateway", + ok: true, + detail: "browser control endpoint reachable", + }); + } catch (err) { + checks.push({ + name: "gateway", + ok: false, + detail: String(err), + }); + return { ok: false, checks }; + } + + checks.push({ + name: "plugin", + ok: status.enabled, + detail: status.enabled ? "enabled" : "disabled in config", + }); + checks.push({ + name: "profile", + ok: true, + detail: `${status.profile ?? "openclaw"} (${usesChromeMcpTransport(status) ? "chrome-mcp" : (status.transport ?? "cdp")})`, + }); + checks.push({ + name: "browser", + ok: status.running, + detail: status.running + ? `running${status.cdpReady === false ? ", CDP not ready" : ""}` + : "not running; run `openclaw browser start`", + }); + + try { + const profiles = await callBrowserRequest<{ profiles: ProfileStatus[] }>( + parent, + { method: "GET", path: "/profiles" }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + checks.push({ + name: "profiles", + ok: true, + detail: `${profiles.profiles?.length ?? 0} configured`, + }); + } catch (err) { + checks.push({ + name: "profiles", + ok: false, + detail: String(err), + }); + } + + if (status.running) { + try { + const result = await callBrowserRequest<{ running: boolean; tabs: BrowserTab[] }>( + parent, + { + method: "GET", + path: "/tabs", + query: resolveProfileQuery(profile), + }, + { timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS }, + ); + const tabs = result.tabs ?? []; + checks.push({ + name: "tabs", + ok: true, + detail: `${tabs.length} visible${tabs.length > 0 && tabs[0]?.suggestedTargetId ? `, use target ${tabs[0].suggestedTargetId}` : ""}`, + }); + } catch (err) { + checks.push({ + name: "tabs", + ok: false, + detail: String(err), + }); + } + } + + return { ok: checks.every((check) => check.ok), checks, status }; +} + function usesChromeMcpTransport(params: { transport?: BrowserTransport; driver?: "openclaw" | "existing-session"; @@ -179,6 +275,24 @@ export function registerBrowserManageCommands( }); }); + browser + .command("doctor") + .description("Check browser plugin readiness") + .action(async (_opts, cmd) => { + const parent = parentOpts(cmd); + const profile = parent?.browserProfile; + await runBrowserCommand(async () => { + const result = await runBrowserDoctor(parent, profile); + if (printJsonResult(parent, result)) { + return; + } + defaultRuntime.log(result.checks.map(formatDoctorLine).join("\n")); + if (!result.ok) { + defaultRuntime.exit(1); + } + }); + }); + browser .command("start") .description("Start the browser (no-op if already running)")