vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 09:00:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(browser): add doctor and richer inspection helpers

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-25 00:56:12 +01:00
parent d1cc54866d
commit 1d4859dc53
20 changed files with 451 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
extensions/browser/skills/browser-automation/SKILL.md
View File

@@ -11,6 +11,7 @@ Use this skill when you need the `browser` tool for anything beyond a single pag
## Operating Loop
1. Check browser state before acting:
- `openclaw browser doctor` or `action="status"` when the browser/plugin setup itself may be broken.
- `action="status"` for availability.
- `action="profiles"` if login state or profile choice matters.
- `action="tabs"` before opening a new tab if retries/timeouts may have left windows behind.
@@ -22,6 +23,8 @@ Use this skill when you need the `browser` tool for anything beyond a single pag
- Use `action="snapshot"` on the intended `targetId`.
- Use the same `targetId` for follow-up actions so refs stay on the same tab.
- For durable Playwright refs, request `refs="aria"` when supported.
- Use `urls=true` when link text is ambiguous or a direct navigation target would avoid brittle clicks.
- Use `labels=true` on snapshot or screenshot when visual position matters.
4. Act narrowly:
- Prefer `action="act"` with a ref from the latest snapshot.
- After navigation, modal changes, or form submission, snapshot again before the next action.
@@ -58,6 +61,8 @@ If a retry creates duplicates, close the extras by `tabId`:
{ "action": "close", "targetId": "t3" }
```
Do not pass bare numbers like `"2"` as `targetId`. Numeric tab positions are only for the CLI `openclaw browser tab select 2` helper; browser tool calls need a `suggestedTargetId`, label, `tabId`, or raw target id.
## Stale Ref Recovery
If an action fails with a missing or stale ref:

2
extensions/browser/src/browser-tool.actions.ts
View File

@@ -244,6 +244,7 @@ export async function executeSnapshotAction(params: {
? "efficient"
: undefined;
const labels = typeof input.labels === "boolean" ? input.labels : undefined;
const urls = typeof input.urls === "boolean" ? input.urls : undefined;
const refs: "aria" | "role" | undefined =
input.refs === "aria" || input.refs === "role" ? input.refs : undefined;
const hasMaxChars = Object.hasOwn(input, "maxChars");
@@ -282,6 +283,7 @@ export async function executeSnapshotAction(params: {
selector,
frame,
labels,
urls,
mode,
};
let refsFallback: "role" | undefined;

1
extensions/browser/src/browser-tool.schema.ts
View File

@@ -105,6 +105,7 @@ export const BrowserToolSchema = Type.Object({
selector: Type.Optional(Type.String()),
frame: Type.Optional(Type.String()),
labels: Type.Optional(Type.Boolean()),
urls: Type.Optional(Type.Boolean()),
fullPage: Type.Optional(Type.Boolean()),
ref: Type.Optional(Type.String()),
element: Type.Optional(Type.String()),

3
extensions/browser/src/browser-tool.ts
View File

@@ -604,6 +604,7 @@ export function createBrowserTool(opts?: {
const fullPage = Boolean(params.fullPage);
const ref = readStringParam(params, "ref");
const element = readStringParam(params, "element");
const labels = typeof params.labels === "boolean" ? params.labels : undefined;
const type = params.type === "jpeg" ? "jpeg" : "png";
const result = proxyRequest
? ((await proxyRequest({
@@ -616,6 +617,7 @@ export function createBrowserTool(opts?: {
ref,
element,
type,
labels,
},
})) as Awaited<ReturnType<typeof browserScreenshotAction>>)
: await browserToolDeps.browserScreenshotAction(baseUrl, {
@@ -624,6 +626,7 @@ export function createBrowserTool(opts?: {
ref,
element,
type,
labels,
profile,
});
return await browserToolDeps.imageResultFromFile({

2
extensions/browser/src/browser/client-actions-core.ts
View File

@@ -175,6 +175,7 @@ export async function browserScreenshotAction(
ref?: string;
element?: string;
type?: "png" | "jpeg";
labels?: boolean;
profile?: string;
},
): Promise<BrowserActionPathResult> {
@@ -188,6 +189,7 @@ export async function browserScreenshotAction(
ref: opts.ref,
element: opts.element,
type: opts.type,
labels: opts.labels,
}),
timeoutMs: 20000,
});

3
extensions/browser/src/browser/client-actions-types.ts
View File

@@ -11,6 +11,9 @@ export type BrowserActionPathResult = {
path: string;
targetId: string;
url?: string;
labels?: boolean;
labelsCount?: number;
labelsSkipped?: number;
};
export type BrowserActionTargetOk = { ok: true; targetId: string };

4
extensions/browser/src/browser/client.ts
View File

@@ -271,6 +271,7 @@ export async function browserSnapshot(
selector?: string;
frame?: string;
labels?: boolean;
urls?: boolean;
mode?: "efficient";
profile?: string;
},
@@ -309,6 +310,9 @@ export async function browserSnapshot(
if (opts.labels === true) {
q.set("labels", "1");
}
if (opts.urls === true) {
q.set("urls", "1");
}
if (opts.mode) {
q.set("mode", opts.mode);
}

12
extensions/browser/src/browser/errors.test.ts Normal file
View File

@@ -0,0 +1,12 @@
import { describe, expect, it } from "vitest";
import { BrowserTabNotFoundError } from "./errors.js";
describe("BrowserTabNotFoundError", () => {
it("teaches agents that bare numbers are not stable tab targets", () => {
const err = new BrowserTabNotFoundError({ input: "2" });
expect(err.message).toContain('browser tab "2" not found');
expect(err.message).toContain("Numeric values are not tab targets");
expect(err.message).toContain("openclaw browser tab select 2");
});
});

9
extensions/browser/src/browser/errors.ts
View File

@@ -42,7 +42,14 @@ export class BrowserTargetAmbiguousError extends BrowserError {
}
export class BrowserTabNotFoundError extends BrowserError {
constructor(message = "tab not found", options?: ErrorOptions) {
constructor(inputOrMessage?: string | { input?: string }, options?: ErrorOptions) {
const input =
typeof inputOrMessage === "object" ? inputOrMessage.input?.trim() : inputOrMessage?.trim();
const message = input
? /^\d+$/.test(input)
? `tab not found: browser tab "${input}" not found. Numeric values are not tab targets; use a stable tab id like "t1", a label, or a raw targetId. For positional selection, use "openclaw browser tab select ${input}".`
: `tab not found: browser tab "${input}" not found. Use action=tabs and pass suggestedTargetId, tabId, label, or raw targetId.`
: "tab not found";
super(message, 404, options);
}
}

60
extensions/browser/src/browser/pw-tools-core.snapshot.ts
View File

@@ -1,4 +1,5 @@
import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
import type { Page } from "playwright-core";
import type { SsrFPolicy } from "../infra/net/ssrf.js";
import { type AriaSnapshotNode, formatAriaSnapshot, type RawAXNode } from "./cdp.js";
import { assertBrowserNavigationAllowed, withBrowserNavigationPolicy } from "./navigation-guard.js";
@@ -19,6 +20,46 @@ import {
} from "./pw-session.js";
import { withPageScopedCdpClient } from "./pw-session.page-cdp.js";
type SnapshotUrlEntry = {
text: string;
url: string;
};
async function collectSnapshotUrls(page: Page): Promise<SnapshotUrlEntry[]> {
const urls = await page
.evaluate(() => {
const seen = new Set<string>();
const out: SnapshotUrlEntry[] = [];
for (const anchor of Array.from(document.querySelectorAll("a[href]"))) {
const href = anchor instanceof HTMLAnchorElement ? anchor.href : "";
if (!href || seen.has(href)) {
continue;
}
const text =
(anchor.textContent || anchor.getAttribute("aria-label") || "")
.replace(/\s+/g, " ")
.trim()
.slice(0, 120) || href;
seen.add(href);
out.push({ text, url: href });
if (out.length >= 100) {
break;
}
}
return out;
})
.catch(() => []);
return Array.isArray(urls) ? urls : [];
}
function appendSnapshotUrls(snapshot: string, urls: SnapshotUrlEntry[]): string {
if (urls.length === 0) {
return snapshot;
}
const lines = urls.map((entry, index) => `${index + 1}. ${entry.text} -> ${entry.url}`);
return `${snapshot}\n\nLinks:\n${lines.join("\n")}`;
}
export async function snapshotAriaViaPlaywright(opts: {
cdpUrl: string;
targetId?: string;
@@ -62,6 +103,7 @@ export async function snapshotAiViaPlaywright(opts: {
targetId?: string;
timeoutMs?: number;
maxChars?: number;
urls?: boolean;
ssrfPolicy?: SsrFPolicy;
}): Promise<{ snapshot: string; truncated?: boolean; refs: RoleRefMap }> {
const page = await getPageForTargetId({
@@ -83,6 +125,9 @@ export async function snapshotAiViaPlaywright(opts: {
mode: "ai",
timeout: Math.max(500, Math.min(60_000, Math.floor(opts.timeoutMs ?? 5000))),
});
if (opts.urls) {
snapshot = appendSnapshotUrls(snapshot, await collectSnapshotUrls(page));
}
const maxChars = opts.maxChars;
const limit =
typeof maxChars === "number" && Number.isFinite(maxChars) && maxChars > 0
@@ -112,6 +157,7 @@ export async function snapshotRoleViaPlaywright(opts: {
frameSelector?: string;
refsMode?: "role" | "aria";
options?: RoleSnapshotOptions;
urls?: boolean;
ssrfPolicy?: SsrFPolicy;
}): Promise<{
snapshot: string;
@@ -142,6 +188,9 @@ export async function snapshotRoleViaPlaywright(opts: {
timeout: 5000,
});
const built = buildRoleSnapshotFromAiSnapshot(snapshot, opts.options);
const snapshotWithUrls = opts.urls
? appendSnapshotUrls(built.snapshot, await collectSnapshotUrls(page))
: built.snapshot;
storeRoleRefsForTarget({
page,
cdpUrl: opts.cdpUrl,
@@ -150,9 +199,9 @@ export async function snapshotRoleViaPlaywright(opts: {
mode: "aria",
});
return {
snapshot: built.snapshot,
snapshot: snapshotWithUrls,
refs: built.refs,
stats: getRoleSnapshotStats(built.snapshot, built.refs),
stats: getRoleSnapshotStats(snapshotWithUrls, built.refs),
};
}
@@ -168,6 +217,9 @@ export async function snapshotRoleViaPlaywright(opts: {
const ariaSnapshot = await locator.ariaSnapshot();
const built = buildRoleSnapshotFromAriaSnapshot(ariaSnapshot ?? "", opts.options);
const snapshotWithUrls = opts.urls
? appendSnapshotUrls(built.snapshot, await collectSnapshotUrls(page))
: built.snapshot;
storeRoleRefsForTarget({
page,
cdpUrl: opts.cdpUrl,
@@ -177,9 +229,9 @@ export async function snapshotRoleViaPlaywright(opts: {
mode: "role",
});
return {
snapshot: built.snapshot,
snapshot: snapshotWithUrls,
refs: built.refs,
stats: getRoleSnapshotStats(built.snapshot, built.refs),
stats: getRoleSnapshotStats(snapshotWithUrls, built.refs),
};
}

11
extensions/browser/src/browser/routes/agent.snapshot.plan.test.ts
View File

@@ -35,4 +35,15 @@ describe("resolveSnapshotPlan", () => {
expect(plan.format).toBe("ai");
});
it("treats urls as a role snapshot feature", () => {
const plan = resolveSnapshotPlan({
profile: profile("openclaw"),
query: { urls: "1" },
hasPlaywright: true,
});
expect(plan.urls).toBe(true);
expect(plan.wantsRoleSnapshot).toBe(true);
});
});

4
extensions/browser/src/browser/routes/agent.snapshot.plan.ts
View File

@@ -23,6 +23,7 @@ export type BrowserSnapshotPlan = {
format: "ai" | "aria";
mode?: "efficient";
labels?: boolean;
urls?: boolean;
limit?: number;
resolvedMaxChars?: number;
interactive?: boolean;
@@ -41,6 +42,7 @@ export function resolveSnapshotPlan(params: {
}): BrowserSnapshotPlan {
const mode = params.query.mode === "efficient" ? "efficient" : undefined;
const labels = toBoolean(params.query.labels) ?? undefined;
const urls = toBoolean(params.query.urls) ?? undefined;
const explicitFormat =
params.query.format === "aria" ? "aria" : params.query.format === "ai" ? "ai" : undefined;
const format = resolveDefaultSnapshotFormat({
@@ -82,6 +84,7 @@ export function resolveSnapshotPlan(params: {
format,
mode,
labels,
urls,
limit,
resolvedMaxChars,
interactive,
@@ -92,6 +95,7 @@ export function resolveSnapshotPlan(params: {
frameSelectorValue,
wantsRoleSnapshot:
labels === true ||
urls === true ||
mode === "efficient" ||
interactive === true ||
compact === true ||

157
extensions/browser/src/browser/routes/agent.snapshot.ts
View File

@@ -44,6 +44,51 @@ import { asyncBrowserRoute, jsonError, toBoolean, toStringOrEmpty } from "./util
const CHROME_MCP_OVERLAY_ATTR = "data-openclaw-mcp-overlay";
async function collectChromeMcpSnapshotUrls(params: {
profileName: string;
userDataDir?: string;
targetId: string;
}): Promise<Array<{ text: string; url: string }>> {
const result = await evaluateChromeMcpScript({
profileName: params.profileName,
userDataDir: params.userDataDir,
targetId: params.targetId,
fn: `() => {
const seen = new Set();
const out = [];
for (const anchor of Array.from(document.querySelectorAll("a[href]"))) {
const href = anchor.href || "";
if (!href || seen.has(href)) continue;
const text = (anchor.innerText || anchor.textContent || anchor.getAttribute("aria-label") || "")
.replace(/\\s+/g, " ")
.trim()
.slice(0, 120) || href;
seen.add(href);
out.push({ text, url: href });
if (out.length >= 100) break;
}
return out;
}`,
}).catch(() => []);
return Array.isArray(result)
? result.filter(
(entry): entry is { text: string; url: string } =>
entry &&
typeof entry === "object" &&
typeof (entry as { text?: unknown }).text === "string" &&
typeof (entry as { url?: unknown }).url === "string",
)
: [];
}
function appendSnapshotUrls(snapshot: string, urls: Array<{ text: string; url: string }>): string {
if (urls.length === 0) {
return snapshot;
}
const lines = urls.map((entry, index) => `${index + 1}. ${entry.text} -> ${entry.url}`);
return `${snapshot}\n\nLinks:\n${lines.join("\n")}`;
}
async function clearChromeMcpOverlay(params: {
profileName: string;
userDataDir?: string;
@@ -135,6 +180,9 @@ async function saveNormalizedScreenshotResponse(params: {
type: "png" | "jpeg";
targetId: string;
url: string;
labels?: boolean;
labelsCount?: number;
labelsSkipped?: number;
}) {
const normalized = await normalizeBrowserScreenshot(params.buffer, {
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
@@ -147,6 +195,9 @@ async function saveNormalizedScreenshotResponse(params: {
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
targetId: params.targetId,
url: params.url,
labels: params.labels,
labelsCount: params.labelsCount,
labelsSkipped: params.labelsSkipped,
});
}
@@ -157,6 +208,9 @@ async function saveBrowserMediaResponse(params: {
maxBytes: number;
targetId: string;
url: string;
labels?: boolean;
labelsCount?: number;
labelsSkipped?: number;
}) {
await ensureMediaDir();
const saved = await saveMediaBuffer(
@@ -170,6 +224,9 @@ async function saveBrowserMediaResponse(params: {
path: path.resolve(saved.path),
targetId: params.targetId,
url: params.url,
...(params.labels ? { labels: true } : {}),
...(typeof params.labelsCount === "number" ? { labelsCount: params.labelsCount } : {}),
...(typeof params.labelsSkipped === "number" ? { labelsSkipped: params.labelsSkipped } : {}),
});
}
@@ -269,6 +326,7 @@ export function registerBrowserAgentSnapshotRoutes(
const fullPage = toBoolean(body.fullPage) ?? false;
const ref = toStringOrEmpty(body.ref) || undefined;
const element = toStringOrEmpty(body.element) || undefined;
const labels = toBoolean(body.labels) ?? false;
const type = body.type === "jpeg" ? "jpeg" : "png";
if (fullPage && (ref || element)) {
@@ -292,6 +350,46 @@ export function registerBrowserAgentSnapshotRoutes(
...ssrfPolicyOpts,
});
}
if (labels) {
const snapshot = await takeChromeMcpSnapshot({
profileName: profileCtx.profile.name,
userDataDir: profileCtx.profile.userDataDir,
targetId: tab.targetId,
});
const built = buildAiSnapshotFromChromeMcpSnapshot({ root: snapshot });
const labelResult = await renderChromeMcpLabels({
profileName: profileCtx.profile.name,
userDataDir: profileCtx.profile.userDataDir,
targetId: tab.targetId,
refs: Object.keys(built.refs),
});
try {
const buffer = await takeChromeMcpScreenshot({
profileName: profileCtx.profile.name,
userDataDir: profileCtx.profile.userDataDir,
targetId: tab.targetId,
fullPage,
format: type,
});
await saveNormalizedScreenshotResponse({
res,
buffer,
type,
targetId: tab.targetId,
url: tab.url,
labels: true,
labelsCount: labelResult.labels,
labelsSkipped: labelResult.skipped,
});
} finally {
await clearChromeMcpOverlay({
profileName: profileCtx.profile.name,
userDataDir: profileCtx.profile.userDataDir,
targetId: tab.targetId,
});
}
return;
}
const buffer = await takeChromeMcpScreenshot({
profileName: profileCtx.profile.name,
userDataDir: profileCtx.profile.userDataDir,
@@ -311,17 +409,43 @@ export function registerBrowserAgentSnapshotRoutes(
}
let buffer: Buffer;
const shouldUsePlaywright = shouldUsePlaywrightForScreenshot({
profile: profileCtx.profile,
wsUrl: tab.wsUrl,
ref,
element,
});
const shouldUsePlaywright =
labels ||
shouldUsePlaywrightForScreenshot({
profile: profileCtx.profile,
wsUrl: tab.wsUrl,
ref,
element,
});
if (shouldUsePlaywright) {
const pw = await requirePwAi(res, "screenshot");
if (!pw) {
return;
}
if (labels) {
const snap = await pw.snapshotRoleViaPlaywright({
cdpUrl,
targetId: tab.targetId,
ssrfPolicy: ctx.state().resolved.ssrfPolicy,
});
const labeled = await pw.screenshotWithLabelsViaPlaywright({
cdpUrl,
targetId: tab.targetId,
refs: snap.refs,
type,
});
await saveNormalizedScreenshotResponse({
res,
buffer: labeled.buffer,
type,
targetId: tab.targetId,
url: tab.url,
labels: true,
labelsCount: labeled.labels,
labelsSkipped: labeled.skipped,
});
return;
}
const snap = await pw.takeScreenshotViaPlaywright({
cdpUrl,
targetId: tab.targetId,
@@ -406,8 +530,21 @@ export function registerBrowserAgentSnapshotRoutes(
},
maxChars: plan.resolvedMaxChars,
});
const builtWithUrls = plan.urls
? {
...built,
snapshot: appendSnapshotUrls(
built.snapshot,
await collectChromeMcpSnapshotUrls({
profileName: profileCtx.profile.name,
userDataDir: profileCtx.profile.userDataDir,
targetId: tab.targetId,
}),
),
}
: built;
if (plan.labels) {
const refs = Object.keys(built.refs);
const refs = Object.keys(builtWithUrls.refs);
const labelResult = await renderChromeMcpLabels({
profileName: profileCtx.profile.name,
userDataDir: profileCtx.profile.userDataDir,
@@ -442,7 +579,7 @@ export function registerBrowserAgentSnapshotRoutes(
labelsSkipped: labelResult.skipped,
imagePath: path.resolve(saved.path),
imageType: normalized.contentType?.includes("jpeg") ? "jpeg" : "png",
...built,
...builtWithUrls,
});
} finally {
await clearChromeMcpOverlay({
@@ -457,7 +594,7 @@ export function registerBrowserAgentSnapshotRoutes(
format: "ai",
targetId: tab.targetId,
url: tab.url,
...built,
...builtWithUrls,
});
}
if (plan.format === "ai") {
@@ -472,6 +609,7 @@ export function registerBrowserAgentSnapshotRoutes(
frameSelector: plan.frameSelectorValue,
refsMode: plan.refsMode,
ssrfPolicy: ctx.state().resolved.ssrfPolicy,
urls: plan.urls,
options: {
interactive: plan.interactive ?? undefined,
compact: plan.compact ?? undefined,
@@ -485,6 +623,7 @@ export function registerBrowserAgentSnapshotRoutes(
cdpUrl: profileCtx.profile.cdpUrl,
targetId: tab.targetId,
ssrfPolicy: ctx.state().resolved.ssrfPolicy,
urls: plan.urls,
...(typeof plan.resolvedMaxChars === "number"
? { maxChars: plan.resolvedMaxChars }
: {}),

4
extensions/browser/src/browser/routes/tabs.ts
View File

@@ -217,11 +217,11 @@ export function registerBrowserTabRoutes(app: BrowserRouteRegistrar, ctx: Browse
if (resolved.reason === "ambiguous") {
throw new BrowserTargetAmbiguousError();
}
throw new BrowserTabNotFoundError();
throw new BrowserTabNotFoundError({ input: id });
}
const tab = tabs.find((currentTab) => currentTab.targetId === resolved.targetId);
if (!tab) {
throw new BrowserTabNotFoundError();
throw new BrowserTabNotFoundError({ input: id });
}
const ssrfPolicyOpts = withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy);
if (ssrfPolicyOpts.ssrfPolicy) {

4
extensions/browser/src/browser/server-context.selection.ts
View File

@@ -76,7 +76,7 @@ export function createProfileSelectionOps({
throw new BrowserTargetAmbiguousError();
}
if (!chosen) {
throw new BrowserTabNotFoundError();
throw new BrowserTabNotFoundError(targetId ? { input: targetId } : undefined);
}
profileState.lastTargetId = chosen.targetId;
return chosen;
@@ -89,7 +89,7 @@ export function createProfileSelectionOps({
if (resolved.reason === "ambiguous") {
throw new BrowserTargetAmbiguousError();
}
throw new BrowserTabNotFoundError();
throw new BrowserTabNotFoundError({ input: targetId });
}
return resolved.targetId;
};

4
extensions/browser/src/browser/server-context.tab-ops.ts
View File

@@ -348,11 +348,11 @@ export function createProfileTabOps({
if (resolved.reason === "ambiguous") {
throw new BrowserTargetAmbiguousError();
}
throw new BrowserTabNotFoundError();
throw new BrowserTabNotFoundError({ input: targetId });
}
const tab = tabs.find((candidate) => candidate.targetId === resolved.targetId);
if (!tab) {
throw new BrowserTabNotFoundError();
throw new BrowserTabNotFoundError({ input: targetId });
}
return assignTabAlias({ profileState: getProfileState(), tab, label: normalizedLabel });
};

17
extensions/browser/src/cli/browser-cli-inspect.test.ts
View File

@@ -143,6 +143,14 @@ describe("browser cli snapshot defaults", () => {
});
});
it("passes URL expansion for snapshots", async () => {
const params = await runSnapshot(["--urls"]);
expect(params?.query).toMatchObject({
format: "ai",
urls: true,
});
});
it("sends screenshot request with trimmed target id and jpeg type", async () => {
const params = await runBrowserInspect(["screenshot", " tab-1 ", "--type", "jpeg"], true);
expect(params?.path).toBe("/screenshot");
@@ -152,4 +160,13 @@ describe("browser cli snapshot defaults", () => {
fullPage: false,
});
});
it("passes screenshot labels", async () => {
const params = await runBrowserInspect(["screenshot", "tab-1", "--labels"], true);
expect(params?.path).toBe("/screenshot");
expect((params as { body?: Record<string, unknown> } | undefined)?.body).toMatchObject({
targetId: "tab-1",
labels: true,
});
});
});

4
extensions/browser/src/cli/browser-cli-inspect.ts
View File

@@ -21,6 +21,7 @@ export function registerBrowserInspectCommands(
.option("--full-page", "Capture full scrollable page", false)
.option("--ref <ref>", "ARIA ref from ai snapshot")
.option("--element <selector>", "CSS selector for element screenshot")
.option("--labels", "Overlay role refs on the screenshot", false)
.option("--type <png|jpeg>", "Output type (default: png)", "png")
.action(async (targetId: string | undefined, opts, cmd) => {
const parent = parentOpts(cmd);
@@ -37,6 +38,7 @@ export function registerBrowserInspectCommands(
fullPage: Boolean(opts.fullPage),
ref: normalizeOptionalString(opts.ref),
element: normalizeOptionalString(opts.element),
labels: Boolean(opts.labels),
type: opts.type === "jpeg" ? "jpeg" : "png",
},
},
@@ -67,6 +69,7 @@ export function registerBrowserInspectCommands(
.option("--selector <sel>", "Role snapshot: scope to CSS selector")
.option("--frame <sel>", "Role snapshot: scope to an iframe selector")
.option("--labels", "Include viewport label overlay screenshot", false)
.option("--urls", "Append discovered link URLs to AI snapshots", false)
.option("--out <path>", "Write snapshot to a file")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
@@ -88,6 +91,7 @@ export function registerBrowserInspectCommands(
selector: normalizeOptionalString(opts.selector),
frame: normalizeOptionalString(opts.frame),
labels: opts.labels ? true : undefined,
urls: opts.urls ? true : undefined,
mode,
profile,
};

51
extensions/browser/src/cli/browser-cli-manage.test.ts
View File

@@ -179,4 +179,55 @@ describe("browser manage output", () => {
expect(output).not.toContain("supersecretpasswordvalue1234");
expect(output).not.toContain("supersecrettokenvalue1234567890");
});
it("prints a readable browser doctor report", async () => {
getBrowserManageCallBrowserRequestMock().mockImplementation(async (_opts: unknown, req) => {
if (req.path === "/") {
return {
enabled: true,
profile: "openclaw",
driver: "openclaw",
transport: "cdp",
running: true,
cdpReady: true,
cdpHttp: true,
pid: 4321,
cdpPort: 18792,
cdpUrl: "http://127.0.0.1:18792",
chosenBrowser: "chrome",
userDataDir: null,
color: "#00AA00",
headless: false,
noSandbox: false,
executablePath: null,
attachOnly: false,
};
}
if (req.path === "/profiles") {
return { profiles: [{ name: "openclaw", running: true }] };
}
if (req.path === "/tabs") {
return {
running: true,
tabs: [
{
targetId: "abc",
tabId: "t1",
suggestedTargetId: "t1",
title: "Example",
url: "https://example.com",
},
],
};
}
return {};
});
const program = createBrowserManageProgram();
await program.parseAsync(["browser", "doctor"], { from: "user" });
const output = getBrowserCliRuntime().log.mock.calls.at(-1)?.[0] as string;
expect(output).toContain("OK gateway: browser control endpoint reachable");
expect(output).toContain("OK tabs: 1 visible, use target t1");
});
});

114
extensions/browser/src/cli/browser-cli-manage.ts
View File

@@ -18,6 +18,12 @@ import {
const BROWSER_MANAGE_REQUEST_TIMEOUT_MS = 45_000;
type BrowserDoctorCheck = {
name: string;
ok: boolean;
detail?: string;
};
function resolveProfileQuery(profile?: string) {
return profile ? { profile } : undefined;
}
@@ -110,6 +116,96 @@ function logBrowserTabs(tabs: BrowserTab[], json?: boolean) {
);
}
function formatDoctorLine(check: BrowserDoctorCheck): string {
return `${check.ok ? "OK" : "FAIL"} ${check.name}${check.detail ? `: ${check.detail}` : ""}`;
}
async function runBrowserDoctor(parent: BrowserParentOpts, profile?: string) {
const checks: BrowserDoctorCheck[] = [];
let status: BrowserStatus | null = null;
try {
status = await fetchBrowserStatus(parent, profile);
checks.push({
name: "gateway",
ok: true,
detail: "browser control endpoint reachable",
});
} catch (err) {
checks.push({
name: "gateway",
ok: false,
detail: String(err),
});
return { ok: false, checks };
}
checks.push({
name: "plugin",
ok: status.enabled,
detail: status.enabled ? "enabled" : "disabled in config",
});
checks.push({
name: "profile",
ok: true,
detail: `${status.profile ?? "openclaw"} (${usesChromeMcpTransport(status) ? "chrome-mcp" : (status.transport ?? "cdp")})`,
});
checks.push({
name: "browser",
ok: status.running,
detail: status.running
? `running${status.cdpReady === false ? ", CDP not ready" : ""}`
: "not running; run `openclaw browser start`",
});
try {
const profiles = await callBrowserRequest<{ profiles: ProfileStatus[] }>(
parent,
{ method: "GET", path: "/profiles" },
{ timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS },
);
checks.push({
name: "profiles",
ok: true,
detail: `${profiles.profiles?.length ?? 0} configured`,
});
} catch (err) {
checks.push({
name: "profiles",
ok: false,
detail: String(err),
});
}
if (status.running) {
try {
const result = await callBrowserRequest<{ running: boolean; tabs: BrowserTab[] }>(
parent,
{
method: "GET",
path: "/tabs",
query: resolveProfileQuery(profile),
},
{ timeoutMs: BROWSER_MANAGE_REQUEST_TIMEOUT_MS },
);
const tabs = result.tabs ?? [];
checks.push({
name: "tabs",
ok: true,
detail: `${tabs.length} visible${tabs.length > 0 && tabs[0]?.suggestedTargetId ? `, use target ${tabs[0].suggestedTargetId}` : ""}`,
});
} catch (err) {
checks.push({
name: "tabs",
ok: false,
detail: String(err),
});
}
}
return { ok: checks.every((check) => check.ok), checks, status };
}
function usesChromeMcpTransport(params: {
transport?: BrowserTransport;
driver?: "openclaw" | "existing-session";
@@ -179,6 +275,24 @@ export function registerBrowserManageCommands(
});
});
browser
.command("doctor")
.description("Check browser plugin readiness")
.action(async (_opts, cmd) => {
const parent = parentOpts(cmd);
const profile = parent?.browserProfile;
await runBrowserCommand(async () => {
const result = await runBrowserDoctor(parent, profile);
if (printJsonResult(parent, result)) {
return;
}
defaultRuntime.log(result.checks.map(formatDoctorLine).join("\n"));
if (!result.ok) {
defaultRuntime.exit(1);
}
});
});
browser
.command("start")
.description("Start the browser (no-op if already running)")