diff --git a/assets/chrome-extension/background-utils.js b/assets/chrome-extension/background-utils.js index fe32d2c0616..82d43359c0a 100644 --- a/assets/chrome-extension/background-utils.js +++ b/assets/chrome-extension/background-utils.js @@ -46,3 +46,19 @@ export function isRetryableReconnectError(err) { } return true; } + +export function isMissingTabError(err) { + const message = (err instanceof Error ? err.message : String(err || "")).toLowerCase(); + return ( + message.includes("no tab with id") || + message.includes("no tab with given id") || + message.includes("tab not found") + ); +} + +export function isLastRemainingTab(allTabs, tabIdToClose) { + if (!Array.isArray(allTabs)) { + return true; + } + return allTabs.filter((tab) => tab && tab.id !== tabIdToClose).length === 0; +} diff --git a/assets/chrome-extension/background.js b/assets/chrome-extension/background.js index 0c4252f3a85..9031a156489 100644 --- a/assets/chrome-extension/background.js +++ b/assets/chrome-extension/background.js @@ -1,4 +1,10 @@ -import { buildRelayWsUrl, isRetryableReconnectError, reconnectDelayMs } from './background-utils.js' +import { + buildRelayWsUrl, + isLastRemainingTab, + isMissingTabError, + isRetryableReconnectError, + reconnectDelayMs, +} from './background-utils.js' const DEFAULT_PORT = 18792 @@ -41,6 +47,9 @@ const reattachPending = new Set() let reconnectAttempt = 0 let reconnectTimer = null +const TAB_VALIDATION_ATTEMPTS = 2 +const TAB_VALIDATION_RETRY_DELAY_MS = 1000 + function nowStack() { try { return new Error().stack || '' @@ -49,6 +58,37 @@ function nowStack() { } } +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +async function validateAttachedTab(tabId) { + try { + await chrome.tabs.get(tabId) + } catch { + return false + } + + for (let attempt = 0; attempt < TAB_VALIDATION_ATTEMPTS; attempt++) { + try { + await chrome.debugger.sendCommand({ tabId }, 'Runtime.evaluate', { + expression: '1', + returnByValue: true, + }) + return true + } catch (err) { + if (isMissingTabError(err)) { + return false + } + if (attempt < TAB_VALIDATION_ATTEMPTS - 1) { + await sleep(TAB_VALIDATION_RETRY_DELAY_MS) + } + } + } + + return false +} + async function getRelayPort() { const stored = await chrome.storage.local.get(['relayPort']) const raw = stored.relayPort @@ -108,15 +148,11 @@ async function rehydrateState() { tabBySession.set(entry.sessionId, entry.tabId) setBadge(entry.tabId, 'on') } - // Phase 2: validate asynchronously, remove dead tabs. + // Retry once so transient busy/navigation states do not permanently drop + // a still-attached tab after a service worker restart. for (const entry of entries) { - try { - await chrome.tabs.get(entry.tabId) - await chrome.debugger.sendCommand({ tabId: entry.tabId }, 'Runtime.evaluate', { - expression: '1', - returnByValue: true, - }) - } catch { + const valid = await validateAttachedTab(entry.tabId) + if (!valid) { tabs.delete(entry.tabId) tabBySession.delete(entry.sessionId) setBadge(entry.tabId, 'off') @@ -259,13 +295,10 @@ async function reannounceAttachedTabs() { for (const [tabId, tab] of tabs.entries()) { if (tab.state !== 'connected' || !tab.sessionId || !tab.targetId) continue - // Verify debugger is still attached. - try { - await chrome.debugger.sendCommand({ tabId }, 'Runtime.evaluate', { - expression: '1', - returnByValue: true, - }) - } catch { + // Retry once here as well; reconnect races can briefly make an otherwise + // healthy tab look unavailable. + const valid = await validateAttachedTab(tabId) + if (!valid) { tabs.delete(tabId) if (tab.sessionId) tabBySession.delete(tab.sessionId) setBadge(tabId, 'off') @@ -672,6 +705,11 @@ async function handleForwardCdpCommand(msg) { const toClose = target ? getTabByTargetId(target) : tabId if (!toClose) return { success: false } try { + const allTabs = await chrome.tabs.query({}) + if (isLastRemainingTab(allTabs, toClose)) { + console.warn('Refusing to close the last tab: this would kill the browser process') + return { success: false, error: 'Cannot close the last tab' } + } await chrome.tabs.remove(toClose) } catch { return { success: false } diff --git a/src/agents/tools/browser-tool.actions.ts b/src/agents/tools/browser-tool.actions.ts index 6c156e0cf2d..673585d16b3 100644 --- a/src/agents/tools/browser-tool.actions.ts +++ b/src/agents/tools/browser-tool.actions.ts @@ -112,16 +112,19 @@ export async function executeSnapshotAction(params: { }): Promise> { const { input, baseUrl, profile, proxyRequest } = params; const snapshotDefaults = loadConfig().browser?.snapshotDefaults; - const format = - input.snapshotFormat === "ai" || input.snapshotFormat === "aria" ? input.snapshotFormat : "ai"; - const mode = + const format: "ai" | "aria" | undefined = + input.snapshotFormat === "ai" || input.snapshotFormat === "aria" + ? input.snapshotFormat + : undefined; + const mode: "efficient" | undefined = input.mode === "efficient" ? "efficient" - : format === "ai" && snapshotDefaults?.mode === "efficient" + : format !== "aria" && snapshotDefaults?.mode === "efficient" ? "efficient" : undefined; const labels = typeof input.labels === "boolean" ? input.labels : undefined; - const refs = input.refs === "aria" || input.refs === "role" ? input.refs : undefined; + const refs: "aria" | "role" | undefined = + input.refs === "aria" || input.refs === "role" ? input.refs : undefined; const hasMaxChars = Object.hasOwn(input, "maxChars"); const targetId = typeof input.targetId === "string" ? input.targetId.trim() : undefined; const limit = @@ -130,6 +133,12 @@ export async function executeSnapshotAction(params: { typeof input.maxChars === "number" && Number.isFinite(input.maxChars) && input.maxChars > 0 ? Math.floor(input.maxChars) : undefined; + const interactive = typeof input.interactive === "boolean" ? input.interactive : undefined; + const compact = typeof input.compact === "boolean" ? input.compact : undefined; + const depth = + typeof input.depth === "number" && Number.isFinite(input.depth) ? input.depth : undefined; + const selector = typeof input.selector === "string" ? input.selector.trim() : undefined; + const frame = typeof input.frame === "string" ? input.frame.trim() : undefined; const resolvedMaxChars = format === "ai" ? hasMaxChars @@ -137,46 +146,32 @@ export async function executeSnapshotAction(params: { : mode === "efficient" ? undefined : DEFAULT_AI_SNAPSHOT_MAX_CHARS - : undefined; - const interactive = typeof input.interactive === "boolean" ? input.interactive : undefined; - const compact = typeof input.compact === "boolean" ? input.compact : undefined; - const depth = - typeof input.depth === "number" && Number.isFinite(input.depth) ? input.depth : undefined; - const selector = typeof input.selector === "string" ? input.selector.trim() : undefined; - const frame = typeof input.frame === "string" ? input.frame.trim() : undefined; + : hasMaxChars + ? maxChars + : undefined; + const snapshotQuery = { + ...(format ? { format } : {}), + targetId, + limit, + ...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}), + refs, + interactive, + compact, + depth, + selector, + frame, + labels, + mode, + }; const snapshot = proxyRequest ? ((await proxyRequest({ method: "GET", path: "/snapshot", profile, - query: { - format, - targetId, - limit, - ...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}), - refs, - interactive, - compact, - depth, - selector, - frame, - labels, - mode, - }, + query: snapshotQuery, })) as Awaited>) : await browserSnapshot(baseUrl, { - format, - targetId, - limit, - ...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}), - refs, - interactive, - compact, - depth, - selector, - frame, - labels, - mode, + ...snapshotQuery, profile, }); if (snapshot.format === "ai") { diff --git a/src/agents/tools/browser-tool.test.ts b/src/agents/tools/browser-tool.test.ts index 79358cf1665..81996afb419 100644 --- a/src/agents/tools/browser-tool.test.ts +++ b/src/agents/tools/browser-tool.test.ts @@ -127,7 +127,7 @@ function registerBrowserToolAfterEachReset() { } async function runSnapshotToolCall(params: { - snapshotFormat: "ai" | "aria"; + snapshotFormat?: "ai" | "aria"; refs?: "aria" | "dom"; maxChars?: number; profile?: string; @@ -243,6 +243,23 @@ describe("browser tool snapshot maxChars", () => { ); }); + it("lets the server choose snapshot format when the user does not request one", async () => { + const tool = createBrowserTool(); + await tool.execute?.("call-1", { action: "snapshot", profile: "chrome" }); + + expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith( + undefined, + expect.objectContaining({ + profile: "chrome", + }), + ); + const opts = browserClientMocks.browserSnapshot.mock.calls.at(-1)?.[1] as + | { format?: string; maxChars?: number } + | undefined; + expect(opts?.format).toBeUndefined(); + expect(Object.hasOwn(opts ?? {}, "maxChars")).toBe(false); + }); + it("routes to node proxy when target=node", async () => { mockSingleBrowserProxyNode(); const tool = createBrowserTool(); @@ -250,15 +267,44 @@ describe("browser tool snapshot maxChars", () => { expect(gatewayMocks.callGatewayTool).toHaveBeenCalledWith( "node.invoke", - { timeoutMs: 20000 }, + { timeoutMs: 25000 }, expect.objectContaining({ nodeId: "node-1", command: "browser.proxy", + params: expect.objectContaining({ + timeoutMs: 20000, + }), }), ); expect(browserClientMocks.browserStatus).not.toHaveBeenCalled(); }); + it("gives node.invoke extra slack beyond the default proxy timeout", async () => { + mockSingleBrowserProxyNode(); + gatewayMocks.callGatewayTool.mockResolvedValueOnce({ + ok: true, + payload: { + result: { ok: true, running: true }, + }, + }); + const tool = createBrowserTool(); + await tool.execute?.("call-1", { + action: "dialog", + target: "node", + accept: true, + }); + + expect(gatewayMocks.callGatewayTool).toHaveBeenCalledWith( + "node.invoke", + { timeoutMs: 25000 }, + expect.objectContaining({ + params: expect.objectContaining({ + timeoutMs: 20000, + }), + }), + ); + }); + it("keeps sandbox bridge url when node proxy is available", async () => { mockSingleBrowserProxyNode(); const tool = createBrowserTool({ sandboxBridgeUrl: "http://127.0.0.1:9999" }); diff --git a/src/agents/tools/browser-tool.ts b/src/agents/tools/browser-tool.ts index 80faf99a1e4..200013ff1a7 100644 --- a/src/agents/tools/browser-tool.ts +++ b/src/agents/tools/browser-tool.ts @@ -115,6 +115,7 @@ type BrowserProxyResult = { }; const DEFAULT_BROWSER_PROXY_TIMEOUT_MS = 20_000; +const BROWSER_PROXY_GATEWAY_TIMEOUT_SLACK_MS = 5_000; type BrowserNodeTarget = { nodeId: string; @@ -206,10 +207,11 @@ async function callBrowserProxy(params: { timeoutMs?: number; profile?: string; }): Promise { - const gatewayTimeoutMs = + const proxyTimeoutMs = typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs) ? Math.max(1, Math.floor(params.timeoutMs)) : DEFAULT_BROWSER_PROXY_TIMEOUT_MS; + const gatewayTimeoutMs = proxyTimeoutMs + BROWSER_PROXY_GATEWAY_TIMEOUT_SLACK_MS; const payload = await callGatewayTool<{ payloadJSON?: string; payload?: string }>( "node.invoke", { timeoutMs: gatewayTimeoutMs }, @@ -221,7 +223,7 @@ async function callBrowserProxy(params: { path: params.path, query: params.query, body: params.body, - timeoutMs: params.timeoutMs, + timeoutMs: proxyTimeoutMs, profile: params.profile, }, idempotencyKey: crypto.randomUUID(), diff --git a/src/browser/chrome-extension-background-utils.test.ts b/src/browser/chrome-extension-background-utils.test.ts index 74b767cb269..b22b602116c 100644 --- a/src/browser/chrome-extension-background-utils.test.ts +++ b/src/browser/chrome-extension-background-utils.test.ts @@ -4,6 +4,11 @@ import { describe, expect, it } from "vitest"; type BackgroundUtilsModule = { buildRelayWsUrl: (port: number, gatewayToken: string) => Promise; deriveRelayToken: (gatewayToken: string, port: number) => Promise; + isLastRemainingTab: ( + allTabs: Array<{ id?: number | undefined } | null | undefined>, + tabIdToClose: number, + ) => boolean; + isMissingTabError: (err: unknown) => boolean; isRetryableReconnectError: (err: unknown) => boolean; reconnectDelayMs: ( attempt: number, @@ -26,8 +31,14 @@ async function loadBackgroundUtils(): Promise { } } -const { buildRelayWsUrl, deriveRelayToken, isRetryableReconnectError, reconnectDelayMs } = - await loadBackgroundUtils(); +const { + buildRelayWsUrl, + deriveRelayToken, + isLastRemainingTab, + isMissingTabError, + isRetryableReconnectError, + reconnectDelayMs, +} = await loadBackgroundUtils(); describe("chrome extension background utils", () => { it("derives relay token as HMAC-SHA256 of gateway token and port", async () => { @@ -107,4 +118,16 @@ describe("chrome extension background utils", () => { expect(isRetryableReconnectError(new Error("WebSocket connect timeout"))).toBe(true); expect(isRetryableReconnectError(new Error("Relay server not reachable"))).toBe(true); }); + + it("recognizes missing-tab debugger errors", () => { + expect(isMissingTabError(new Error("No tab with given id"))).toBe(true); + expect(isMissingTabError(new Error("tab not found"))).toBe(true); + expect(isMissingTabError(new Error("Cannot access a chrome:// URL"))).toBe(false); + }); + + it("blocks closing the final remaining tab only", () => { + expect(isLastRemainingTab([{ id: 7 }], 7)).toBe(true); + expect(isLastRemainingTab([{ id: 7 }, { id: 8 }], 7)).toBe(false); + expect(isLastRemainingTab([{ id: 7 }, { id: 8 }], 8)).toBe(false); + }); }); diff --git a/src/browser/client.test.ts b/src/browser/client.test.ts index 7922fd94820..a4f95c23007 100644 --- a/src/browser/client.test.ts +++ b/src/browser/client.test.ts @@ -101,6 +101,21 @@ describe("browser client", () => { expect(parsed.searchParams.get("refs")).toBe("aria"); }); + it("omits format when the caller wants server-side snapshot capability defaults", async () => { + const calls: string[] = []; + stubSnapshotFetch(calls); + + await browserSnapshot("http://127.0.0.1:18791", { + profile: "chrome", + }); + + const snapshotCall = calls.find((url) => url.includes("/snapshot?")); + expect(snapshotCall).toBeTruthy(); + const parsed = new URL(snapshotCall as string); + expect(parsed.searchParams.get("format")).toBeNull(); + expect(parsed.searchParams.get("profile")).toBe("chrome"); + }); + it("uses the expected endpoints + methods for common calls", async () => { const calls: Array<{ url: string; init?: RequestInit }> = []; diff --git a/src/browser/client.ts b/src/browser/client.ts index 5085825cb6e..76b799bde64 100644 --- a/src/browser/client.ts +++ b/src/browser/client.ts @@ -276,7 +276,7 @@ export async function browserTabAction( export async function browserSnapshot( baseUrl: string | undefined, opts: { - format: "aria" | "ai"; + format?: "aria" | "ai"; targetId?: string; limit?: number; maxChars?: number; @@ -292,7 +292,9 @@ export async function browserSnapshot( }, ): Promise { const q = new URLSearchParams(); - q.set("format", opts.format); + if (opts.format) { + q.set("format", opts.format); + } if (opts.targetId) { q.set("targetId", opts.targetId); } diff --git a/src/browser/pw-session.get-page-for-targetid.extension-fallback.test.ts b/src/browser/pw-session.get-page-for-targetid.extension-fallback.test.ts index f1909ad33fb..43f1a6c7e09 100644 --- a/src/browser/pw-session.get-page-for-targetid.extension-fallback.test.ts +++ b/src/browser/pw-session.get-page-for-targetid.extension-fallback.test.ts @@ -115,4 +115,67 @@ describe("pw-session getPageForTargetId", () => { fetchSpy.mockRestore(); } }); + + it("resolves extension-relay pages from /json/list without probing page CDP sessions first", async () => { + const pageOn = vi.fn(); + const contextOn = vi.fn(); + const browserOn = vi.fn(); + const browserClose = vi.fn(async () => {}); + const newCDPSession = vi.fn(async () => { + throw new Error("Target.attachToBrowserTarget: Not allowed"); + }); + + const context = { + pages: () => [], + on: contextOn, + newCDPSession, + } as unknown as import("playwright-core").BrowserContext; + + const pageA = { + on: pageOn, + context: () => context, + url: () => "https://alpha.example", + } as unknown as import("playwright-core").Page; + const pageB = { + on: pageOn, + context: () => context, + url: () => "https://beta.example", + } as unknown as import("playwright-core").Page; + + (context as unknown as { pages: () => unknown[] }).pages = () => [pageA, pageB]; + + const browser = { + contexts: () => [context], + on: browserOn, + close: browserClose, + } as unknown as import("playwright-core").Browser; + + connectOverCdpSpy.mockResolvedValue(browser); + getChromeWebSocketUrlSpy.mockResolvedValue(null); + + const fetchSpy = vi.spyOn(globalThis, "fetch"); + fetchSpy + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ Browser: "OpenClaw/extension-relay" }), + } as Response) + .mockResolvedValueOnce({ + ok: true, + json: async () => [ + { id: "TARGET_A", url: "https://alpha.example" }, + { id: "TARGET_B", url: "https://beta.example" }, + ], + } as Response); + + try { + const resolved = await getPageForTargetId({ + cdpUrl: "http://127.0.0.1:19993", + targetId: "TARGET_B", + }); + expect(resolved).toBe(pageB); + expect(newCDPSession).not.toHaveBeenCalled(); + } finally { + fetchSpy.mockRestore(); + } + }); }); diff --git a/src/browser/pw-session.page-cdp.test.ts b/src/browser/pw-session.page-cdp.test.ts new file mode 100644 index 00000000000..1347cca20a1 --- /dev/null +++ b/src/browser/pw-session.page-cdp.test.ts @@ -0,0 +1,94 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const cdpHelperMocks = vi.hoisted(() => ({ + fetchJson: vi.fn(), + withCdpSocket: vi.fn(), +})); + +const chromeMocks = vi.hoisted(() => ({ + getChromeWebSocketUrl: vi.fn(async () => "ws://127.0.0.1:18792/cdp"), +})); + +vi.mock("./cdp.helpers.js", async () => { + const actual = await vi.importActual("./cdp.helpers.js"); + return { + ...actual, + fetchJson: cdpHelperMocks.fetchJson, + withCdpSocket: cdpHelperMocks.withCdpSocket, + }; +}); + +vi.mock("./chrome.js", () => chromeMocks); + +import { isExtensionRelayCdpEndpoint, withPageScopedCdpClient } from "./pw-session.page-cdp.js"; + +describe("pw-session page-scoped CDP client", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("uses raw relay /cdp commands for extension endpoints when targetId is known", async () => { + cdpHelperMocks.fetchJson.mockResolvedValue({ Browser: "OpenClaw/extension-relay" }); + const send = vi.fn(async () => ({ ok: true })); + cdpHelperMocks.withCdpSocket.mockImplementation(async (_wsUrl, fn) => await fn(send)); + const newCDPSession = vi.fn(); + const page = { + context: () => ({ + newCDPSession, + }), + }; + + await withPageScopedCdpClient({ + cdpUrl: "http://127.0.0.1:18792", + page: page as never, + targetId: "tab-1", + fn: async (pageSend) => { + await pageSend("Page.bringToFront", { foo: "bar" }); + }, + }); + + expect(send).toHaveBeenCalledWith("Page.bringToFront", { + foo: "bar", + targetId: "tab-1", + }); + expect(newCDPSession).not.toHaveBeenCalled(); + }); + + it("falls back to Playwright page sessions for non-relay endpoints", async () => { + cdpHelperMocks.fetchJson.mockResolvedValue({ Browser: "Chrome/145.0" }); + const sessionSend = vi.fn(async () => ({ ok: true })); + const sessionDetach = vi.fn(async () => {}); + const newCDPSession = vi.fn(async () => ({ + send: sessionSend, + detach: sessionDetach, + })); + const page = { + context: () => ({ + newCDPSession, + }), + }; + + await withPageScopedCdpClient({ + cdpUrl: "http://127.0.0.1:9222", + page: page as never, + targetId: "tab-1", + fn: async (pageSend) => { + await pageSend("Emulation.setLocaleOverride", { locale: "en-US" }); + }, + }); + + expect(newCDPSession).toHaveBeenCalledWith(page); + expect(sessionSend).toHaveBeenCalledWith("Emulation.setLocaleOverride", { locale: "en-US" }); + expect(sessionDetach).toHaveBeenCalledTimes(1); + expect(cdpHelperMocks.withCdpSocket).not.toHaveBeenCalled(); + }); + + it("caches extension-relay endpoint detection by cdpUrl", async () => { + cdpHelperMocks.fetchJson.mockResolvedValue({ Browser: "OpenClaw/extension-relay" }); + + await expect(isExtensionRelayCdpEndpoint("http://127.0.0.1:19992")).resolves.toBe(true); + await expect(isExtensionRelayCdpEndpoint("http://127.0.0.1:19992/")).resolves.toBe(true); + + expect(cdpHelperMocks.fetchJson).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/browser/pw-session.page-cdp.ts b/src/browser/pw-session.page-cdp.ts new file mode 100644 index 00000000000..8c2109293cd --- /dev/null +++ b/src/browser/pw-session.page-cdp.ts @@ -0,0 +1,81 @@ +import type { CDPSession, Page } from "playwright-core"; +import { + appendCdpPath, + fetchJson, + normalizeCdpHttpBaseForJsonEndpoints, + withCdpSocket, +} from "./cdp.helpers.js"; +import { getChromeWebSocketUrl } from "./chrome.js"; + +const OPENCLAW_EXTENSION_RELAY_BROWSER = "OpenClaw/extension-relay"; + +type PageCdpSend = (method: string, params?: Record) => Promise; + +const extensionRelayByCdpUrl = new Map(); + +function normalizeCdpUrl(raw: string) { + return raw.replace(/\/$/, ""); +} + +export async function isExtensionRelayCdpEndpoint(cdpUrl: string): Promise { + const normalized = normalizeCdpUrl(cdpUrl); + const cached = extensionRelayByCdpUrl.get(normalized); + if (cached !== undefined) { + return cached; + } + + try { + const cdpHttpBase = normalizeCdpHttpBaseForJsonEndpoints(normalized); + const version = await fetchJson<{ Browser?: string }>( + appendCdpPath(cdpHttpBase, "/json/version"), + 2000, + ); + const isRelay = String(version?.Browser ?? "").trim() === OPENCLAW_EXTENSION_RELAY_BROWSER; + extensionRelayByCdpUrl.set(normalized, isRelay); + return isRelay; + } catch { + extensionRelayByCdpUrl.set(normalized, false); + return false; + } +} + +async function withPlaywrightPageCdpSession( + page: Page, + fn: (session: CDPSession) => Promise, +): Promise { + const session = await page.context().newCDPSession(page); + try { + return await fn(session); + } finally { + await session.detach().catch(() => {}); + } +} + +export async function withPageScopedCdpClient(opts: { + cdpUrl: string; + page: Page; + targetId?: string; + fn: (send: PageCdpSend) => Promise; +}): Promise { + const targetId = opts.targetId?.trim(); + if (targetId && (await isExtensionRelayCdpEndpoint(opts.cdpUrl))) { + const wsUrl = await getChromeWebSocketUrl(opts.cdpUrl, 2000); + if (!wsUrl) { + throw new Error("CDP websocket unavailable"); + } + return await withCdpSocket(wsUrl, async (send) => { + return await opts.fn((method, params) => send(method, { ...params, targetId })); + }); + } + + return await withPlaywrightPageCdpSession(opts.page, async (session) => { + return await opts.fn((method, params) => + ( + session.send as unknown as ( + method: string, + params?: Record, + ) => Promise + )(method, params), + ); + }); +} diff --git a/src/browser/pw-session.ts b/src/browser/pw-session.ts index a5f1b11ec02..53f9c241142 100644 --- a/src/browser/pw-session.ts +++ b/src/browser/pw-session.ts @@ -24,6 +24,7 @@ import { assertBrowserNavigationResultAllowed, withBrowserNavigationPolicy, } from "./navigation-guard.js"; +import { isExtensionRelayCdpEndpoint, withPageScopedCdpClient } from "./pw-session.page-cdp.js"; export type BrowserConsoleMessage = { type: string; @@ -398,14 +399,70 @@ async function pageTargetId(page: Page): Promise { } } +function matchPageByTargetList( + pages: Page[], + targets: Array<{ id: string; url: string; title?: string }>, + targetId: string, +): Page | null { + const target = targets.find((entry) => entry.id === targetId); + if (!target) { + return null; + } + + const urlMatch = pages.filter((page) => page.url() === target.url); + if (urlMatch.length === 1) { + return urlMatch[0] ?? null; + } + if (urlMatch.length > 1) { + const sameUrlTargets = targets.filter((entry) => entry.url === target.url); + if (sameUrlTargets.length === urlMatch.length) { + const idx = sameUrlTargets.findIndex((entry) => entry.id === targetId); + if (idx >= 0 && idx < urlMatch.length) { + return urlMatch[idx] ?? null; + } + } + } + return null; +} + +async function findPageByTargetIdViaTargetList( + pages: Page[], + targetId: string, + cdpUrl: string, +): Promise { + const cdpHttpBase = normalizeCdpHttpBaseForJsonEndpoints(cdpUrl); + const targets = await fetchJson< + Array<{ + id: string; + url: string; + title?: string; + }> + >(appendCdpPath(cdpHttpBase, "/json/list"), 2000); + return matchPageByTargetList(pages, targets, targetId); +} + async function findPageByTargetId( browser: Browser, targetId: string, cdpUrl?: string, ): Promise { const pages = await getAllPages(browser); + const isExtensionRelay = cdpUrl + ? await isExtensionRelayCdpEndpoint(cdpUrl).catch(() => false) + : false; + if (cdpUrl && isExtensionRelay) { + try { + const matched = await findPageByTargetIdViaTargetList(pages, targetId, cdpUrl); + if (matched) { + return matched; + } + } catch { + // Ignore fetch errors and fall through to best-effort single-page fallback. + } + return pages.length === 1 ? (pages[0] ?? null) : null; + } + let resolvedViaCdp = false; - // First, try the standard CDP session approach for (const page of pages) { let tid: string | null = null; try { @@ -418,46 +475,16 @@ async function findPageByTargetId( return page; } } - // Extension relays can block CDP attachment APIs entirely. If that happens and - // Playwright only exposes one page, return it as the best available mapping. - if (!resolvedViaCdp && pages.length === 1) { - return pages[0]; - } - // If CDP sessions fail (e.g., extension relay blocks Target.attachToBrowserTarget), - // fall back to URL-based matching using the /json/list endpoint if (cdpUrl) { try { - const cdpHttpBase = normalizeCdpHttpBaseForJsonEndpoints(cdpUrl); - const targets = await fetchJson< - Array<{ - id: string; - url: string; - title?: string; - }> - >(appendCdpPath(cdpHttpBase, "/json/list"), 2000); - const target = targets.find((t) => t.id === targetId); - if (target) { - // Try to find a page with matching URL - const urlMatch = pages.filter((p) => p.url() === target.url); - if (urlMatch.length === 1) { - return urlMatch[0]; - } - // If multiple URL matches, use index-based matching as fallback - // This works when Playwright and the relay enumerate tabs in the same order - if (urlMatch.length > 1) { - const sameUrlTargets = targets.filter((t) => t.url === target.url); - if (sameUrlTargets.length === urlMatch.length) { - const idx = sameUrlTargets.findIndex((t) => t.id === targetId); - if (idx >= 0 && idx < urlMatch.length) { - return urlMatch[idx]; - } - } - } - } + return await findPageByTargetIdViaTargetList(pages, targetId, cdpUrl); } catch { - // Ignore fetch errors and fall through to return null + // Ignore fetch errors and fall through to return null. } } + if (!resolvedViaCdp && pages.length === 1) { + return pages[0] ?? null; + } return null; } @@ -806,14 +833,18 @@ export async function focusPageByTargetIdViaPlaywright(opts: { try { await page.bringToFront(); } catch (err) { - const session = await page.context().newCDPSession(page); try { - await session.send("Page.bringToFront"); + await withPageScopedCdpClient({ + cdpUrl: opts.cdpUrl, + page, + targetId: opts.targetId, + fn: async (send) => { + await send("Page.bringToFront"); + }, + }); return; } catch { throw err; - } finally { - await session.detach().catch(() => {}); } } } diff --git a/src/browser/pw-tools-core.snapshot.ts b/src/browser/pw-tools-core.snapshot.ts index 419aba6357d..b3dc8dec7b0 100644 --- a/src/browser/pw-tools-core.snapshot.ts +++ b/src/browser/pw-tools-core.snapshot.ts @@ -19,6 +19,7 @@ import { storeRoleRefsForTarget, type WithSnapshotForAI, } from "./pw-session.js"; +import { withPageScopedCdpClient } from "./pw-session.page-cdp.js"; export async function snapshotAriaViaPlaywright(opts: { cdpUrl: string; @@ -31,17 +32,21 @@ export async function snapshotAriaViaPlaywright(opts: { targetId: opts.targetId, }); ensurePageState(page); - const session = await page.context().newCDPSession(page); - try { - await session.send("Accessibility.enable").catch(() => {}); - const res = (await session.send("Accessibility.getFullAXTree")) as { - nodes?: RawAXNode[]; - }; - const nodes = Array.isArray(res?.nodes) ? res.nodes : []; - return { nodes: formatAriaSnapshot(nodes, limit) }; - } finally { - await session.detach().catch(() => {}); - } + const res = (await withPageScopedCdpClient({ + cdpUrl: opts.cdpUrl, + page, + targetId: opts.targetId, + fn: async (send) => { + await send("Accessibility.enable").catch(() => {}); + return (await send("Accessibility.getFullAXTree")) as { + nodes?: RawAXNode[]; + }; + }, + })) as { + nodes?: RawAXNode[]; + }; + const nodes = Array.isArray(res?.nodes) ? res.nodes : []; + return { nodes: formatAriaSnapshot(nodes, limit) }; } export async function snapshotAiViaPlaywright(opts: { diff --git a/src/browser/pw-tools-core.state.ts b/src/browser/pw-tools-core.state.ts index aeeb8859d8f..580fadba108 100644 --- a/src/browser/pw-tools-core.state.ts +++ b/src/browser/pw-tools-core.state.ts @@ -1,15 +1,6 @@ -import type { CDPSession, Page } from "playwright-core"; import { devices as playwrightDevices } from "playwright-core"; import { ensurePageState, getPageForTargetId } from "./pw-session.js"; - -async function withCdpSession(page: Page, fn: (session: CDPSession) => Promise): Promise { - const session = await page.context().newCDPSession(page); - try { - return await fn(session); - } finally { - await session.detach().catch(() => {}); - } -} +import { withPageScopedCdpClient } from "./pw-session.page-cdp.js"; export async function setOfflineViaPlaywright(opts: { cdpUrl: string; @@ -112,15 +103,20 @@ export async function setLocaleViaPlaywright(opts: { if (!locale) { throw new Error("locale is required"); } - await withCdpSession(page, async (session) => { - try { - await session.send("Emulation.setLocaleOverride", { locale }); - } catch (err) { - if (String(err).includes("Another locale override is already in effect")) { - return; + await withPageScopedCdpClient({ + cdpUrl: opts.cdpUrl, + page, + targetId: opts.targetId, + fn: async (send) => { + try { + await send("Emulation.setLocaleOverride", { locale }); + } catch (err) { + if (String(err).includes("Another locale override is already in effect")) { + return; + } + throw err; } - throw err; - } + }, }); } @@ -135,19 +131,24 @@ export async function setTimezoneViaPlaywright(opts: { if (!timezoneId) { throw new Error("timezoneId is required"); } - await withCdpSession(page, async (session) => { - try { - await session.send("Emulation.setTimezoneOverride", { timezoneId }); - } catch (err) { - const msg = String(err); - if (msg.includes("Timezone override is already in effect")) { - return; + await withPageScopedCdpClient({ + cdpUrl: opts.cdpUrl, + page, + targetId: opts.targetId, + fn: async (send) => { + try { + await send("Emulation.setTimezoneOverride", { timezoneId }); + } catch (err) { + const msg = String(err); + if (msg.includes("Timezone override is already in effect")) { + return; + } + if (msg.includes("Invalid timezone")) { + throw new Error(`Invalid timezone ID: ${timezoneId}`, { cause: err }); + } + throw err; } - if (msg.includes("Invalid timezone")) { - throw new Error(`Invalid timezone ID: ${timezoneId}`, { cause: err }); - } - throw err; - } + }, }); } @@ -183,27 +184,32 @@ export async function setDeviceViaPlaywright(opts: { }); } - await withCdpSession(page, async (session) => { - if (descriptor.userAgent || descriptor.locale) { - await session.send("Emulation.setUserAgentOverride", { - userAgent: descriptor.userAgent ?? "", - acceptLanguage: descriptor.locale ?? undefined, - }); - } - if (descriptor.viewport) { - await session.send("Emulation.setDeviceMetricsOverride", { - mobile: Boolean(descriptor.isMobile), - width: descriptor.viewport.width, - height: descriptor.viewport.height, - deviceScaleFactor: descriptor.deviceScaleFactor ?? 1, - screenWidth: descriptor.viewport.width, - screenHeight: descriptor.viewport.height, - }); - } - if (descriptor.hasTouch) { - await session.send("Emulation.setTouchEmulationEnabled", { - enabled: true, - }); - } + await withPageScopedCdpClient({ + cdpUrl: opts.cdpUrl, + page, + targetId: opts.targetId, + fn: async (send) => { + if (descriptor.userAgent || descriptor.locale) { + await send("Emulation.setUserAgentOverride", { + userAgent: descriptor.userAgent ?? "", + acceptLanguage: descriptor.locale ?? undefined, + }); + } + if (descriptor.viewport) { + await send("Emulation.setDeviceMetricsOverride", { + mobile: Boolean(descriptor.isMobile), + width: descriptor.viewport.width, + height: descriptor.viewport.height, + deviceScaleFactor: descriptor.deviceScaleFactor ?? 1, + screenWidth: descriptor.viewport.width, + screenHeight: descriptor.viewport.height, + }); + } + if (descriptor.hasTouch) { + await send("Emulation.setTouchEmulationEnabled", { + enabled: true, + }); + } + }, }); } diff --git a/src/browser/server-lifecycle.test.ts b/src/browser/server-lifecycle.test.ts index 9c11a3d48f8..e2395f99f04 100644 --- a/src/browser/server-lifecycle.test.ts +++ b/src/browser/server-lifecycle.test.ts @@ -5,17 +5,27 @@ const { resolveProfileMock, ensureChromeExtensionRelayServerMock } = vi.hoisted( ensureChromeExtensionRelayServerMock: vi.fn(), })); +const { stopOpenClawChromeMock, stopChromeExtensionRelayServerMock } = vi.hoisted(() => ({ + stopOpenClawChromeMock: vi.fn(async () => {}), + stopChromeExtensionRelayServerMock: vi.fn(async () => true), +})); + const { createBrowserRouteContextMock, listKnownProfileNamesMock } = vi.hoisted(() => ({ createBrowserRouteContextMock: vi.fn(), listKnownProfileNamesMock: vi.fn(), })); +vi.mock("./chrome.js", () => ({ + stopOpenClawChrome: stopOpenClawChromeMock, +})); + vi.mock("./config.js", () => ({ resolveProfile: resolveProfileMock, })); vi.mock("./extension-relay.js", () => ({ ensureChromeExtensionRelayServer: ensureChromeExtensionRelayServerMock, + stopChromeExtensionRelayServer: stopChromeExtensionRelayServerMock, })); vi.mock("./server-context.js", () => ({ @@ -76,6 +86,8 @@ describe("stopKnownBrowserProfiles", () => { beforeEach(() => { createBrowserRouteContextMock.mockClear(); listKnownProfileNamesMock.mockClear(); + stopOpenClawChromeMock.mockClear(); + stopChromeExtensionRelayServerMock.mockClear(); }); it("stops all known profiles and ignores per-profile failures", async () => { @@ -104,6 +116,53 @@ describe("stopKnownBrowserProfiles", () => { expect(onWarn).not.toHaveBeenCalled(); }); + it("stops tracked runtime browsers even when the profile no longer resolves", async () => { + listKnownProfileNamesMock.mockReturnValue(["deleted-local", "deleted-extension"]); + createBrowserRouteContextMock.mockReturnValue({ + forProfile: vi.fn(() => { + throw new Error("profile not found"); + }), + }); + const localRuntime = { + profile: { + name: "deleted-local", + driver: "openclaw", + }, + running: { + pid: 42, + cdpPort: 18888, + }, + }; + const launchedBrowser = localRuntime.running; + const extensionRuntime = { + profile: { + name: "deleted-extension", + driver: "extension", + cdpUrl: "http://127.0.0.1:19999", + }, + running: null, + }; + const profiles = new Map([ + ["deleted-local", localRuntime], + ["deleted-extension", extensionRuntime], + ]); + const state = { + resolved: { profiles: {} }, + profiles, + }; + + await stopKnownBrowserProfiles({ + getState: () => state as never, + onWarn: vi.fn(), + }); + + expect(stopOpenClawChromeMock).toHaveBeenCalledWith(launchedBrowser); + expect(localRuntime.running).toBeNull(); + expect(stopChromeExtensionRelayServerMock).toHaveBeenCalledWith({ + cdpUrl: "http://127.0.0.1:19999", + }); + }); + it("warns when profile enumeration fails", async () => { listKnownProfileNamesMock.mockImplementation(() => { throw new Error("oops"); diff --git a/src/browser/server-lifecycle.ts b/src/browser/server-lifecycle.ts index 10a4569095a..7053d924b6d 100644 --- a/src/browser/server-lifecycle.ts +++ b/src/browser/server-lifecycle.ts @@ -1,6 +1,10 @@ +import { stopOpenClawChrome } from "./chrome.js"; import type { ResolvedBrowserConfig } from "./config.js"; import { resolveProfile } from "./config.js"; -import { ensureChromeExtensionRelayServer } from "./extension-relay.js"; +import { + ensureChromeExtensionRelayServer, + stopChromeExtensionRelayServer, +} from "./extension-relay.js"; import { type BrowserServerState, createBrowserRouteContext, @@ -40,6 +44,18 @@ export async function stopKnownBrowserProfiles(params: { try { for (const name of listKnownProfileNames(current)) { try { + const runtime = current.profiles.get(name); + if (runtime?.running) { + await stopOpenClawChrome(runtime.running); + runtime.running = null; + continue; + } + if (runtime?.profile.driver === "extension") { + await stopChromeExtensionRelayServer({ cdpUrl: runtime.profile.cdpUrl }).catch( + () => false, + ); + continue; + } await ctx.forProfile(name).stopRunningBrowser(); } catch { // ignore diff --git a/src/node-host/invoke-browser.test.ts b/src/node-host/invoke-browser.test.ts new file mode 100644 index 00000000000..ca9232823c1 --- /dev/null +++ b/src/node-host/invoke-browser.test.ts @@ -0,0 +1,99 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const controlServiceMocks = vi.hoisted(() => ({ + createBrowserControlContext: vi.fn(() => ({ control: true })), + startBrowserControlServiceFromConfig: vi.fn(async () => true), +})); + +const dispatcherMocks = vi.hoisted(() => ({ + dispatch: vi.fn(), + createBrowserRouteDispatcher: vi.fn(() => ({ + dispatch: dispatcherMocks.dispatch, + })), +})); + +const configMocks = vi.hoisted(() => ({ + loadConfig: vi.fn(() => ({ + browser: {}, + nodeHost: { browserProxy: { enabled: true } }, + })), +})); + +const browserConfigMocks = vi.hoisted(() => ({ + resolveBrowserConfig: vi.fn(() => ({ + enabled: true, + defaultProfile: "chrome", + })), +})); + +vi.mock("../browser/control-service.js", () => controlServiceMocks); +vi.mock("../browser/routes/dispatcher.js", () => dispatcherMocks); +vi.mock("../config/config.js", () => configMocks); +vi.mock("../browser/config.js", () => browserConfigMocks); +vi.mock("../media/mime.js", () => ({ + detectMime: vi.fn(async () => "image/png"), +})); + +import { runBrowserProxyCommand } from "./invoke-browser.js"; + +describe("runBrowserProxyCommand", () => { + beforeEach(() => { + vi.clearAllMocks(); + configMocks.loadConfig.mockReturnValue({ + browser: {}, + nodeHost: { browserProxy: { enabled: true } }, + }); + browserConfigMocks.resolveBrowserConfig.mockReturnValue({ + enabled: true, + defaultProfile: "chrome", + }); + controlServiceMocks.startBrowserControlServiceFromConfig.mockResolvedValue(true); + }); + + it("adds profile and browser status details on ws-backed timeouts", async () => { + dispatcherMocks.dispatch + .mockImplementationOnce(async () => { + await new Promise(() => {}); + }) + .mockResolvedValueOnce({ + status: 200, + body: { + running: true, + cdpHttp: true, + cdpReady: false, + cdpUrl: "http://127.0.0.1:18792", + }, + }); + + await expect( + runBrowserProxyCommand( + JSON.stringify({ + method: "GET", + path: "/snapshot", + profile: "chrome", + timeoutMs: 5, + }), + ), + ).rejects.toThrow( + /browser proxy timed out for GET \/snapshot after 5ms; ws-backed browser action; profile=chrome; status\(running=true, cdpHttp=true, cdpReady=false, cdpUrl=http:\/\/127\.0\.0\.1:18792\)/, + ); + }); + + it("keeps non-timeout browser errors intact", async () => { + dispatcherMocks.dispatch.mockResolvedValue({ + status: 500, + body: { error: "tab not found" }, + }); + + await expect( + runBrowserProxyCommand( + JSON.stringify({ + method: "POST", + path: "/act", + profile: "chrome", + timeoutMs: 50, + }), + ), + ).rejects.toThrow("tab not found"); + }); +}); diff --git a/src/node-host/invoke-browser.ts b/src/node-host/invoke-browser.ts index 115fcef6717..8587dff72c3 100644 --- a/src/node-host/invoke-browser.ts +++ b/src/node-host/invoke-browser.ts @@ -30,6 +30,8 @@ type BrowserProxyResult = { }; const BROWSER_PROXY_MAX_FILE_BYTES = 10 * 1024 * 1024; +const DEFAULT_BROWSER_PROXY_TIMEOUT_MS = 20_000; +const BROWSER_PROXY_STATUS_TIMEOUT_MS = 750; function normalizeProfileAllowlist(raw?: string[]): string[] { return Array.isArray(raw) ? raw.map((entry) => entry.trim()).filter(Boolean) : []; @@ -119,6 +121,87 @@ function decodeParams(raw?: string | null): T { return JSON.parse(raw) as T; } +function resolveBrowserProxyTimeout(timeoutMs?: number): number { + return typeof timeoutMs === "number" && Number.isFinite(timeoutMs) + ? Math.max(1, Math.floor(timeoutMs)) + : DEFAULT_BROWSER_PROXY_TIMEOUT_MS; +} + +function isBrowserProxyTimeoutError(err: unknown): boolean { + return String(err).includes("browser proxy request timed out"); +} + +function isWsBackedBrowserProxyPath(path: string): boolean { + return ( + path === "/act" || + path === "/navigate" || + path === "/pdf" || + path === "/screenshot" || + path === "/snapshot" + ); +} + +async function readBrowserProxyStatus(params: { + dispatcher: ReturnType; + profile?: string; +}): Promise | null> { + const query = params.profile ? { profile: params.profile } : {}; + try { + const response = await withTimeout( + (signal) => + params.dispatcher.dispatch({ + method: "GET", + path: "/", + query, + signal, + }), + BROWSER_PROXY_STATUS_TIMEOUT_MS, + "browser proxy status", + ); + if (response.status >= 400 || !response.body || typeof response.body !== "object") { + return null; + } + const body = response.body as Record; + return { + running: body.running, + cdpHttp: body.cdpHttp, + cdpReady: body.cdpReady, + cdpUrl: body.cdpUrl, + }; + } catch { + return null; + } +} + +function formatBrowserProxyTimeoutMessage(params: { + method: string; + path: string; + profile?: string; + timeoutMs: number; + wsBacked: boolean; + status: Record | null; +}): string { + const parts = [ + `browser proxy timed out for ${params.method} ${params.path} after ${params.timeoutMs}ms`, + params.wsBacked ? "ws-backed browser action" : "browser action", + ]; + if (params.profile) { + parts.push(`profile=${params.profile}`); + } + if (params.status) { + const statusParts = [ + `running=${String(params.status.running)}`, + `cdpHttp=${String(params.status.cdpHttp)}`, + `cdpReady=${String(params.status.cdpReady)}`, + ]; + if (typeof params.status.cdpUrl === "string" && params.status.cdpUrl.trim()) { + statusParts.push(`cdpUrl=${params.status.cdpUrl}`); + } + parts.push(`status(${statusParts.join(", ")})`); + } + return parts.join("; "); +} + export async function runBrowserProxyCommand(paramsJSON?: string | null): Promise { const params = decodeParams(paramsJSON); const pathValue = typeof params.path === "string" ? params.path.trim() : ""; @@ -151,6 +234,7 @@ export async function runBrowserProxyCommand(paramsJSON?: string | null): Promis const method = typeof params.method === "string" ? params.method.toUpperCase() : "GET"; const path = pathValue.startsWith("/") ? pathValue : `/${pathValue}`; const body = params.body; + const timeoutMs = resolveBrowserProxyTimeout(params.timeoutMs); const query: Record = {}; if (requestedProfile) { query.profile = requestedProfile; @@ -164,18 +248,41 @@ export async function runBrowserProxyCommand(paramsJSON?: string | null): Promis } const dispatcher = createBrowserRouteDispatcher(createBrowserControlContext()); - const response = await withTimeout( - (signal) => - dispatcher.dispatch({ - method: method === "DELETE" ? "DELETE" : method === "POST" ? "POST" : "GET", + let response; + try { + response = await withTimeout( + (signal) => + dispatcher.dispatch({ + method: method === "DELETE" ? "DELETE" : method === "POST" ? "POST" : "GET", + path, + query, + body, + signal, + }), + timeoutMs, + "browser proxy request", + ); + } catch (err) { + if (!isBrowserProxyTimeoutError(err)) { + throw err; + } + const profileForStatus = requestedProfile || resolved.defaultProfile; + const status = await readBrowserProxyStatus({ + dispatcher, + profile: path === "/profiles" ? undefined : profileForStatus, + }); + throw new Error( + formatBrowserProxyTimeoutMessage({ + method, path, - query, - body, - signal, + profile: path === "/profiles" ? undefined : profileForStatus || undefined, + timeoutMs, + wsBacked: isWsBackedBrowserProxyPath(path), + status, }), - params.timeoutMs, - "browser proxy request", - ); + { cause: err }, + ); + } if (response.status >= 400) { const message = response.body && typeof response.body === "object" && "error" in response.body