From e10f20032a01e34dc169e720aeb6408828831124 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Apr 2026 09:42:11 +0100 Subject: [PATCH] fix(browser): resolve aria snapshot refs via DOM markers Co-authored-by: MrKipler --- CHANGELOG.md | 1 + extensions/browser/src/browser/pw-ai.ts | 1 + .../src/browser/pw-session.page-cdp.test.ts | 109 +++++++++++++++++- .../src/browser/pw-session.page-cdp.ts | 75 ++++++++++++ .../browser/src/browser/pw-session.test.ts | 26 ++++- extensions/browser/src/browser/pw-session.ts | 31 ++++- .../pw-tools-core.browser-ssrf-guard.test.ts | 1 + .../browser/pw-tools-core.snapshot.test.ts | 101 ++++++++++++++++ .../src/browser/pw-tools-core.snapshot.ts | 83 ++++++++++++- .../src/browser/routes/agent.snapshot.ts | 15 ++- .../server.agent-contract-core.test.ts | 5 + .../server.control-server.test-harness.ts | 1 + 12 files changed, 434 insertions(+), 15 deletions(-) create mode 100644 extensions/browser/src/browser/pw-tools-core.snapshot.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b5f55cef65f..99dd51120c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Docs: https://docs.openclaw.ai ### Fixes - Browser/Playwright: ignore benign already-handled route races during guarded navigation so browser-page tasks no longer fail when Playwright tears down a route mid-flight. (#68708) Thanks @Steady-ai. +- Browser/aria snapshots: bind `format=aria` `axN` refs to live DOM nodes through backend DOM ids when Playwright is available, so follow-up browser actions can use those refs without timing out. (#62434) Thanks @MrKipler. - Telegram: prevent duplicate in-process long pollers for the same bot token and add clearer `getUpdates` conflict diagnostics for external duplicate pollers. Fixes #56230. - Browser/Linux: detect Chromium-based installs under `/opt/google`, `/opt/brave.com`, `/usr/lib/chromium`, and `/usr/lib/chromium-browser` before asking users to set `browser.executablePath`. (#48563) Thanks @lupuletic. - Sessions/browser: close tracked browser tabs when idle, daily, `/new`, or `/reset` session rollover archives the previous transcript, preventing tabs from leaking past the old session. Thanks @jakozloski. diff --git a/extensions/browser/src/browser/pw-ai.ts b/extensions/browser/src/browser/pw-ai.ts index cd968270db3..0f9f3ca844c 100644 --- a/extensions/browser/src/browser/pw-ai.ts +++ b/extensions/browser/src/browser/pw-ai.ts @@ -53,6 +53,7 @@ export { snapshotAiViaPlaywright, snapshotAriaViaPlaywright, snapshotRoleViaPlaywright, + storeAriaSnapshotRefsViaPlaywright, screenshotWithLabelsViaPlaywright, storageClearViaPlaywright, storageGetViaPlaywright, diff --git a/extensions/browser/src/browser/pw-session.page-cdp.test.ts b/extensions/browser/src/browser/pw-session.page-cdp.test.ts index c00f8af5a02..217de9865f4 100644 --- a/extensions/browser/src/browser/pw-session.page-cdp.test.ts +++ b/extensions/browser/src/browser/pw-session.page-cdp.test.ts @@ -1,5 +1,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { withPageScopedCdpClient } from "./pw-session.page-cdp.js"; +import { + BROWSER_REF_MARKER_ATTRIBUTE, + markBackendDomRefsOnPage, + withPageScopedCdpClient, +} from "./pw-session.page-cdp.js"; describe("pw-session page-scoped CDP client", () => { beforeEach(() => { @@ -32,4 +36,107 @@ describe("pw-session page-scoped CDP client", () => { expect(sessionSend).toHaveBeenCalledWith("Emulation.setLocaleOverride", { locale: "en-US" }); expect(sessionDetach).toHaveBeenCalledTimes(1); }); + + it("marks backend DOM refs on the page", async () => { + const sessionSend = vi.fn(async (method: string, params?: Record) => { + if (method === "DOM.pushNodesByBackendIdsToFrontend") { + expect(params).toEqual({ backendNodeIds: [42, 84] }); + return { nodeIds: [101, 202] }; + } + return {}; + }); + const sessionDetach = vi.fn(async () => {}); + const newCDPSession = vi.fn(async () => ({ + send: sessionSend, + detach: sessionDetach, + })); + const evaluateAll = vi.fn(async () => {}); + const page = { + context: () => ({ + newCDPSession, + }), + locator: vi.fn(() => ({ evaluateAll })), + }; + + const marked = await markBackendDomRefsOnPage({ + page: page as never, + refs: [ + { ref: "ax1", backendDOMNodeId: 42 }, + { ref: "ax2", backendDOMNodeId: 84 }, + ], + }); + + expect(page.locator).toHaveBeenCalledWith(`[${BROWSER_REF_MARKER_ATTRIBUTE}]`); + expect(evaluateAll).toHaveBeenCalledTimes(1); + expect(sessionSend).toHaveBeenNthCalledWith(1, "DOM.enable", undefined); + expect(sessionSend).toHaveBeenNthCalledWith(2, "DOM.pushNodesByBackendIdsToFrontend", { + backendNodeIds: [42, 84], + }); + expect(sessionSend).toHaveBeenNthCalledWith(3, "DOM.setAttributeValue", { + nodeId: 101, + name: BROWSER_REF_MARKER_ATTRIBUTE, + value: "ax1", + }); + expect(sessionSend).toHaveBeenNthCalledWith(4, "DOM.setAttributeValue", { + nodeId: 202, + name: BROWSER_REF_MARKER_ATTRIBUTE, + value: "ax2", + }); + expect(marked).toEqual(new Set(["ax1", "ax2"])); + expect(sessionDetach).toHaveBeenCalledTimes(1); + }); + + it("clears stale markers even when no backend refs are valid", async () => { + const newCDPSession = vi.fn(); + const evaluateAll = vi.fn(async () => {}); + const page = { + context: () => ({ + newCDPSession, + }), + locator: vi.fn(() => ({ evaluateAll })), + }; + + const marked = await markBackendDomRefsOnPage({ + page: page as never, + refs: [{ ref: "e1", backendDOMNodeId: 0 }], + }); + + expect(page.locator).toHaveBeenCalledWith(`[${BROWSER_REF_MARKER_ATTRIBUTE}]`); + expect(evaluateAll).toHaveBeenCalledTimes(1); + expect(newCDPSession).not.toHaveBeenCalled(); + expect(marked).toEqual(new Set()); + }); + + it("keeps unmarked refs out of the marked set when marker writes fail", async () => { + const sessionSend = vi.fn(async (method: string) => { + if (method === "DOM.pushNodesByBackendIdsToFrontend") { + return { nodeIds: [101, 202] }; + } + if (method === "DOM.setAttributeValue") { + throw new Error("detached"); + } + return {}; + }); + const sessionDetach = vi.fn(async () => {}); + const page = { + context: () => ({ + newCDPSession: vi.fn(async () => ({ + send: sessionSend, + detach: sessionDetach, + })), + }), + locator: vi.fn(() => ({ evaluateAll: vi.fn(async () => {}) })), + }; + + const marked = await markBackendDomRefsOnPage({ + page: page as never, + refs: [ + { ref: "ax1", backendDOMNodeId: 42 }, + { ref: "ax2", backendDOMNodeId: 84 }, + ], + }); + + expect(marked).toEqual(new Set()); + expect(sessionDetach).toHaveBeenCalledTimes(1); + }); }); diff --git a/extensions/browser/src/browser/pw-session.page-cdp.ts b/extensions/browser/src/browser/pw-session.page-cdp.ts index ccfc2ee7f34..3870ea02945 100644 --- a/extensions/browser/src/browser/pw-session.page-cdp.ts +++ b/extensions/browser/src/browser/pw-session.page-cdp.ts @@ -1,6 +1,9 @@ import type { CDPSession, Page } from "playwright-core"; type PageCdpSend = (method: string, params?: Record) => Promise; +type MarkBackendDomRef = { ref: string; backendDOMNodeId: number }; + +export const BROWSER_REF_MARKER_ATTRIBUTE = "data-openclaw-browser-ref"; async function withPlaywrightPageCdpSession( page: Page, @@ -31,3 +34,75 @@ export async function withPageScopedCdpClient(opts: { ); }); } + +export async function markBackendDomRefsOnPage(opts: { + page: Page; + refs: MarkBackendDomRef[]; +}): Promise> { + await opts.page + .locator(`[${BROWSER_REF_MARKER_ATTRIBUTE}]`) + .evaluateAll((elements, attr) => { + for (const element of elements) { + if (element instanceof Element) { + element.removeAttribute(attr); + } + } + }, BROWSER_REF_MARKER_ATTRIBUTE) + .catch(() => {}); + + const refs = opts.refs.filter( + (entry) => + /^ax\d+$/.test(entry.ref) && + Number.isFinite(entry.backendDOMNodeId) && + Math.floor(entry.backendDOMNodeId) > 0, + ); + const marked = new Set(); + if (!refs.length) { + return marked; + } + + return await withPlaywrightPageCdpSession(opts.page, async (session) => { + const send = async (method: string, params?: Record) => + await ( + session.send as unknown as ( + method: string, + params?: Record, + ) => Promise + )(method, params); + + await send("DOM.enable").catch(() => {}); + + const backendNodeIds = [...new Set(refs.map((entry) => Math.floor(entry.backendDOMNodeId)))]; + const pushed = (await send("DOM.pushNodesByBackendIdsToFrontend", { + backendNodeIds, + }).catch(() => ({}))) as { nodeIds?: number[] }; + const nodeIds = Array.isArray(pushed.nodeIds) ? pushed.nodeIds : []; + const nodeIdByBackendId = new Map(); + for (let index = 0; index < backendNodeIds.length; index += 1) { + const backendNodeId = backendNodeIds[index]; + const nodeId = nodeIds[index]; + if (backendNodeId && typeof nodeId === "number" && nodeId > 0) { + nodeIdByBackendId.set(backendNodeId, nodeId); + } + } + + for (const entry of refs) { + const nodeId = nodeIdByBackendId.get(Math.floor(entry.backendDOMNodeId)); + if (!nodeId) { + continue; + } + try { + await send("DOM.setAttributeValue", { + nodeId, + name: BROWSER_REF_MARKER_ATTRIBUTE, + value: entry.ref, + }); + marked.add(entry.ref); + } catch { + // Best-effort marker write. Unmarked refs fall back to role metadata. + } + } + + return marked; + }); +} diff --git a/extensions/browser/src/browser/pw-session.test.ts b/extensions/browser/src/browser/pw-session.test.ts index ef6f2749c7c..14a9c1858c4 100644 --- a/extensions/browser/src/browser/pw-session.test.ts +++ b/extensions/browser/src/browser/pw-session.test.ts @@ -6,6 +6,7 @@ import { rememberRoleRefsForTarget, restoreRoleRefsForTarget, } from "./pw-session.js"; +import { BROWSER_REF_MARKER_ATTRIBUTE } from "./pw-session.page-cdp.js"; function fakePage(): { page: Page; @@ -27,6 +28,7 @@ function fakePage(): { const getByRole = vi.fn(() => ({ nth: vi.fn(() => ({ ok: true })) })); const frameLocator = vi.fn(() => ({ getByRole: vi.fn(() => ({ nth: vi.fn(() => ({ ok: true })) })), + locator: vi.fn(() => ({ nth: vi.fn(() => ({ ok: true })) })), })); const locator = vi.fn(() => ({ nth: vi.fn(() => ({ ok: true })) })); @@ -72,10 +74,30 @@ describe("pw-session refLocator", () => { expect(mocks.locator).toHaveBeenCalledWith("aria-ref=e1"); }); - it("rejects axN refs from format=aria snapshots instead of timing out", () => { + it("uses backend-marked DOM locators for ax refs", () => { + const { page, mocks } = fakePage(); + const state = ensurePageState(page); + state.roleRefs = { ax12: { role: "button", name: "OK", domMarker: true } }; + + refLocator(page, "ax12"); + + expect(mocks.locator).toHaveBeenCalledWith(`[${BROWSER_REF_MARKER_ATTRIBUTE}="ax12"]`); + }); + + it("falls back to role heuristics for ax refs without backend markers", () => { + const { page, mocks } = fakePage(); + const state = ensurePageState(page); + state.roleRefs = { ax12: { role: "button", name: "OK" } }; + + refLocator(page, "ax12"); + + expect(mocks.getByRole).toHaveBeenCalledWith("button", { name: "OK", exact: true }); + }); + + it("rejects unknown ax refs instead of timing out on aria-ref locators", () => { const { page, mocks } = fakePage(); - expect(() => refLocator(page, "ax12")).toThrow(/format=aria snapshot/); + expect(() => refLocator(page, "ax12")).toThrow(/Unknown ref/); expect(mocks.locator).not.toHaveBeenCalled(); }); }); diff --git a/extensions/browser/src/browser/pw-session.ts b/extensions/browser/src/browser/pw-session.ts index a9fcd8fafc3..b80a8a1a02a 100644 --- a/extensions/browser/src/browser/pw-session.ts +++ b/extensions/browser/src/browser/pw-session.ts @@ -31,7 +31,7 @@ import { InvalidBrowserNavigationUrlError, withBrowserNavigationPolicy, } from "./navigation-guard.js"; -import { withPageScopedCdpClient } from "./pw-session.page-cdp.js"; +import { BROWSER_REF_MARKER_ATTRIBUTE, withPageScopedCdpClient } from "./pw-session.page-cdp.js"; export type BrowserConsoleMessage = { type: string; @@ -84,7 +84,7 @@ type PageState = { * Mode "role" refs are generated from ariaSnapshot and resolved via getByRole. * Mode "aria" refs are Playwright aria-ref ids and resolved via `aria-ref=...`. */ - roleRefs?: Record; + roleRefs?: Record; roleRefsMode?: "role" | "aria"; roleRefsFrameSelector?: string; }; @@ -935,10 +935,29 @@ export function refLocator(page: Page, ref: string) { } if (AX_REF_PATTERN.test(normalized)) { - throw new Error( - `Ref "${normalized}" comes from a format=aria snapshot and cannot be used with act. ` + - `Re-snapshot with format=ai and use the eN refs from that snapshot.`, - ); + const state = pageStates.get(page); + const info = state?.roleRefs?.[normalized]; + if (!info) { + throw new Error( + `Unknown ref "${normalized}". Run a new snapshot and use a ref from that snapshot.`, + ); + } + const scope = state.roleRefsFrameSelector + ? page.frameLocator(state.roleRefsFrameSelector) + : page; + if (info.domMarker) { + return scope.locator(`[${BROWSER_REF_MARKER_ATTRIBUTE}="${normalized}"]`); + } + const locAny = scope as unknown as { + getByRole: ( + role: never, + opts?: { name?: string; exact?: boolean }, + ) => ReturnType; + }; + const locator = info.name + ? locAny.getByRole(info.role as never, { name: info.name, exact: true }) + : locAny.getByRole(info.role as never); + return info.nth !== undefined ? locator.nth(info.nth) : locator; } return page.locator(`aria-ref=${normalized}`); diff --git a/extensions/browser/src/browser/pw-tools-core.browser-ssrf-guard.test.ts b/extensions/browser/src/browser/pw-tools-core.browser-ssrf-guard.test.ts index d1a146bf8a4..c9086f440c7 100644 --- a/extensions/browser/src/browser/pw-tools-core.browser-ssrf-guard.test.ts +++ b/extensions/browser/src/browser/pw-tools-core.browser-ssrf-guard.test.ts @@ -27,6 +27,7 @@ const sessionMocks = vi.hoisted(() => ({ })); const pageCdpMocks = vi.hoisted(() => ({ + markBackendDomRefsOnPage: vi.fn(async () => new Set()), withPageScopedCdpClient: vi.fn( async ({ fn }: { fn: (send: () => Promise) => unknown }) => await fn(async () => ({ nodes: [] })), diff --git a/extensions/browser/src/browser/pw-tools-core.snapshot.test.ts b/extensions/browser/src/browser/pw-tools-core.snapshot.test.ts new file mode 100644 index 00000000000..97aebadf2e7 --- /dev/null +++ b/extensions/browser/src/browser/pw-tools-core.snapshot.test.ts @@ -0,0 +1,101 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const getPageForTargetId = vi.fn(); +const ensurePageState = vi.fn(); +const storeRoleRefsForTarget = vi.fn(); +const withPageScopedCdpClient = vi.fn(); +const markBackendDomRefsOnPage = vi.fn(); +const formatAriaSnapshot = vi.fn(); + +vi.mock("./pw-session.js", () => ({ + assertPageNavigationCompletedSafely: vi.fn(), + ensurePageState, + forceDisconnectPlaywrightForTarget: vi.fn(), + getPageForTargetId, + gotoPageWithNavigationGuard: vi.fn(), + storeRoleRefsForTarget, +})); + +vi.mock("./pw-session.page-cdp.js", () => ({ + markBackendDomRefsOnPage, + withPageScopedCdpClient, +})); + +vi.mock("./cdp.js", () => ({ + formatAriaSnapshot, +})); + +describe("pw-tools-core aria snapshot storage", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("reuses the resolved page when storing aria refs", async () => { + const page = { id: "page-1" }; + const rawNodes = [{ backendDOMNodeId: 42 }]; + const formattedNodes = [{ ref: "ax1", role: "button", name: "OK", backendDOMNodeId: 42 }]; + + getPageForTargetId.mockResolvedValue(page); + withPageScopedCdpClient.mockResolvedValue({ nodes: rawNodes }); + formatAriaSnapshot.mockReturnValue(formattedNodes); + markBackendDomRefsOnPage.mockResolvedValue(new Set(["ax1"])); + + const mod = await import("./pw-tools-core.snapshot.js"); + const result = await mod.snapshotAriaViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + limit: 5, + }); + + expect(result).toEqual({ nodes: formattedNodes }); + expect(getPageForTargetId).toHaveBeenCalledTimes(1); + expect(ensurePageState).toHaveBeenCalledWith(page); + expect(withPageScopedCdpClient).toHaveBeenCalledWith({ + cdpUrl: "http://127.0.0.1:9222", + page, + targetId: "tab-1", + fn: expect.any(Function), + }); + expect(markBackendDomRefsOnPage).toHaveBeenCalledWith({ + page, + refs: [{ ref: "ax1", backendDOMNodeId: 42 }], + }); + expect(storeRoleRefsForTarget).toHaveBeenCalledWith({ + page, + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + refs: { + ax1: { role: "button", name: "OK", domMarker: true }, + }, + mode: "role", + }); + }); + + it("stores role fallback metadata when backend markers are unavailable", async () => { + const page = { id: "page-1" }; + const mod = await import("./pw-tools-core.snapshot.js"); + + getPageForTargetId.mockResolvedValue(page); + markBackendDomRefsOnPage.mockResolvedValue(new Set()); + + await mod.storeAriaSnapshotRefsViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + nodes: [ + { ref: "ax1", role: "Button", name: "OK", backendDOMNodeId: 42, depth: 0 }, + { ref: "ax2", role: "Button", name: "OK", backendDOMNodeId: 84, depth: 0 }, + ], + }); + + expect(storeRoleRefsForTarget).toHaveBeenCalledWith({ + page, + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + refs: { + ax1: { role: "button", name: "OK" }, + ax2: { role: "button", name: "OK", nth: 1 }, + }, + mode: "role", + }); + }); +}); diff --git a/extensions/browser/src/browser/pw-tools-core.snapshot.ts b/extensions/browser/src/browser/pw-tools-core.snapshot.ts index 447b7c6a830..084fa9ec059 100644 --- a/extensions/browser/src/browser/pw-tools-core.snapshot.ts +++ b/extensions/browser/src/browser/pw-tools-core.snapshot.ts @@ -1,4 +1,7 @@ -import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; +import { + normalizeLowercaseStringOrEmpty, + normalizeOptionalString, +} from "openclaw/plugin-sdk/text-runtime"; import type { Page } from "playwright-core"; import type { SsrFPolicy } from "../infra/net/ssrf.js"; import { type AriaSnapshotNode, formatAriaSnapshot, type RawAXNode } from "./cdp.js"; @@ -22,7 +25,7 @@ import { gotoPageWithNavigationGuard, storeRoleRefsForTarget, } from "./pw-session.js"; -import { withPageScopedCdpClient } from "./pw-session.page-cdp.js"; +import { markBackendDomRefsOnPage, withPageScopedCdpClient } from "./pw-session.page-cdp.js"; type SnapshotUrlEntry = { text: string; @@ -64,6 +67,73 @@ function appendSnapshotUrls(snapshot: string, urls: SnapshotUrlEntry[]): string return `${snapshot}\n\nLinks:\n${lines.join("\n")}`; } +function buildStoredAriaRefs( + nodes: AriaSnapshotNode[], + markedRefs: Set, +): Record { + const refs: Record = + {}; + const counts = new Map(); + const refsByKey = new Map(); + + for (const node of nodes) { + const role = normalizeLowercaseStringOrEmpty(node.role) || "unknown"; + const name = node.name.trim() || undefined; + const key = `${role}:${name ?? ""}`; + const nth = counts.get(key) ?? 0; + counts.set(key, nth + 1); + refsByKey.set(key, [...(refsByKey.get(key) ?? []), node.ref]); + refs[node.ref] = { + role, + ...(name ? { name } : {}), + ...(nth > 0 ? { nth } : {}), + ...(markedRefs.has(node.ref) ? { domMarker: true } : {}), + }; + } + + for (const refsForKey of refsByKey.values()) { + if (refsForKey.length > 1) { + continue; + } + const ref = refsForKey[0]; + if (ref) { + delete refs[ref]?.nth; + } + } + + return refs; +} + +export async function storeAriaSnapshotRefsViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + nodes: AriaSnapshotNode[]; + page?: Page; +}): Promise { + const page = + opts.page ?? + (await getPageForTargetId({ + cdpUrl: opts.cdpUrl, + targetId: opts.targetId, + })); + ensurePageState(page); + const markedRefs = await markBackendDomRefsOnPage({ + page, + refs: opts.nodes.flatMap((node) => + typeof node.backendDOMNodeId === "number" + ? [{ ref: node.ref, backendDOMNodeId: node.backendDOMNodeId }] + : [], + ), + }); + storeRoleRefsForTarget({ + page, + cdpUrl: opts.cdpUrl, + targetId: opts.targetId, + refs: buildStoredAriaRefs(opts.nodes, markedRefs), + mode: "role", + }); +} + export async function snapshotAriaViaPlaywright(opts: { cdpUrl: string; targetId?: string; @@ -99,7 +169,14 @@ export async function snapshotAriaViaPlaywright(opts: { nodes?: RawAXNode[]; }; const nodes = Array.isArray(res?.nodes) ? res.nodes : []; - return { nodes: formatAriaSnapshot(nodes, limit) }; + const formatted = formatAriaSnapshot(nodes, limit); + await storeAriaSnapshotRefsViaPlaywright({ + cdpUrl: opts.cdpUrl, + targetId: opts.targetId, + nodes: formatted, + page, + }); + return { nodes: formatted }; } export async function snapshotAiViaPlaywright(opts: { diff --git a/extensions/browser/src/browser/routes/agent.snapshot.ts b/extensions/browser/src/browser/routes/agent.snapshot.ts index 9999842c690..b00195d2c28 100644 --- a/extensions/browser/src/browser/routes/agent.snapshot.ts +++ b/extensions/browser/src/browser/routes/agent.snapshot.ts @@ -505,7 +505,8 @@ export function registerBrowserAgentSnapshotRoutes( return; } const targetId = typeof req.query.targetId === "string" ? req.query.targetId.trim() : ""; - const hasPlaywright = Boolean(await getPwAiModule()); + const pwModule = await getPwAiModule(); + const hasPlaywright = Boolean(pwModule); const plan = resolveSnapshotPlan({ profile: profileCtx.profile, query: req.query, @@ -691,10 +692,11 @@ export function registerBrowserAgentSnapshotRoutes( }); } - const snap = shouldUsePlaywrightForAriaSnapshot({ + const usePlaywrightAriaSnapshot = shouldUsePlaywrightForAriaSnapshot({ profile: profileCtx.profile, wsUrl: tab.wsUrl, - }) + }); + const snap = usePlaywrightAriaSnapshot ? (() => { // Extension relay doesn't expose per-page WS URLs; run AX snapshot via Playwright CDP session. // Also covers cases where wsUrl is missing/unusable. @@ -716,6 +718,13 @@ export function registerBrowserAgentSnapshotRoutes( if (!resolved) { return; } + if (!usePlaywrightAriaSnapshot) { + await pwModule?.storeAriaSnapshotRefsViaPlaywright?.({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + nodes: resolved.nodes, + }); + } return res.json({ ok: true, format: plan.format, diff --git a/extensions/browser/src/browser/server.agent-contract-core.test.ts b/extensions/browser/src/browser/server.agent-contract-core.test.ts index 96f800f2d9f..39e5c4a3534 100644 --- a/extensions/browser/src/browser/server.agent-contract-core.test.ts +++ b/extensions/browser/src/browser/server.agent-contract-core.test.ts @@ -219,6 +219,11 @@ describe("browser control server", () => { wsUrl: "ws://127.0.0.1/devtools/page/abcd1234", limit: 1, }); + expect(pwMocks.storeAriaSnapshotRefsViaPlaywright).toHaveBeenCalledWith({ + cdpUrl: state.cdpBaseUrl, + targetId: "abcd1234", + nodes: [{ ref: "1", role: "link", name: "x", depth: 0 }], + }); const snapAi = (await realFetch(`${base}/snapshot?format=ai`).then((r) => r.json())) as { ok: boolean; diff --git a/extensions/browser/src/browser/server.control-server.test-harness.ts b/extensions/browser/src/browser/server.control-server.test-harness.ts index 6dd393e0133..4a699ad52b7 100644 --- a/extensions/browser/src/browser/server.control-server.test-harness.ts +++ b/extensions/browser/src/browser/server.control-server.test-harness.ts @@ -175,6 +175,7 @@ const pwMocks = vi.hoisted(() => ({ selectOptionViaPlaywright: vi.fn(async (_opts?: unknown) => {}), setInputFilesViaPlaywright: vi.fn(async () => {}), snapshotAiViaPlaywright: vi.fn(async () => ({ snapshot: "ok" })), + storeAriaSnapshotRefsViaPlaywright: vi.fn(async () => {}), traceStopViaPlaywright: vi.fn(async () => {}), takeScreenshotViaPlaywright: vi.fn(async () => ({ buffer: Buffer.from("png"),