diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e557cd2bf..3428dd41a10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Browser/tool: keep explicit AI snapshots from inheriting the efficient role-snapshot default and preserve numeric Playwright AI refs, so `--format ai` remains a real AI snapshot path. Fixes #62550. Thanks @ly85206559. - Slack/messages: serialize write-client requests and whole outbound sends per target so rapid multi-message Slack replies preserve send order. Fixes #69101. (#69105) Thanks @nightq and @ztexydt-cqh. - Slack/messages: keep Slack bot tokens out of internal message-ordering and DM cache keys. - Slack/exec approvals: resolve native approval button clicks over the Gateway instead of delivering `/approve ...` as plain agent text, preserving retry buttons if Gateway resolution fails. Fixes #71023. (#71025) Thanks @marusan03. diff --git a/extensions/browser/src/browser-tool.actions.ts b/extensions/browser/src/browser-tool.actions.ts index 8338c0df22c..c6c941a98eb 100644 --- a/extensions/browser/src/browser-tool.actions.ts +++ b/extensions/browser/src/browser-tool.actions.ts @@ -234,13 +234,12 @@ export async function executeSnapshotAction(params: { const { input, baseUrl, profile, proxyRequest } = params; const snapshotDefaults = browserToolActionDeps.loadConfig().browser?.snapshotDefaults; const format: "ai" | "aria" | undefined = - input.snapshotFormat === "ai" || input.snapshotFormat === "aria" - ? input.snapshotFormat - : undefined; + input.snapshotFormat === "ai" ? "ai" : input.snapshotFormat === "aria" ? "aria" : undefined; + const formatExplicit = format !== undefined; const mode: "efficient" | undefined = input.mode === "efficient" ? "efficient" - : format !== "aria" && snapshotDefaults?.mode === "efficient" + : !formatExplicit && format !== "aria" && snapshotDefaults?.mode === "efficient" ? "efficient" : undefined; const labels = typeof input.labels === "boolean" ? input.labels : undefined; diff --git a/extensions/browser/src/browser-tool.test.ts b/extensions/browser/src/browser-tool.test.ts index b5fbfb24f82..ea351e42053 100644 --- a/extensions/browser/src/browser-tool.test.ts +++ b/extensions/browser/src/browser-tool.test.ts @@ -404,7 +404,8 @@ describe("browser tool snapshot maxChars", () => { configMocks.loadConfig.mockReturnValue({ browser: { snapshotDefaults: { mode: "efficient" } }, }); - await runSnapshotToolCall({ snapshotFormat: "ai" }); + const tool = createBrowserTool(); + await tool.execute?.("call-1", { action: "snapshot", target: "host" }); expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith( undefined, @@ -414,6 +415,19 @@ describe("browser tool snapshot maxChars", () => { ); }); + it("does not apply config snapshot defaults to explicit ai snapshots", async () => { + configMocks.loadConfig.mockReturnValue({ + browser: { snapshotDefaults: { mode: "efficient" } }, + }); + await runSnapshotToolCall({ snapshotFormat: "ai" }); + + expect(browserClientMocks.browserSnapshot).toHaveBeenCalled(); + const opts = browserClientMocks.browserSnapshot.mock.calls.at(-1)?.[1] as + | { mode?: string } + | undefined; + expect(opts?.mode).toBeUndefined(); + }); + it("does not apply config snapshot defaults to aria snapshots", async () => { configMocks.loadConfig.mockReturnValue({ browser: { snapshotDefaults: { mode: "efficient" } }, diff --git a/extensions/browser/src/browser/pw-ai.e2e.test.ts b/extensions/browser/src/browser/pw-ai.e2e.test.ts index c1450f9bffe..1de0725ee18 100644 --- a/extensions/browser/src/browser/pw-ai.e2e.test.ts +++ b/extensions/browser/src/browser/pw-ai.e2e.test.ts @@ -136,6 +136,34 @@ describe("pw-ai", () => { expect(res.snapshot).toContain("TRUNCATED"); }); + it("returns numeric ai snapshot refs in the public snapshot output", async () => { + const snapshot = ['- button "OK" [ref=1]', '- link "Docs" [ref=2]'].join("\n"); + const p1 = createPage({ targetId: "T1", snapshotFull: snapshot }); + const browser = createBrowser([p1.page]); + (chromiumMock.connectOverCDP as unknown as ReturnType).mockResolvedValue(browser); + + const res = await snapshotAiViaPlaywright({ + cdpUrl: "http://127.0.0.1:18792", + targetId: "T1", + }); + + expect(res.snapshot).toContain("[ref=1]"); + expect(res.snapshot).toContain("[ref=2]"); + expect(res.refs).toMatchObject({ + 1: { role: "button", name: "OK" }, + 2: { role: "link", name: "Docs" }, + }); + + await clickViaPlaywright({ + cdpUrl: "http://127.0.0.1:18792", + targetId: "T1", + ref: "1", + }); + + expect(p1.locator).toHaveBeenCalledWith("aria-ref=1"); + expect(p1.click).toHaveBeenCalledTimes(1); + }); + it("clicks a ref using aria-ref locator", async () => { const p1 = createPage({ targetId: "T1" }); const browser = createBrowser([p1.page]); diff --git a/extensions/browser/src/browser/pw-role-snapshot.test.ts b/extensions/browser/src/browser/pw-role-snapshot.test.ts index 7fdce9a179a..c5063743a3d 100644 --- a/extensions/browser/src/browser/pw-role-snapshot.test.ts +++ b/extensions/browser/src/browser/pw-role-snapshot.test.ts @@ -64,7 +64,7 @@ describe("pw-role-snapshot", () => { expect(parseRoleRef("e12")).toBe("e12"); expect(parseRoleRef("@e12")).toBe("e12"); expect(parseRoleRef("ref=e12")).toBe("e12"); - expect(parseRoleRef("12")).toBeNull(); + expect(parseRoleRef("12")).toBe("12"); expect(parseRoleRef("")).toBeNull(); }); @@ -87,4 +87,18 @@ describe("pw-role-snapshot", () => { expect(res.refs.e5).toMatchObject({ role: "link", name: "Home" }); expect(res.refs.e7).toMatchObject({ role: "button", name: "Save" }); }); + + it("preserves numeric Playwright AI snapshot refs", () => { + const ai = [ + "- navigation [ref=1]:", + ' - link "Home" [ref=5]', + ' - button "Save" [ref=7] [cursor=pointer]:', + ].join("\n"); + + const res = buildRoleSnapshotFromAiSnapshot(ai, { interactive: true }); + expect(res.snapshot).toContain("[ref=5]"); + expect(Object.keys(res.refs).toSorted()).toEqual(["5", "7"]); + expect(res.refs["5"]).toMatchObject({ role: "link", name: "Home" }); + expect(res.refs["7"]).toMatchObject({ role: "button", name: "Save" }); + }); }); diff --git a/extensions/browser/src/browser/pw-role-snapshot.ts b/extensions/browser/src/browser/pw-role-snapshot.ts index d3bba2c31f7..0b9ad502320 100644 --- a/extensions/browser/src/browser/pw-role-snapshot.ts +++ b/extensions/browser/src/browser/pw-role-snapshot.ts @@ -265,7 +265,13 @@ export function parseRoleRef(raw: string): string | null { : trimmed.startsWith("ref=") ? trimmed.slice(4) : trimmed; - return /^e\d+$/.test(normalized) ? normalized : null; + if (/^e\d+$/i.test(normalized)) { + return normalized; + } + if (/^\d{1,9}$/.test(normalized)) { + return normalized; + } + return null; } export function buildRoleSnapshotFromAriaSnapshot( @@ -328,8 +334,12 @@ export function buildRoleSnapshotFromAriaSnapshot( } function parseAiSnapshotRef(suffix: string): string | null { - const match = suffix.match(/\[ref=(e\d+)\]/i); - return match ? match[1] : null; + const eMatch = suffix.match(/\[ref=(e\d+)\]/i); + if (eMatch) { + return eMatch[1]; + } + const numMatch = suffix.match(/\[ref=(\d{1,9})\]/); + return numMatch ? numMatch[1] : null; } /** diff --git a/extensions/browser/src/cli/browser-cli-inspect.test.ts b/extensions/browser/src/cli/browser-cli-inspect.test.ts index 46cc6c12971..fb41e939629 100644 --- a/extensions/browser/src/cli/browser-cli-inspect.test.ts +++ b/extensions/browser/src/cli/browser-cli-inspect.test.ts @@ -101,12 +101,17 @@ describe("browser cli snapshot defaults", () => { args: ["--format", "aria"], expectMode: undefined, }, + { + label: "does not apply config snapshot defaults to explicit ai snapshots", + args: ["--format", "ai"], + expectMode: undefined, + }, ])("$label", async ({ args, expectMode }) => { configMocks.loadConfig.mockReturnValue({ browser: { snapshotDefaults: { mode: "efficient" } }, }); - if (args.includes("--format")) { + if (args.includes("--format") && args.includes("aria")) { gatewayMocks.callGatewayFromCli.mockResolvedValueOnce({ ok: true, format: "aria", diff --git a/extensions/browser/src/cli/browser-cli-inspect.ts b/extensions/browser/src/cli/browser-cli-inspect.ts index 65a300fe7b5..c2b2221b245 100644 --- a/extensions/browser/src/cli/browser-cli-inspect.ts +++ b/extensions/browser/src/cli/browser-cli-inspect.ts @@ -75,8 +75,13 @@ export function registerBrowserInspectCommands( const parent = parentOpts(cmd); const profile = parent?.browserProfile; const format = opts.format === "aria" ? "aria" : "ai"; + const formatWasExplicit = + typeof cmd.getOptionValueSource === "function" && + cmd.getOptionValueSource("format") === "cli"; const configMode = - format === "ai" && loadConfig().browser?.snapshotDefaults?.mode === "efficient" + !formatWasExplicit && + format === "ai" && + loadConfig().browser?.snapshotDefaults?.mode === "efficient" ? "efficient" : undefined; const mode = opts.efficient === true || opts.mode === "efficient" ? "efficient" : configMode;