From fc0fd1bddaa3de02eafed01e7ef4e6b87f58c3f3 Mon Sep 17 00:00:00 2001 From: xingzhou Date: Sun, 12 Jul 2026 05:20:45 +0800 Subject: [PATCH] fix(ui): browser annotation prompts corrupt boundary emoji (#104396) --- .../browser/browser-annotation.test.ts | 62 ++++++++++++++++++- .../components/browser/browser-annotation.ts | 3 +- ui/src/components/browser/browser-client.ts | 6 +- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/ui/src/components/browser/browser-annotation.test.ts b/ui/src/components/browser/browser-annotation.test.ts index 47ecef085b4d..b42a884158c2 100644 --- a/ui/src/components/browser/browser-annotation.test.ts +++ b/ui/src/components/browser/browser-annotation.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; import { buildAnnotationPrompt, describeInspectedNode, @@ -7,7 +8,7 @@ import { BROWSER_ANNOTATION_EVENT, type BrowserAnnotationDraft, } from "./browser-annotation.ts"; -import type { BrowserInspectedNode } from "./browser-client.ts"; +import { inspectBrowserElementAt, type BrowserInspectedNode } from "./browser-client.ts"; function node(overrides: Partial = {}): BrowserInspectedNode { return { @@ -120,6 +121,63 @@ describe("buildAnnotationPrompt", () => { expect(prompt.split("\n").length).toBe(3); }); + it("keeps bounded page-reported fields on valid UTF-16 boundaries", () => { + const titleAndName = `${"a".repeat(79)}😀tail`; + const role = `${"r".repeat(39)}😀tail`; + const prompt = buildAnnotationPrompt({ + url: "https://example.com", + title: titleAndName, + strokes: [], + element: node({ name: titleAndName, role }), + }); + + expect(prompt).toContain(`page-reported title: "${"a".repeat(79)}"`); + expect(prompt).toContain(`button "${"a".repeat(79)}" (role=${"r".repeat(39)})`); + }); + + it("preserves valid UTF-16 from inspected accessible names through prompt construction", async () => { + const element = document.createElement("button"); + element.setAttribute("aria-label", `${"a".repeat(78)}${" ".repeat(41)}😀tail`); + const originalElementFromPoint = Object.getOwnPropertyDescriptor(document, "elementFromPoint"); + Object.defineProperty(document, "elementFromPoint", { + configurable: true, + value: vi.fn(() => element), + }); + const client = { + request: vi.fn(async (_method: string, envelope: { body?: { fn?: string } }) => { + const fn = envelope.body?.fn; + if (!fn) { + throw new Error("missing browser evaluation function"); + } + return { result: (0, eval)(`(${fn})`)() }; + }), + }; + + try { + const inspected = await inspectBrowserElementAt(client as unknown as GatewayBrowserClient, { + targetId: "proof-tab", + x: 10, + y: 20, + }); + expect(inspected).not.toBeNull(); + const prompt = buildAnnotationPrompt({ + url: "https://example.com", + title: "Boundary proof", + strokes: [], + element: inspected, + }); + + expect(inspected?.name.charCodeAt((inspected?.name.length ?? 0) - 1)).not.toBe(0xd83d); + expect(prompt).toContain(`button "${"a".repeat(78)}"`); + } finally { + if (originalElementFromPoint) { + Object.defineProperty(document, "elementFromPoint", originalElementFromPoint); + } else { + Reflect.deleteProperty(document, "elementFromPoint"); + } + } + }); + it("strips hostile characters from selector fragments", () => { const descriptor = describeInspectedNode( node({ diff --git a/ui/src/components/browser/browser-annotation.ts b/ui/src/components/browser/browser-annotation.ts index 5d3e46732155..333e53931a41 100644 --- a/ui/src/components/browser/browser-annotation.ts +++ b/ui/src/components/browser/browser-annotation.ts @@ -1,6 +1,7 @@ // Annotation model for the browser panel: freehand strokes drawn over a page // screenshot, plus the prepackaged prompt handed to the chat composer so the // agent knows what was marked up. +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { t } from "../../i18n/index.ts"; import type { BrowserInspectedNode } from "./browser-client.ts"; @@ -68,7 +69,7 @@ function percent(value: number): string { * prompt template additionally labels these values as page-reported. */ function sanitizePageText(value: string, maxLength = 80): string { - return value.replace(/\s+/g, " ").trim().slice(0, maxLength); + return truncateUtf16Safe(value.replace(/\s+/g, " ").trim(), maxLength); } /** Selector fragments (tag/id/class) are page-controlled too: keep only diff --git a/ui/src/components/browser/browser-client.ts b/ui/src/components/browser/browser-client.ts index ce51372c03b7..9d81aff204e7 100644 --- a/ui/src/components/browser/browser-client.ts +++ b/ui/src/components/browser/browser-client.ts @@ -266,12 +266,16 @@ export async function inspectBrowserElementAt( const rect = el.getBoundingClientRect(); const label = el.getAttribute("aria-label") || el.getAttribute("alt") || el.getAttribute("title") || ""; const text = (el.textContent || "").replace(/\\s+/g, " ").trim(); + const nameSource = label || text; + const nameLimit = 120; + // This serialized page function cannot call imported helpers; back up only when the cap splits a surrogate pair. + const nameEnd = (nameSource.codePointAt(nameLimit - 1) || 0) > 0xffff ? nameLimit - 1 : nameLimit; return { tag: el.tagName.toLowerCase(), id: el.id || "", classes: Array.from(el.classList).slice(0, 6), role: el.getAttribute("role") || "", - name: (label || text).slice(0, 120), + name: nameSource.slice(0, nameEnd), rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, focusable: typeof el.tabIndex === "number" && el.tabIndex >= 0, };