fix(ui): browser annotation prompts corrupt boundary emoji (#104396)

This commit is contained in:
xingzhou
2026-07-12 05:20:45 +08:00
committed by GitHub
parent c60b1b25bf
commit fc0fd1bdda
3 changed files with 67 additions and 4 deletions

View File

@@ -1,4 +1,5 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import type { GatewayBrowserClient } from "../../api/gateway.ts";
import {
buildAnnotationPrompt,
describeInspectedNode,
@@ -7,7 +8,7 @@ import {
BROWSER_ANNOTATION_EVENT,
type BrowserAnnotationDraft,
} from "./browser-annotation.ts";
import type { BrowserInspectedNode } from "./browser-client.ts";
import { inspectBrowserElementAt, type BrowserInspectedNode } from "./browser-client.ts";
function node(overrides: Partial<BrowserInspectedNode> = {}): BrowserInspectedNode {
return {
@@ -120,6 +121,63 @@ describe("buildAnnotationPrompt", () => {
expect(prompt.split("\n").length).toBe(3);
});
it("keeps bounded page-reported fields on valid UTF-16 boundaries", () => {
const titleAndName = `${"a".repeat(79)}😀tail`;
const role = `${"r".repeat(39)}😀tail`;
const prompt = buildAnnotationPrompt({
url: "https://example.com",
title: titleAndName,
strokes: [],
element: node({ name: titleAndName, role }),
});
expect(prompt).toContain(`page-reported title: "${"a".repeat(79)}"`);
expect(prompt).toContain(`button "${"a".repeat(79)}" (role=${"r".repeat(39)})`);
});
it("preserves valid UTF-16 from inspected accessible names through prompt construction", async () => {
const element = document.createElement("button");
element.setAttribute("aria-label", `${"a".repeat(78)}${" ".repeat(41)}😀tail`);
const originalElementFromPoint = Object.getOwnPropertyDescriptor(document, "elementFromPoint");
Object.defineProperty(document, "elementFromPoint", {
configurable: true,
value: vi.fn(() => element),
});
const client = {
request: vi.fn(async (_method: string, envelope: { body?: { fn?: string } }) => {
const fn = envelope.body?.fn;
if (!fn) {
throw new Error("missing browser evaluation function");
}
return { result: (0, eval)(`(${fn})`)() };
}),
};
try {
const inspected = await inspectBrowserElementAt(client as unknown as GatewayBrowserClient, {
targetId: "proof-tab",
x: 10,
y: 20,
});
expect(inspected).not.toBeNull();
const prompt = buildAnnotationPrompt({
url: "https://example.com",
title: "Boundary proof",
strokes: [],
element: inspected,
});
expect(inspected?.name.charCodeAt((inspected?.name.length ?? 0) - 1)).not.toBe(0xd83d);
expect(prompt).toContain(`button "${"a".repeat(78)}"`);
} finally {
if (originalElementFromPoint) {
Object.defineProperty(document, "elementFromPoint", originalElementFromPoint);
} else {
Reflect.deleteProperty(document, "elementFromPoint");
}
}
});
it("strips hostile characters from selector fragments", () => {
const descriptor = describeInspectedNode(
node({

View File

@@ -1,6 +1,7 @@
// Annotation model for the browser panel: freehand strokes drawn over a page
// screenshot, plus the prepackaged prompt handed to the chat composer so the
// agent knows what was marked up.
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { t } from "../../i18n/index.ts";
import type { BrowserInspectedNode } from "./browser-client.ts";
@@ -68,7 +69,7 @@ function percent(value: number): string {
* prompt template additionally labels these values as page-reported.
*/
function sanitizePageText(value: string, maxLength = 80): string {
return value.replace(/\s+/g, " ").trim().slice(0, maxLength);
return truncateUtf16Safe(value.replace(/\s+/g, " ").trim(), maxLength);
}
/** Selector fragments (tag/id/class) are page-controlled too: keep only

View File

@@ -266,12 +266,16 @@ export async function inspectBrowserElementAt(
const rect = el.getBoundingClientRect();
const label = el.getAttribute("aria-label") || el.getAttribute("alt") || el.getAttribute("title") || "";
const text = (el.textContent || "").replace(/\\s+/g, " ").trim();
const nameSource = label || text;
const nameLimit = 120;
// This serialized page function cannot call imported helpers; back up only when the cap splits a surrogate pair.
const nameEnd = (nameSource.codePointAt(nameLimit - 1) || 0) > 0xffff ? nameLimit - 1 : nameLimit;
return {
tag: el.tagName.toLowerCase(),
id: el.id || "",
classes: Array.from(el.classList).slice(0, 6),
role: el.getAttribute("role") || "",
name: (label || text).slice(0, 120),
name: nameSource.slice(0, nameEnd),
rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
focusable: typeof el.tabIndex === "number" && el.tabIndex >= 0,
};