mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-16 18:31:41 +00:00
fix(ui): browser annotation prompts corrupt boundary emoji (#104396)
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { GatewayBrowserClient } from "../../api/gateway.ts";
|
||||
import {
|
||||
buildAnnotationPrompt,
|
||||
describeInspectedNode,
|
||||
@@ -7,7 +8,7 @@ import {
|
||||
BROWSER_ANNOTATION_EVENT,
|
||||
type BrowserAnnotationDraft,
|
||||
} from "./browser-annotation.ts";
|
||||
import type { BrowserInspectedNode } from "./browser-client.ts";
|
||||
import { inspectBrowserElementAt, type BrowserInspectedNode } from "./browser-client.ts";
|
||||
|
||||
function node(overrides: Partial<BrowserInspectedNode> = {}): BrowserInspectedNode {
|
||||
return {
|
||||
@@ -120,6 +121,63 @@ describe("buildAnnotationPrompt", () => {
|
||||
expect(prompt.split("\n").length).toBe(3);
|
||||
});
|
||||
|
||||
it("keeps bounded page-reported fields on valid UTF-16 boundaries", () => {
|
||||
const titleAndName = `${"a".repeat(79)}😀tail`;
|
||||
const role = `${"r".repeat(39)}😀tail`;
|
||||
const prompt = buildAnnotationPrompt({
|
||||
url: "https://example.com",
|
||||
title: titleAndName,
|
||||
strokes: [],
|
||||
element: node({ name: titleAndName, role }),
|
||||
});
|
||||
|
||||
expect(prompt).toContain(`page-reported title: "${"a".repeat(79)}"`);
|
||||
expect(prompt).toContain(`button "${"a".repeat(79)}" (role=${"r".repeat(39)})`);
|
||||
});
|
||||
|
||||
it("preserves valid UTF-16 from inspected accessible names through prompt construction", async () => {
|
||||
const element = document.createElement("button");
|
||||
element.setAttribute("aria-label", `${"a".repeat(78)}${" ".repeat(41)}😀tail`);
|
||||
const originalElementFromPoint = Object.getOwnPropertyDescriptor(document, "elementFromPoint");
|
||||
Object.defineProperty(document, "elementFromPoint", {
|
||||
configurable: true,
|
||||
value: vi.fn(() => element),
|
||||
});
|
||||
const client = {
|
||||
request: vi.fn(async (_method: string, envelope: { body?: { fn?: string } }) => {
|
||||
const fn = envelope.body?.fn;
|
||||
if (!fn) {
|
||||
throw new Error("missing browser evaluation function");
|
||||
}
|
||||
return { result: (0, eval)(`(${fn})`)() };
|
||||
}),
|
||||
};
|
||||
|
||||
try {
|
||||
const inspected = await inspectBrowserElementAt(client as unknown as GatewayBrowserClient, {
|
||||
targetId: "proof-tab",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
expect(inspected).not.toBeNull();
|
||||
const prompt = buildAnnotationPrompt({
|
||||
url: "https://example.com",
|
||||
title: "Boundary proof",
|
||||
strokes: [],
|
||||
element: inspected,
|
||||
});
|
||||
|
||||
expect(inspected?.name.charCodeAt((inspected?.name.length ?? 0) - 1)).not.toBe(0xd83d);
|
||||
expect(prompt).toContain(`button "${"a".repeat(78)}"`);
|
||||
} finally {
|
||||
if (originalElementFromPoint) {
|
||||
Object.defineProperty(document, "elementFromPoint", originalElementFromPoint);
|
||||
} else {
|
||||
Reflect.deleteProperty(document, "elementFromPoint");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("strips hostile characters from selector fragments", () => {
|
||||
const descriptor = describeInspectedNode(
|
||||
node({
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Annotation model for the browser panel: freehand strokes drawn over a page
|
||||
// screenshot, plus the prepackaged prompt handed to the chat composer so the
|
||||
// agent knows what was marked up.
|
||||
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { t } from "../../i18n/index.ts";
|
||||
import type { BrowserInspectedNode } from "./browser-client.ts";
|
||||
|
||||
@@ -68,7 +69,7 @@ function percent(value: number): string {
|
||||
* prompt template additionally labels these values as page-reported.
|
||||
*/
|
||||
function sanitizePageText(value: string, maxLength = 80): string {
|
||||
return value.replace(/\s+/g, " ").trim().slice(0, maxLength);
|
||||
return truncateUtf16Safe(value.replace(/\s+/g, " ").trim(), maxLength);
|
||||
}
|
||||
|
||||
/** Selector fragments (tag/id/class) are page-controlled too: keep only
|
||||
|
||||
@@ -266,12 +266,16 @@ export async function inspectBrowserElementAt(
|
||||
const rect = el.getBoundingClientRect();
|
||||
const label = el.getAttribute("aria-label") || el.getAttribute("alt") || el.getAttribute("title") || "";
|
||||
const text = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
||||
const nameSource = label || text;
|
||||
const nameLimit = 120;
|
||||
// This serialized page function cannot call imported helpers; back up only when the cap splits a surrogate pair.
|
||||
const nameEnd = (nameSource.codePointAt(nameLimit - 1) || 0) > 0xffff ? nameLimit - 1 : nameLimit;
|
||||
return {
|
||||
tag: el.tagName.toLowerCase(),
|
||||
id: el.id || "",
|
||||
classes: Array.from(el.classList).slice(0, 6),
|
||||
role: el.getAttribute("role") || "",
|
||||
name: (label || text).slice(0, 120),
|
||||
name: nameSource.slice(0, nameEnd),
|
||||
rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
|
||||
focusable: typeof el.tabIndex === "number" && el.tabIndex >= 0,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user