From b58a75d7263160c9b4c356df1f28ea0cc7bf5d8d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 14 Jul 2026 07:57:03 +0100 Subject: [PATCH] refactor(html): share entity decoding --- CHANGELOG.md | 1 + .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/plugins/sdk-subpaths.md | 2 +- extensions/duckduckgo/src/ddg-client.ts | 62 ++++--------------- .../matrix/src/matrix/monitor/mentions.ts | 41 +++--------- extensions/msteams/src/graph-thread.test.ts | 8 +-- extensions/msteams/src/graph-thread.ts | 13 +--- extensions/msteams/src/inbound.test.ts | 4 +- extensions/msteams/src/inbound.ts | 26 ++------ scripts/plugin-sdk-surface-report.mjs | 5 +- .../tool-call-argument-decoding.ts | 2 +- src/agents/tools/web-fetch-utils.ts | 2 +- src/agents/utils/syntax-highlight.ts | 2 +- src/plugin-sdk/text-utility-runtime.test.ts | 8 ++- src/plugin-sdk/text-utility-runtime.ts | 2 + .../html-entities.test.ts} | 2 +- .../utils/html.ts => shared/html-entities.ts} | 0 17 files changed, 55 insertions(+), 129 deletions(-) rename src/{agents/utils/html.test.ts => shared/html-entities.test.ts} (94%) rename src/{agents/utils/html.ts => shared/html-entities.ts} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c2dd174bc35..4f0ee258ce25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- **Microsoft Teams HTML text:** decode HTML5 entities consistently in quoted and Graph-fetched messages while preserving literal escaped entity text. - **ClawHub plugin API ranges:** delegate each supported comparator to `semver` so tilde, partial-wildcard, and prerelease caret bounds are correct while preserving OpenClaw version normalization and the existing restricted range grammar. (#106877) - **Web Readability relative links:** seed parsed documents with the request URL so article links resolve correctly while removing the plugin's duplicate lazy-loader facade. (#106860) - **Browser auto-routing:** fall back to the Gateway host when an implicitly selected browser node reports that its control host is unreachable, while preserving explicit node pins and ambiguous action failures. diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 4e47d20844ed..c07933b447ed 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -6b509179fce1f5115037be7432b2dc21dda507351296ce5ca769e24741cece87 plugin-sdk-api-baseline.json -d317f5ef0a767f557357a2f7699e27e0f436edd18298401ee7a982805bcd1f76 plugin-sdk-api-baseline.jsonl +b01a5fcd6cf5e4f6189638b28a2fe17d227d7028a15e047f3a82915ab6b96ace plugin-sdk-api-baseline.json +03d5d1c2a6cefe69677f44e7df2a8864873b3bda812bd18d996043eb861e5c4e plugin-sdk-api-baseline.jsonl diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index a0cc92cece75..eccb310c01f4 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -336,7 +336,7 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/session-binding-runtime` | Current conversation binding state without configured binding routing or pairing stores | | `plugin-sdk/context-visibility-runtime` | Context visibility resolution and supplemental context filtering without broad config/security imports | | `plugin-sdk/string-coerce-runtime` | Narrow primitive record/string coercion and normalization helpers without markdown/logging imports | - | `plugin-sdk/text-utility-runtime` | Low-level text and path helpers, including five-entity HTML escaping | + | `plugin-sdk/text-utility-runtime` | Low-level text and path helpers, including five-entity HTML escaping and single-pass semicolon-terminated HTML5 entity decoding | | `plugin-sdk/host-runtime` | Hostname and SCP host normalization helpers | | `plugin-sdk/retry-runtime` | Retry config and retry runner helpers | | `plugin-sdk/agent-runtime` | Deprecated broad barrel for agent dir/identity/workspace helpers, including `resolveAgentDir`, `resolveDefaultAgentDir`, and the deprecated `resolveOpenClawAgentDir` compatibility export; prefer focused agent/runtime subpaths | diff --git a/extensions/duckduckgo/src/ddg-client.ts b/extensions/duckduckgo/src/ddg-client.ts index a05dd40b470f..b6a1955ec214 100644 --- a/extensions/duckduckgo/src/ddg-client.ts +++ b/extensions/duckduckgo/src/ddg-client.ts @@ -15,10 +15,19 @@ import { wrapWebContent, writeCache, } from "openclaw/plugin-sdk/provider-web-search"; +import { decodeHtmlEntities as decodeHtmlEntity } from "openclaw/plugin-sdk/text-utility-runtime"; import { resolveDdgRegion, resolveDdgSafeSearch, type DdgSafeSearch } from "./config.js"; const DDG_HTML_ENDPOINT = "https://html.duckduckgo.com/html"; const DEFAULT_TIMEOUT_SECONDS = 20; +const DDG_HTML_ENTITY_RE = + /&(?:lt|gt|quot|apos|#39|#x27|#x2F|nbsp|ndash|mdash|hellip|amp|#\d+|#x[0-9a-f]+);/gi; +const DDG_ENTITY_TEXT: Readonly> = { + "\u00a0": " ", + "–": "-", + "—": "--", + "…": "...", +}; const DDG_SAFE_SEARCH_PARAM: Record = { strict: "1", moderate: "-1", @@ -36,56 +45,11 @@ type DuckDuckGoResult = { snippet: string; }; -function isDecodableCodePoint(cp: number): boolean { - return Number.isInteger(cp) && cp >= 0 && cp <= 0x10ffff && (cp < 0xd800 || cp > 0xdfff); -} - function decodeHtmlEntities(text: string): string { - return text.replace( - /&(?:lt|gt|quot|apos|#39|#x27|#x2F|nbsp|ndash|mdash|hellip|amp|#\d+|#x[0-9a-f]+);/gi, - (entity) => { - const normalized = entity.toLowerCase(); - if (normalized === "<") { - return "<"; - } - if (normalized === ">") { - return ">"; - } - if (normalized === """) { - return '"'; - } - if (normalized === "'" || normalized === "'" || normalized === "'") { - return "'"; - } - if (normalized === "/") { - return "/"; - } - if (normalized === " ") { - return " "; - } - if (normalized === "–") { - return "-"; - } - if (normalized === "—") { - return "--"; - } - if (normalized === "…") { - return "..."; - } - if (normalized === "&") { - return "&"; - } - if (normalized.startsWith("&#x")) { - const codePoint = Number.parseInt(normalized.slice(3, -1), 16); - return isDecodableCodePoint(codePoint) ? String.fromCodePoint(codePoint) : entity; - } - if (normalized.startsWith("&#")) { - const codePoint = Number.parseInt(normalized.slice(2, -1), 10); - return isDecodableCodePoint(codePoint) ? String.fromCodePoint(codePoint) : entity; - } - return entity; - }, - ); + return text.replace(DDG_HTML_ENTITY_RE, (entity) => { + const decoded = decodeHtmlEntity(entity.startsWith("&#") ? entity : entity.toLowerCase()); + return DDG_ENTITY_TEXT[decoded] ?? decoded; + }); } function stripHtml(html: string): string { diff --git a/extensions/matrix/src/matrix/monitor/mentions.ts b/extensions/matrix/src/matrix/monitor/mentions.ts index 942282f5458c..f17fadea91f1 100644 --- a/extensions/matrix/src/matrix/monitor/mentions.ts +++ b/extensions/matrix/src/matrix/monitor/mentions.ts @@ -1,48 +1,21 @@ // Matrix plugin module implements mentions behavior. import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { escapeRegExp } from "openclaw/plugin-sdk/text-utility-runtime"; +import { decodeHtmlEntities, escapeRegExp } from "openclaw/plugin-sdk/text-utility-runtime"; import { getMatrixRuntime } from "../../runtime.js"; import type { RoomMessageEventContent } from "./types.js"; -const HTML_ENTITY_REPLACEMENTS: Readonly> = { - amp: "&", - apos: "'", - gt: ">", - lt: "<", - nbsp: " ", - quot: '"', -}; -const MAX_UNICODE_SCALAR_VALUE = 0x10ffff; +const MATRIX_HTML_ENTITY_RE = /&(?:#x?[0-9a-f]+|amp|apos|gt|lt|nbsp|quot);/gi; -function decodeNumericHtmlEntity(match: string, rawValue: string, radix: 10 | 16): string { - const codePoint = Number.parseInt(rawValue, radix); - if ( - !Number.isSafeInteger(codePoint) || - codePoint < 0 || - codePoint > MAX_UNICODE_SCALAR_VALUE || - (codePoint >= 0xd800 && codePoint <= 0xdfff) - ) { - return match; - } - return String.fromCodePoint(codePoint); -} - -function decodeHtmlEntities(value: string): string { - return value.replace(/&(#x?[0-9a-f]+|\w+);/gi, (match, entity: string) => { - const normalized = normalizeLowercaseStringOrEmpty(entity); - if (normalized.startsWith("#x")) { - return decodeNumericHtmlEntity(match, normalized.slice(2), 16); - } - if (normalized.startsWith("#")) { - return decodeNumericHtmlEntity(match, normalized.slice(1), 10); - } - return HTML_ENTITY_REPLACEMENTS[normalized] ?? match; +function decodeVisibleHtmlEntities(value: string): string { + return value.replace(MATRIX_HTML_ENTITY_RE, (entity) => { + const decoded = decodeHtmlEntities(entity.startsWith("&#") ? entity : entity.toLowerCase()); + return decoded === "\u00a0" ? " " : decoded; }); } function normalizeVisibleMentionText(value: string): string { return normalizeLowercaseStringOrEmpty( - decodeHtmlEntities( + decodeVisibleHtmlEntities( value.replace(/<[^>]+>/g, " ").replace(/[\u200b-\u200f\u202a-\u202e\u2060-\u206f]/g, ""), ).replace(/\s+/g, " "), ); diff --git a/extensions/msteams/src/graph-thread.test.ts b/extensions/msteams/src/graph-thread.test.ts index de501dacfce1..adb7bcafb411 100644 --- a/extensions/msteams/src/graph-thread.test.ts +++ b/extensions/msteams/src/graph-thread.test.ts @@ -30,10 +30,10 @@ describe("stripHtmlFromTeamsMessage", () => { expect(stripHtmlFromTeamsMessage("

Hello world

")).toBe("Hello world"); }); - it("decodes common HTML entities", () => { - expect(stripHtmlFromTeamsMessage("& <b> "x" 'y'  z")).toBe( - "& \"x\" 'y' z", - ); + it("decodes HTML5 entities", () => { + expect( + stripHtmlFromTeamsMessage("& <b> "x" 'y'  z ©"), + ).toBe("& \"x\" 'y' z ©"); }); it("does not double-decode escaped entities (decodes & last)", () => { diff --git a/extensions/msteams/src/graph-thread.ts b/extensions/msteams/src/graph-thread.ts index dd4322584106..5e8f8e9a9863 100644 --- a/extensions/msteams/src/graph-thread.ts +++ b/extensions/msteams/src/graph-thread.ts @@ -1,4 +1,5 @@ // Msteams plugin module implements graph thread behavior. +import { decodeHtmlEntities } from "openclaw/plugin-sdk/text-utility-runtime"; import { fetchGraphJson, type GraphResponse } from "./graph.js"; import type { MSTeamsRequestDeadline } from "./request-timeout.js"; @@ -21,16 +22,8 @@ export function stripHtmlFromTeamsMessage(html: string): string { let text = html.replace(/]*>(.*?)<\/at>/gi, "@$1"); // Strip remaining HTML tags. text = text.replace(/<[^>]*>/g, " "); - // Decode common HTML entities. & must be decoded LAST to prevent - // double-decoding (e.g. &lt; → < not <), matching decodeHtmlEntities - // in inbound.ts. - text = text - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/ /g, " ") - .replace(/&/g, "&"); + // Single-pass decoding preserves literally typed entity text such as "<". + text = decodeHtmlEntities(text).replaceAll("\u00a0", " "); // Normalize whitespace. return text.replace(/\s+/g, " ").trim(); } diff --git a/extensions/msteams/src/inbound.test.ts b/extensions/msteams/src/inbound.test.ts index a0ddfd79fce2..29289e458225 100644 --- a/extensions/msteams/src/inbound.test.ts +++ b/extensions/msteams/src/inbound.test.ts @@ -128,11 +128,11 @@ describe("msteams inbound", () => { content: '
' + 'Bob' + - '

2 < 3 & 4 > 1

' + + '

2 < 3 & 4 > 1; © keep &lt; literal

' + "
", }, ]); - expect(result).toEqual({ sender: "Bob", body: "2 < 3 & 4 > 1" }); + expect(result).toEqual({ sender: "Bob", body: "2 < 3 & 4 > 1; © keep < literal" }); }); it("handles multiline body by collapsing whitespace", () => { diff --git a/extensions/msteams/src/inbound.ts b/extensions/msteams/src/inbound.ts index 1c561ba5a336..5e986581d7fe 100644 --- a/extensions/msteams/src/inbound.ts +++ b/extensions/msteams/src/inbound.ts @@ -1,4 +1,6 @@ // Msteams plugin module implements inbound behavior. +import { decodeHtmlEntities } from "openclaw/plugin-sdk/text-utility-runtime"; + type MSTeamsQuoteInfo = { sender: string; body: string; @@ -10,30 +12,14 @@ type MSTeamsQuoteInfo = { id?: string; }; -/** - * Decode common HTML entities to plain text. - */ -function decodeHtmlEntities(html: string): string { - return html - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/'/g, "'") - .replace(/ /g, " ") - .replace(/&/g, "&"); // must be last to prevent double-decoding (e.g. &lt; → < not <) -} - /** * Strip HTML tags, preserving text content. */ function htmlToPlainText(html: string): string { - return decodeHtmlEntities( - html - .replace(/<[^>]*>/g, " ") - .replace(/\s+/g, " ") - .trim(), - ); + return decodeHtmlEntities(html.replace(/<[^>]*>/g, " ")) + .replaceAll("\u00a0", " ") + .replace(/\s+/g, " ") + .trim(); } /** diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 253a69dadeb0..9b1876646ab6 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -205,14 +205,15 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // ScopeTree adds six channel-policy exports, mirrored by compat, including three functions. // Its flat channel-groups builder adds one function, also mirrored by compat. // Its case-insensitive scope-key resolver adds one function, also mirrored by compat. + // The focused text utility HTML decoder adds one public function. publicExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", - 10674, + 10675, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", - 5375, + 5376, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/agents/embedded-agent-runner/tool-call-argument-decoding.ts b/src/agents/embedded-agent-runner/tool-call-argument-decoding.ts index d7239d4bce39..a0ee750430cb 100644 --- a/src/agents/embedded-agent-runner/tool-call-argument-decoding.ts +++ b/src/agents/embedded-agent-runner/tool-call-argument-decoding.ts @@ -2,10 +2,10 @@ * Decodes HTML-entity escaped tool-call arguments in stream wrappers. */ import { streamSimple } from "../../llm/stream.js"; +import { decodeHtmlEntities } from "../../shared/html-entities.js"; import { visitObjectContentBlocks } from "../../shared/message-content-blocks.js"; import type { StreamFn } from "../runtime/index.js"; import type { MutableAssistantMessageEventStream } from "../stream-compat.js"; -import { decodeHtmlEntities } from "../utils/html.js"; /** * Decodes HTML entities inside streamed tool-call arguments before downstream execution. diff --git a/src/agents/tools/web-fetch-utils.ts b/src/agents/tools/web-fetch-utils.ts index 2d172a340e5c..b2279eea79fe 100644 --- a/src/agents/tools/web-fetch-utils.ts +++ b/src/agents/tools/web-fetch-utils.ts @@ -4,7 +4,7 @@ * Converts lightweight HTML into bounded markdown/text without pulling in a full renderer. */ import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; -import { decodeHtmlEntities } from "../utils/html.js"; +import { decodeHtmlEntities } from "../../shared/html-entities.js"; import { sanitizeHtml, stripInvisibleUnicode } from "./web-fetch-visibility.js"; /** Output mode requested by web_fetch extraction. */ diff --git a/src/agents/utils/syntax-highlight.ts b/src/agents/utils/syntax-highlight.ts index 6692365b02d3..9516b270abdc 100644 --- a/src/agents/utils/syntax-highlight.ts +++ b/src/agents/utils/syntax-highlight.ts @@ -5,7 +5,7 @@ * maps active scopes to caller-provided text formatters. */ import { createRequire } from "node:module"; -import { decodeHtmlEntities } from "./html.js"; +import { decodeHtmlEntities } from "../../shared/html-entities.js"; type HighlightJs = { getLanguage(name: string): unknown; diff --git a/src/plugin-sdk/text-utility-runtime.test.ts b/src/plugin-sdk/text-utility-runtime.test.ts index 247f63fc14bc..965de8f91541 100644 --- a/src/plugin-sdk/text-utility-runtime.test.ts +++ b/src/plugin-sdk/text-utility-runtime.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { escapeHtml } from "./text-utility-runtime.js"; +import { decodeHtmlEntities, escapeHtml } from "./text-utility-runtime.js"; describe("escapeHtml", () => { it("escapes five HTML-sensitive characters and existing entity markers", () => { @@ -7,3 +7,9 @@ describe("escapeHtml", () => { expect(escapeHtml("already & escaped")).toBe("already &amp; escaped"); }); }); + +describe("decodeHtmlEntities", () => { + it("exposes single-pass HTML5 decoding to plugins", () => { + expect(decodeHtmlEntities("© &lt; 😀 �")).toBe("© < 😀 �"); + }); +}); diff --git a/src/plugin-sdk/text-utility-runtime.ts b/src/plugin-sdk/text-utility-runtime.ts index 2f6ed1d72c4d..881148ec474d 100644 --- a/src/plugin-sdk/text-utility-runtime.ts +++ b/src/plugin-sdk/text-utility-runtime.ts @@ -10,6 +10,8 @@ export function escapeHtml(value: string): string { .replaceAll("'", "'"); } +export { decodeHtmlEntities } from "../shared/html-entities.js"; + export { CONFIG_DIR, clamp, diff --git a/src/agents/utils/html.test.ts b/src/shared/html-entities.test.ts similarity index 94% rename from src/agents/utils/html.test.ts rename to src/shared/html-entities.test.ts index 62e8580a36d0..402441b9f0cd 100644 --- a/src/agents/utils/html.test.ts +++ b/src/shared/html-entities.test.ts @@ -1,6 +1,6 @@ // Shared HTML entity tests cover all agent renderers and tool-argument repair callers. import { describe, expect, it } from "vitest"; -import { decodeHtmlEntities } from "./html.js"; +import { decodeHtmlEntities } from "./html-entities.js"; describe("decodeHtmlEntities", () => { it("decodes HTML5 named entities beyond the XML subset", () => { diff --git a/src/agents/utils/html.ts b/src/shared/html-entities.ts similarity index 100% rename from src/agents/utils/html.ts rename to src/shared/html-entities.ts