diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ec8aa49998..b7bf13daf6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ Docs: https://docs.openclaw.ai - Channels/streaming: cap progress-draft tool lines by default so edited progress boxes avoid jumpy reflow from long wrapped lines. - Agents/verbose: use compact explain-mode tool summaries for `/verbose` and progress drafts by default, with `agents.defaults.toolProgressDetail: "raw"` and per-agent overrides for debugging raw command/detail output. - Control UI/chat: add an agent-first filter to the chat session picker, keep chat controls/composer responsive across phone/tablet/desktop widths, keep desktop chat controls on one row, avoid duplicate avatar refreshes during initial chat load, and hide that row while scrolling down the transcript. Thanks @BunsDev. -- Control UI/chat: collapse consecutive duplicate text messages into one bubble with a count so no-op heartbeat acknowledgements stay compact without hiding nearby context. +- Control UI/chat: collapse consecutive duplicate text messages into one bubble with a count so repeated text-only messages stay compact without hiding nearby context. - Agents/subagents: preserve every grouped child result when direct completion fallback has to bypass the requester-agent announce turn. Thanks @vincentkoc. - TTS/telephony: honor provider voice/model overrides in telephony synthesis providers so Google Meet agent speech logs match the backend that actually produced the audio. Thanks @vincentkoc. - Voice Call/realtime: bound the paced Twilio audio queue and close overloaded realtime streams before provider audio can pile up behind the websocket backpressure guard. Thanks @vincentkoc. @@ -70,6 +70,7 @@ Docs: https://docs.openclaw.ai - Plugins/install: honor the beta update channel for onboarding and doctor-managed plugin installs by requesting floating npm and ClawHub specs with `@beta` while keeping persistent install records on the catalog default. Thanks @vincentkoc. - WhatsApp/onboarding: canonicalize setup and pairing allowlist entries to WhatsApp's digit-only phone ids while still accepting E.164, JID, and `whatsapp:` inputs, so personal-phone allowlists match WhatsApp Web sender ids after setup. Thanks @vincentkoc. - Gateway/startup: load provider plugins that own explicitly configured image, video, or music generation defaults so generation tools become live after gateway restart instead of remaining catalog-only. Fixes #77244. Thanks @buyuangtampan, @Nikoxx99, and @vincentkoc. +- Control UI/chat: suppress `HEARTBEAT_OK` acknowledgement history, streams, deltas, and final events before they enter the transcript view, so repeated heartbeat no-op turns do not stack noisy bubbles. Thanks @BunsDev. - Slack/subagents: keep resumed parent `message.send` calls in the originating Slack thread when ambient session thread context is present, and suppress successful silent child completion rows from follow-up findings. Thanks @bek91. - Slack/mentions: record thread participation for successful visible threaded Slack sends, including message-tool and media delivery paths, so unmentioned replies in bot-participated threads can bypass mention gating as documented. Fixes #77648. Thanks @bek91. - Infra/Windows: skip the POSIX `/tmp/openclaw` preferred path on Windows in `resolvePreferredOpenClawTmpDir` so log files, TTS temp files, and other writes land in `%TEMP%\openclaw-` instead of `C:\tmp\openclaw`. Fixes #60713. Thanks @juan-flores077. diff --git a/docs/web/control-ui.md b/docs/web/control-ui.md index 0fb72fc76b6..d3a1e9032fc 100644 --- a/docs/web/control-ui.md +++ b/docs/web/control-ui.md @@ -154,7 +154,7 @@ Imported themes are stored only in the current browser profile. They are not wri - Re-sending with the same `idempotencyKey` returns `{ status: "in_flight" }` while running, and `{ status: "ok" }` after completion. - `chat.history` responses are size-bounded for UI safety. When transcript entries are too large, Gateway may truncate long text fields, omit heavy metadata blocks, and replace oversized messages with a placeholder (`[chat.history omitted: message too large]`). - Assistant/generated images are persisted as managed media references and served back through authenticated Gateway media URLs, so reloads do not depend on raw base64 image payloads staying in the chat history response. - - `chat.history` also strips display-only inline directive tags from visible assistant text (for example `[[reply_to_*]]` and `[[audio_as_voice]]`), plain-text tool-call XML payloads (including `...`, `...`, `...`, `...`, and truncated tool-call blocks), and leaked ASCII/full-width model control tokens, and omits assistant entries whose whole visible text is only the exact silent token `NO_REPLY` / `no_reply`. + - When rendering `chat.history`, the Control UI strips display-only inline directive tags from visible assistant text (for example `[[reply_to_*]]` and `[[audio_as_voice]]`), plain-text tool-call XML payloads (including `...`, `...`, `...`, `...`, and truncated tool-call blocks), and leaked ASCII/full-width model control tokens, and omits assistant entries whose whole visible text is only the exact silent token `NO_REPLY` / `no_reply` or the heartbeat acknowledgement token `HEARTBEAT_OK`. - During an active send and the final history refresh, the chat view keeps local optimistic user/assistant messages visible if `chat.history` briefly returns an older snapshot; the canonical transcript replaces those local messages once the Gateway history catches up. - Live `chat` events are delivery state, while `chat.history` is rebuilt from the durable session transcript. After tool-final events the Control UI reloads history and merges only a small optimistic tail; the transcript boundary is documented in [WebChat](/web/webchat). - `chat.inject` appends an assistant note to the session transcript and broadcasts a `chat` event for UI-only updates (no agent run, no channel delivery). diff --git a/ui/src/ui/chat/build-chat-items.test.ts b/ui/src/ui/chat/build-chat-items.test.ts index b01e0517455..30e0cc322aa 100644 --- a/ui/src/ui/chat/build-chat-items.test.ts +++ b/ui/src/ui/chat/build-chat-items.test.ts @@ -50,9 +50,9 @@ describe("buildChatItems", () => { it("collapses consecutive duplicate text messages into one rendered item with a count", () => { const groups = messageGroups({ messages: [ - { role: "assistant", content: [{ type: "text", text: "HEARTBEAT_OK" }], timestamp: 1 }, - { role: "assistant", content: [{ type: "text", text: "HEARTBEAT_OK" }], timestamp: 2 }, - { role: "assistant", content: [{ type: "text", text: "HEARTBEAT_OK" }], timestamp: 3 }, + { role: "assistant", content: [{ type: "text", text: "Same update" }], timestamp: 1 }, + { role: "assistant", content: [{ type: "text", text: "Same update" }], timestamp: 2 }, + { role: "assistant", content: [{ type: "text", text: "Same update" }], timestamp: 3 }, ], }); @@ -61,6 +61,96 @@ describe("buildChatItems", () => { expect(groups[0].messages[0]).toMatchObject({ duplicateCount: 3 }); }); + it("suppresses assistant HEARTBEAT_OK acknowledgements before rendering history", () => { + const groups = messageGroups({ + messages: [ + { role: "assistant", content: [{ type: "text", text: "HEARTBEAT_OK" }], timestamp: 1 }, + { role: "assistant", content: "HEARTBEAT_OK", timestamp: 2 }, + { role: "user", content: [{ type: "text", text: "HEARTBEAT_OK" }], timestamp: 3 }, + { role: "assistant", content: [{ type: "text", text: "Visible reply" }], timestamp: 4 }, + ], + }); + + expect(groups).toHaveLength(2); + expect(groups[0].role).toBe("user"); + expect(groups[1].role).toBe("assistant"); + expect(groups[1].messages[0].message).toMatchObject({ + content: [{ type: "text", text: "Visible reply" }], + }); + }); + + it("suppresses assistant HEARTBEAT_OK acknowledgements that carry hidden thinking blocks", () => { + const groups = messageGroups({ + messages: [ + { + role: "assistant", + content: [ + { type: "thinking", thinking: "Checking scheduled work." }, + { + type: "text", + text: "HEARTBEAT_OK", + textSignature: JSON.stringify({ v: 1, phase: "final_answer" }), + }, + ], + timestamp: 1, + }, + { + role: "assistant", + content: [ + { id: "rs_1", type: "reasoning" }, + { type: "text", text: "HEARTBEAT_OK" }, + ], + timestamp: 2, + }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "Useful hidden reasoning." }, + { type: "text", text: "Visible reply" }, + ], + timestamp: 3, + }, + ], + }); + + expect(groups).toHaveLength(1); + expect(groups[0].messages).toHaveLength(1); + expect(groups[0].messages[0].message).toMatchObject({ + content: [ + { type: "thinking", thinking: "Useful hidden reasoning." }, + { type: "text", text: "Visible reply" }, + ], + }); + }); + + it("keeps HEARTBEAT_OK turns that carry visible non-text content", () => { + const canvasBlock = createAssistantCanvasBlock({ suffix: "heartbeat_visible_content" }); + const groups = messageGroups({ + messages: [ + { + role: "assistant", + content: [{ type: "text", text: "HEARTBEAT_OK" }, canvasBlock], + timestamp: 1, + }, + ], + }); + + expect(groups).toHaveLength(1); + expect(groups[0].messages).toHaveLength(1); + expect(firstMessageContent(groups[0]).some((block) => isCanvasBlock(block))).toBe(true); + }); + + it("suppresses active HEARTBEAT_OK streams before rendering", () => { + const items = buildChatItems( + createProps({ + stream: "HEARTBEAT_OK", + streamStartedAt: 1, + }), + ); + + expect(items).toEqual([]); + }); + it("does not collapse duplicate text messages separated by another message", () => { const groups = messageGroups({ messages: [ diff --git a/ui/src/ui/chat/build-chat-items.ts b/ui/src/ui/chat/build-chat-items.ts index 086605bfe6b..1fee49777a8 100644 --- a/ui/src/ui/chat/build-chat-items.ts +++ b/ui/src/ui/chat/build-chat-items.ts @@ -1,4 +1,8 @@ import type { ChatItem, MessageGroup, ToolCard } from "../types/chat-types.ts"; +import { + isAssistantHeartbeatAckForDisplay, + stripHeartbeatTokenForDisplay, +} from "./heartbeat-display.ts"; import { extractTextCached } from "./message-extract.ts"; import { normalizeMessage } from "./message-normalizer.ts"; import { normalizeRoleForGrouping } from "./role-normalizer.ts"; @@ -248,7 +252,9 @@ function collapseSequentialDuplicateMessages(items: ChatItem[]): ChatItem[] { export function buildChatItems(props: BuildChatItemsProps): Array { const items: ChatItem[] = []; - const history = Array.isArray(props.messages) ? props.messages : []; + const history = (Array.isArray(props.messages) ? props.messages : []).filter( + (message) => !isAssistantHeartbeatAckForDisplay(message), + ); const tools = Array.isArray(props.toolMessages) ? props.toolMessages : []; const historyStart = Math.max(0, history.length - CHAT_HISTORY_RENDER_LIMIT); if (historyStart > 0) { @@ -349,12 +355,14 @@ export function buildChatItems(props: BuildChatItemsProps): Array 0) { - items.push({ - kind: "stream", - key, - text: props.stream, - startedAt: props.streamStartedAt ?? Date.now(), - }); + if (!stripHeartbeatTokenForDisplay(props.stream).shouldSkip) { + items.push({ + kind: "stream", + key, + text: props.stream, + startedAt: props.streamStartedAt ?? Date.now(), + }); + } } else { items.push({ kind: "reading-indicator", key }); } diff --git a/ui/src/ui/chat/heartbeat-display.ts b/ui/src/ui/chat/heartbeat-display.ts new file mode 100644 index 00000000000..a2e7677473f --- /dev/null +++ b/ui/src/ui/chat/heartbeat-display.ts @@ -0,0 +1,111 @@ +import { normalizeLowercaseStringOrEmpty } from "../string-coerce.ts"; + +const HEARTBEAT_TOKEN = "HEARTBEAT_OK"; +const DEFAULT_HEARTBEAT_ACK_MAX_CHARS = 300; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +export function stripHeartbeatTokenForDisplay( + raw: string, + maxAckChars = DEFAULT_HEARTBEAT_ACK_MAX_CHARS, +): { shouldSkip: boolean } { + let text = raw.trim(); + if (!text) { + return { shouldSkip: true }; + } + const strippedMarkup = text + .replace(/<[^>]*>/g, " ") + .replace(/ /gi, " ") + .replace(/^[*`~_]+/, "") + .replace(/[*`~_]+$/, ""); + if (!text.includes(HEARTBEAT_TOKEN) && !strippedMarkup.includes(HEARTBEAT_TOKEN)) { + return { shouldSkip: false }; + } + + const tokenAtEnd = new RegExp(`${escapeRegExp(HEARTBEAT_TOKEN)}[^\\w]{0,4}$`); + let changed = true; + let didStrip = false; + text = strippedMarkup.trim(); + while (changed) { + changed = false; + const next = text.trim(); + if (next.startsWith(HEARTBEAT_TOKEN)) { + text = next.slice(HEARTBEAT_TOKEN.length).trimStart(); + didStrip = true; + changed = true; + continue; + } + if (tokenAtEnd.test(next)) { + const index = next.lastIndexOf(HEARTBEAT_TOKEN); + const before = next.slice(0, index).trimEnd(); + const after = next.slice(index + HEARTBEAT_TOKEN.length).trimStart(); + text = before ? `${before}${after}`.trimEnd() : ""; + didStrip = true; + changed = true; + } + } + + if (!didStrip) { + return { shouldSkip: false }; + } + return { shouldSkip: !text || text.length <= maxAckChars }; +} + +function isHiddenDisplayBlockType(type: unknown): boolean { + return type === "thinking" || type === "reasoning"; +} + +function resolveDisplayContent(content: unknown): { + text: string; + hasVisibleNonTextContent: boolean; +} { + if (typeof content === "string") { + return { text: content, hasVisibleNonTextContent: false }; + } + if (!Array.isArray(content)) { + return { text: "", hasVisibleNonTextContent: content != null }; + } + let hasVisibleNonTextContent = false; + const text = content + .filter((block): block is { type: "text"; text: string } => { + if (!block || typeof block !== "object" || !("type" in block)) { + hasVisibleNonTextContent = true; + return false; + } + if ((block as { type?: unknown }).type !== "text") { + if (!isHiddenDisplayBlockType((block as { type?: unknown }).type)) { + hasVisibleNonTextContent = true; + } + return false; + } + if (typeof (block as { text?: unknown }).text !== "string") { + hasVisibleNonTextContent = true; + return false; + } + return true; + }) + .map((block) => block.text) + .join(""); + return { text, hasVisibleNonTextContent }; +} + +export function isAssistantHeartbeatAckForDisplay(message: unknown): boolean { + if (!message || typeof message !== "object") { + return false; + } + const entry = message as Record; + const role = normalizeLowercaseStringOrEmpty(entry.role); + if (role !== "assistant") { + return false; + } + + const content = + typeof entry.content === "string" || Array.isArray(entry.content) ? entry.content : entry.text; + const { text, hasVisibleNonTextContent } = resolveDisplayContent(content); + if (hasVisibleNonTextContent) { + return false; + } + return stripHeartbeatTokenForDisplay(text).shouldSkip; +} diff --git a/ui/src/ui/controllers/chat.test.ts b/ui/src/ui/controllers/chat.test.ts index 0115c549614..6fcaa480d0f 100644 --- a/ui/src/ui/controllers/chat.test.ts +++ b/ui/src/ui/controllers/chat.test.ts @@ -223,6 +223,17 @@ describe("handleChatEvent", () => { expect(state.chatMessages).toEqual([]); }); + it("drops HEARTBEAT_OK final payload from another run without clearing active stream", () => { + const state = createActiveStreamingState(); + const payload = createOtherRunSilentFinalPayload("HEARTBEAT_OK"); + + expect(handleChatEvent(state, payload)).toBe("final"); + expect(state.chatRunId).toBe("run-user"); + expect(state.chatStream).toBe("Working..."); + expect(state.chatStreamStartedAt).toBe(123); + expect(state.chatMessages).toEqual([]); + }); + it.each(["no_reply", "ANNOUNCE_SKIP", "REPLY_SKIP"])( "keeps plain-text %s final payload from another run without clearing active stream", (text) => { @@ -237,6 +248,23 @@ describe("handleChatEvent", () => { }, ); + it("ignores HEARTBEAT_OK delta updates", () => { + const state = createState({ + sessionKey: "main", + chatRunId: "run-1", + chatStream: "Previous visible text", + }); + const payload: ChatEventPayload = { + runId: "run-1", + sessionKey: "main", + state: "delta", + message: { role: "assistant", content: [{ type: "text", text: "HEARTBEAT_OK" }] }, + }; + + expect(handleChatEvent(state, payload)).toBe("delta"); + expect(state.chatStream).toBe("Previous visible text"); + }); + it("replaces the stream when a delta snapshot gets shorter", () => { const state = createState({ sessionKey: "main", diff --git a/ui/src/ui/controllers/chat.ts b/ui/src/ui/controllers/chat.ts index a7673ac0d0f..9e6647f1fa3 100644 --- a/ui/src/ui/controllers/chat.ts +++ b/ui/src/ui/controllers/chat.ts @@ -3,6 +3,10 @@ import { getChatAttachmentDataUrl, getChatAttachmentPreviewUrl, } from "../chat/attachment-payload-store.ts"; +import { + isAssistantHeartbeatAckForDisplay, + stripHeartbeatTokenForDisplay, +} from "../chat/heartbeat-display.ts"; import { extractText } from "../chat/message-extract.ts"; import { formatConnectError } from "../connect-error.ts"; import { GatewayRequestError, type GatewayBrowserClient } from "../gateway.ts"; @@ -14,9 +18,7 @@ import { isMissingOperatorReadScopeError, } from "./scope-errors.ts"; -const HEARTBEAT_TOKEN = "HEARTBEAT_OK"; const SILENT_REPLY_PATTERN = /^\s*NO_REPLY\s*$/; -const DEFAULT_HEARTBEAT_ACK_MAX_CHARS = 300; const SYNTHETIC_TRANSCRIPT_REPAIR_RESULT = "[openclaw] missing tool result in session history; inserted synthetic error result for transcript repair."; const STARTUP_CHAT_HISTORY_RETRY_TIMEOUT_MS = 60_000; @@ -47,96 +49,6 @@ function isSilentReplyStream(text: string): boolean { return SILENT_REPLY_PATTERN.test(text); } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -function stripHeartbeatTokenForDisplay( - raw: string, - maxAckChars = DEFAULT_HEARTBEAT_ACK_MAX_CHARS, -): { shouldSkip: boolean } { - let text = raw.trim(); - if (!text) { - return { shouldSkip: true }; - } - const strippedMarkup = text - .replace(/<[^>]*>/g, " ") - .replace(/ /gi, " ") - .replace(/^[*`~_]+/, "") - .replace(/[*`~_]+$/, ""); - if (!text.includes(HEARTBEAT_TOKEN) && !strippedMarkup.includes(HEARTBEAT_TOKEN)) { - return { shouldSkip: false }; - } - - const tokenAtEnd = new RegExp(`${escapeRegExp(HEARTBEAT_TOKEN)}[^\\w]{0,4}$`); - let changed = true; - let didStrip = false; - text = strippedMarkup.trim(); - while (changed) { - changed = false; - const next = text.trim(); - if (next.startsWith(HEARTBEAT_TOKEN)) { - text = next.slice(HEARTBEAT_TOKEN.length).trimStart(); - didStrip = true; - changed = true; - continue; - } - if (tokenAtEnd.test(next)) { - const index = next.lastIndexOf(HEARTBEAT_TOKEN); - const before = next.slice(0, index).trimEnd(); - const after = next.slice(index + HEARTBEAT_TOKEN.length).trimStart(); - text = before ? `${before}${after}`.trimEnd() : ""; - didStrip = true; - changed = true; - } - } - - if (!didStrip) { - return { shouldSkip: false }; - } - return { shouldSkip: !text || text.length <= maxAckChars }; -} - -function isHeartbeatOkResponse(message: { role: string; content?: unknown }): boolean { - if (message.role !== "assistant") { - return false; - } - const { text, hasNonTextContent } = resolveMessageText(message.content); - if (hasNonTextContent) { - return false; - } - return stripHeartbeatTokenForDisplay(text).shouldSkip; -} - -function resolveMessageText(content: unknown): { text: string; hasNonTextContent: boolean } { - if (typeof content === "string") { - return { text: content, hasNonTextContent: false }; - } - if (!Array.isArray(content)) { - return { text: "", hasNonTextContent: content != null }; - } - let hasNonTextContent = false; - const text = content - .filter((block): block is { type: "text"; text: string } => { - if (!block || typeof block !== "object" || !("type" in block)) { - hasNonTextContent = true; - return false; - } - if ((block as { type?: unknown }).type !== "text") { - hasNonTextContent = true; - return false; - } - if (typeof (block as { text?: unknown }).text !== "string") { - hasNonTextContent = true; - return false; - } - return true; - }) - .map((block) => block.text) - .join(""); - return { text, hasNonTextContent }; -} - /** Client-side defense-in-depth: detect assistant messages whose text is purely NO_REPLY. */ function isAssistantSilentReply(message: unknown): boolean { if (!message || typeof message !== "object") { @@ -209,23 +121,17 @@ function isEmptyUserTextOnlyMessage(message: unknown): boolean { return (extractText(message)?.trim() ?? "") === ""; } -function isAssistantHeartbeatAck(message: unknown): boolean { - if (!message || typeof message !== "object") { - return false; - } - const entry = message as Record; - const role = normalizeLowercaseStringOrEmpty(entry.role); - if (role !== "assistant") { - return false; - } - const content = entry.content ?? entry.text; - return isHeartbeatOkResponse({ role, content }); +function isHeartbeatAckStream(text: string): boolean { + return stripHeartbeatTokenForDisplay(text).shouldSkip; +} + +function shouldHideAssistantChatMessage(message: unknown): boolean { + return isAssistantSilentReply(message) || isAssistantHeartbeatAckForDisplay(message); } function shouldHideHistoryMessage(message: unknown): boolean { return ( - isAssistantSilentReply(message) || - isAssistantHeartbeatAck(message) || + shouldHideAssistantChatMessage(message) || isSyntheticTranscriptRepairToolResult(message) || isEmptyUserTextOnlyMessage(message) ); @@ -738,7 +644,7 @@ export function handleChatEvent(state: ChatState, payload?: ChatEventPayload) { if (state.chatRunId && payload.runId !== state.chatRunId) { if (payload.state === "final") { const finalMessage = normalizeFinalAssistantMessage(payload.message); - if (finalMessage && !isAssistantSilentReply(finalMessage)) { + if (finalMessage && !shouldHideAssistantChatMessage(finalMessage)) { state.chatMessages = [...state.chatMessages, finalMessage]; return null; } @@ -749,14 +655,22 @@ export function handleChatEvent(state: ChatState, payload?: ChatEventPayload) { if (payload.state === "delta") { const next = extractText(payload.message); - if (typeof next === "string" && !isSilentReplyStream(next)) { + if ( + typeof next === "string" && + !isSilentReplyStream(next) && + !isAssistantHeartbeatAckForDisplay(payload.message) + ) { state.chatStream = next; } } else if (payload.state === "final") { const finalMessage = normalizeFinalAssistantMessage(payload.message); - if (finalMessage && !isAssistantSilentReply(finalMessage)) { + if (finalMessage && !shouldHideAssistantChatMessage(finalMessage)) { state.chatMessages = [...state.chatMessages, finalMessage]; - } else if (state.chatStream?.trim() && !isSilentReplyStream(state.chatStream)) { + } else if ( + state.chatStream?.trim() && + !isSilentReplyStream(state.chatStream) && + !isHeartbeatAckStream(state.chatStream) + ) { state.chatMessages = [ ...state.chatMessages, { @@ -771,11 +685,15 @@ export function handleChatEvent(state: ChatState, payload?: ChatEventPayload) { state.chatStreamStartedAt = null; } else if (payload.state === "aborted") { const normalizedMessage = normalizeAbortedAssistantMessage(payload.message); - if (normalizedMessage && !isAssistantSilentReply(normalizedMessage)) { + if (normalizedMessage && !shouldHideAssistantChatMessage(normalizedMessage)) { state.chatMessages = [...state.chatMessages, normalizedMessage]; } else { const streamedText = state.chatStream ?? ""; - if (streamedText.trim() && !isSilentReplyStream(streamedText)) { + if ( + streamedText.trim() && + !isSilentReplyStream(streamedText) && + !isHeartbeatAckStream(streamedText) + ) { state.chatMessages = [ ...state.chatMessages, {