diff --git a/extensions/qa-lab/src/web-runtime.test.ts b/extensions/qa-lab/src/web-runtime.test.ts index 16403bf7fb6..df819c81d28 100644 --- a/extensions/qa-lab/src/web-runtime.test.ts +++ b/extensions/qa-lab/src/web-runtime.test.ts @@ -53,6 +53,7 @@ import { beforeEach(async () => { const page = { + on: vi.fn(), goto, title: pageTitle, url: pageUrl, diff --git a/extensions/qa-lab/src/web-runtime.ts b/extensions/qa-lab/src/web-runtime.ts index 0a29623cc00..975f4da2594 100644 --- a/extensions/qa-lab/src/web-runtime.ts +++ b/extensions/qa-lab/src/web-runtime.ts @@ -5,6 +5,12 @@ type QaWebSession = { browser: Browser; context: BrowserContext; page: Page; + diagnostics: QaWebDiagnosticEntry[]; +}; + +type QaWebDiagnosticEntry = { + kind: "console" | "pageerror" | "requestfailed"; + text: string; }; type QaWebOpenPageParams = { @@ -44,6 +50,18 @@ type QaWebEvaluateParams = { const sessions = new Map(); const DEFAULT_WEB_TIMEOUT_MS = 20_000; +const MAX_DIAGNOSTIC_ENTRIES = 50; +const MAX_DIAGNOSTIC_TEXT_CHARS = 2_000; + +function appendDiagnostic(diagnostics: QaWebDiagnosticEntry[], entry: QaWebDiagnosticEntry): void { + diagnostics.push({ + kind: entry.kind, + text: entry.text.slice(0, MAX_DIAGNOSTIC_TEXT_CHARS), + }); + if (diagnostics.length > MAX_DIAGNOSTIC_ENTRIES) { + diagnostics.splice(0, diagnostics.length - MAX_DIAGNOSTIC_ENTRIES); + } +} function resolveTimeoutMs(timeoutMs: number | undefined, fallbackMs = DEFAULT_WEB_TIMEOUT_MS) { if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) { @@ -71,12 +89,31 @@ export async function qaWebOpenPage(params: QaWebOpenPageParams) { viewport: params.viewport ?? { width: 1440, height: 1080 }, }); const page = await context.newPage(); + const diagnostics: QaWebDiagnosticEntry[] = []; + page.on("console", (message) => { + appendDiagnostic(diagnostics, { + kind: "console", + text: `[${message.type()}] ${message.text()}`, + }); + }); + page.on("pageerror", (error) => { + appendDiagnostic(diagnostics, { + kind: "pageerror", + text: error instanceof Error ? (error.stack ?? error.message) : String(error), + }); + }); + page.on("requestfailed", (request) => { + appendDiagnostic(diagnostics, { + kind: "requestfailed", + text: `${request.method()} ${request.url()} ${request.failure()?.errorText ?? "failed"}`, + }); + }); await page.goto(params.url, { waitUntil: "domcontentloaded", timeout: timeoutMs, }); const pageId = randomUUID(); - sessions.set(pageId, { browser, context, page }); + sessions.set(pageId, { browser, context, page, diagnostics }); return { pageId, url: page.url(), @@ -128,6 +165,7 @@ export async function qaWebSnapshot(params: QaWebSnapshotParams) { url: session.page.url(), title: await session.page.title().catch(() => ""), text: maxChars ? text.slice(0, maxChars) : text, + diagnostics: [...session.diagnostics], }; } diff --git a/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.md b/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.md index 31ac791d7cc..a18d6ae3ff8 100644 --- a/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.md +++ b/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.md @@ -65,7 +65,7 @@ steps: expr: "buildAgentSessionKey({ agentId: env.cfg.agents?.list?.find((agent) => agent.default)?.id ?? env.cfg.agents?.list?.[0]?.id ?? 'main', channel: 'qa-channel', accountId: 'default', peer: { kind: 'direct', id: config.conversationId }, dmScope: env.cfg.session?.dmScope, identityLinks: env.cfg.session?.identityLinks })" - set: controlUiChatUrl value: - expr: "(() => { const url = new URL(String(bootstrap.controlUiEmbeddedUrl)); url.pathname = `${url.pathname.replace(/\\/$/, '')}/chat`; url.searchParams.set('session', uiSessionKey); return url.toString(); })()" + expr: "(() => { const url = new URL(`${env.gateway.baseUrl}/`); url.searchParams.set('session', uiSessionKey); url.hash = `token=${encodeURIComponent(env.gateway.token ?? '')}`; return url.toString(); })()" - call: webOpenPage saveAs: uiTab args: @@ -80,17 +80,38 @@ steps: args: - pageId: ref: uiPageId - selector: textarea + selector: openclaw-app timeoutMs: expr: liveTurnTimeoutMs(env, 45000) - - call: waitForCondition - saveAs: uiReadySnapshot - args: - - lambda: - async: true - expr: "await (async () => { const snapshot = await webSnapshot({ pageId: uiPageId, maxChars: 12000, timeoutMs: liveTurnTimeoutMs(env, 30000) }); const text = normalizeLowercaseStringOrEmpty(snapshot.text); return text.includes('ready to chat') ? snapshot : undefined; })()" - - expr: liveTurnTimeoutMs(env, 45000) - - 500 + - try: + actions: + - call: waitForCondition + saveAs: uiReadySnapshot + args: + - lambda: + async: true + expr: "await (async () => { const snapshot = await webSnapshot({ pageId: uiPageId, maxChars: 12000, timeoutMs: liveTurnTimeoutMs(env, 30000) }); const text = normalizeLowercaseStringOrEmpty(snapshot.text); return text.includes('ready to chat') ? snapshot : undefined; })()" + - expr: liveTurnTimeoutMs(env, 45000) + - 500 + catch: + - call: webSnapshot + saveAs: uiReadyFailureSnapshot + args: + - pageId: + ref: uiPageId + maxChars: 12000 + timeoutMs: + expr: liveTurnTimeoutMs(env, 15000) + - call: webEvaluate + saveAs: uiReadyFailureState + args: + - pageId: + ref: uiPageId + expression: "(() => { const app = document.querySelector('openclaw-app'); const resources = performance.getEntriesByType('resource').map((entry) => ({ name: entry.name, type: entry.initiatorType, duration: Math.round(entry.duration), transferSize: entry.transferSize, decodedBodySize: entry.decodedBodySize })); return { url: location.href, readyState: document.readyState, appDefined: Boolean(customElements.get('openclaw-app')), appState: app ? { sessionKey: app.sessionKey, settingsSessionKey: app.settings?.sessionKey, lastActiveSessionKey: app.settings?.lastActiveSessionKey, chatMessages: Array.isArray(app.chatMessages) ? app.chatMessages.length : null, chatLoading: app.chatLoading, lastError: app.lastError, connected: app.connected, tab: app.tab } : null, scripts: Array.from(document.scripts).map((script) => script.src || script.textContent?.slice(0, 80)), links: Array.from(document.querySelectorAll('link')).map((link) => link.href), resources, bodyHtml: document.body.innerHTML.slice(0, 400) }; })()" + timeoutMs: + expr: liveTurnTimeoutMs(env, 15000) + - throw: + expr: "`control ui did not become ready. state=${JSON.stringify(uiReadyFailureState)} diagnostics=${JSON.stringify(uiReadyFailureSnapshot.diagnostics ?? [])} snapshot: ${uiReadyFailureSnapshot.text}`" - assert: expr: "Boolean(uiPageId)" message: control ui page was not available @@ -149,7 +170,7 @@ steps: args: - pageId: ref: uiAckPageId - selector: textarea + selector: openclaw-app timeoutMs: expr: liveTurnTimeoutMs(env, 45000) - try: @@ -240,7 +261,7 @@ steps: args: - pageId: ref: uiImagePageId - selector: textarea + selector: openclaw-app timeoutMs: expr: liveTurnTimeoutMs(env, 45000) - try: diff --git a/ui/src/ui/app-render.ts b/ui/src/ui/app-render.ts index acee2a115c2..c06c5b99f78 100644 --- a/ui/src/ui/app-render.ts +++ b/ui/src/ui/app-render.ts @@ -1,10 +1,5 @@ import { html, nothing } from "lit"; import { applyMergePatch } from "../../../src/config/merge-patch.ts"; -import { - buildAgentMainSessionKey, - parseAgentSessionKey, - resolveAgentIdFromSessionKey, -} from "../../../src/routing/session-key.js"; import { t } from "../i18n/index.ts"; import { getSafeLocalStorage } from "../local-storage.ts"; import { refreshChat } from "./app-chat.ts"; @@ -120,9 +115,14 @@ import { } from "./controllers/skills.ts"; import { buildExternalLinkRel, EXTERNAL_LINK_TARGET } from "./external-link.ts"; import { icons } from "./icons.ts"; -import "./components/dashboard-header.ts"; import { normalizeBasePath, TAB_GROUPS, subtitleForTab, titleForTab } from "./navigation.ts"; +import "./components/dashboard-header.ts"; import { isPluginEnabledInConfigSnapshot } from "./plugin-activation.ts"; +import { + buildAgentMainSessionKey, + parseAgentSessionKey, + resolveAgentIdFromSessionKey, +} from "./session-key.ts"; import { isRenderableControlUiAvatarUrl } from "./views/agents-utils.ts"; import { agentLogoUrl } from "./views/agents-utils.ts"; import { diff --git a/ui/src/ui/app.ts b/ui/src/ui/app.ts index 96fb261e003..3efc2634e25 100644 --- a/ui/src/ui/app.ts +++ b/ui/src/ui/app.ts @@ -1,6 +1,5 @@ import { LitElement } from "lit"; import { customElement, state } from "lit/decorators.js"; -import { resolveAgentIdFromSessionKey } from "../../../src/routing/session-key.js"; import { i18n, I18nController, isSupportedLocale } from "../i18n/index.ts"; import { handleChannelConfigReload as handleChannelConfigReloadInternal, @@ -80,6 +79,7 @@ import type { import { importCustomThemeFromUrl } from "./custom-theme.ts"; import type { GatewayBrowserClient, GatewayHelloOk } from "./gateway.ts"; import type { Tab } from "./navigation.ts"; +import { resolveAgentIdFromSessionKey } from "./session-key.ts"; import type { SidebarContent } from "./sidebar-content.ts"; import { loadLocalUserIdentity, loadSettings, type UiSettings } from "./storage.ts"; import { VALID_THEME_NAMES, type ResolvedTheme, type ThemeMode, type ThemeName } from "./theme.ts"; diff --git a/ui/src/ui/controllers/chat.ts b/ui/src/ui/controllers/chat.ts index 44ff6681149..ad7848bd7d7 100644 --- a/ui/src/ui/controllers/chat.ts +++ b/ui/src/ui/controllers/chat.ts @@ -1,4 +1,3 @@ -import { isHeartbeatOkResponse } from "../../../../src/auto-reply/heartbeat-filter.js"; import { resetToolStream } from "../app-tool-stream.ts"; import { extractText } from "../chat/message-extract.ts"; import { formatConnectError } from "../connect-error.ts"; @@ -12,6 +11,8 @@ import { } from "./scope-errors.ts"; const SILENT_REPLY_PATTERN = /^\s*NO_REPLY\s*$/; +const HEARTBEAT_TOKEN = "HEARTBEAT_OK"; +const DEFAULT_HEARTBEAT_ACK_MAX_CHARS = 300; const SYNTHETIC_TRANSCRIPT_REPAIR_RESULT = "[openclaw] missing tool result in session history; inserted synthetic error result for transcript repair."; const STARTUP_CHAT_HISTORY_RETRY_TIMEOUT_MS = 60_000; @@ -41,6 +42,97 @@ function shouldApplyChatHistoryResult( function isSilentReplyStream(text: string): boolean { return SILENT_REPLY_PATTERN.test(text); } + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function stripHeartbeatTokenForDisplay( + raw: string, + maxAckChars = DEFAULT_HEARTBEAT_ACK_MAX_CHARS, +): { shouldSkip: boolean } { + let text = raw.trim(); + if (!text) { + return { shouldSkip: true }; + } + const strippedMarkup = text + .replace(/<[^>]*>/g, " ") + .replace(/ /gi, " ") + .replace(/^[*`~_]+/, "") + .replace(/[*`~_]+$/, ""); + if (!text.includes(HEARTBEAT_TOKEN) && !strippedMarkup.includes(HEARTBEAT_TOKEN)) { + return { shouldSkip: false }; + } + + const tokenAtEnd = new RegExp(`${escapeRegExp(HEARTBEAT_TOKEN)}[^\\w]{0,4}$`); + let changed = true; + let didStrip = false; + text = strippedMarkup.trim(); + while (changed) { + changed = false; + const next = text.trim(); + if (next.startsWith(HEARTBEAT_TOKEN)) { + text = next.slice(HEARTBEAT_TOKEN.length).trimStart(); + didStrip = true; + changed = true; + continue; + } + if (tokenAtEnd.test(next)) { + const index = next.lastIndexOf(HEARTBEAT_TOKEN); + const before = next.slice(0, index).trimEnd(); + const after = next.slice(index + HEARTBEAT_TOKEN.length).trimStart(); + text = before ? `${before}${after}`.trimEnd() : ""; + didStrip = true; + changed = true; + } + } + + if (!didStrip) { + return { shouldSkip: false }; + } + return { shouldSkip: !text || text.length <= maxAckChars }; +} + +function isHeartbeatOkResponse(message: { role: string; content?: unknown }): boolean { + if (message.role !== "assistant") { + return false; + } + const { text, hasNonTextContent } = resolveMessageText(message.content); + if (hasNonTextContent) { + return false; + } + return stripHeartbeatTokenForDisplay(text).shouldSkip; +} + +function resolveMessageText(content: unknown): { text: string; hasNonTextContent: boolean } { + if (typeof content === "string") { + return { text: content, hasNonTextContent: false }; + } + if (!Array.isArray(content)) { + return { text: "", hasNonTextContent: content != null }; + } + let hasNonTextContent = false; + const text = content + .filter((block): block is { type: "text"; text: string } => { + if (!block || typeof block !== "object" || !("type" in block)) { + hasNonTextContent = true; + return false; + } + if ((block as { type?: unknown }).type !== "text") { + hasNonTextContent = true; + return false; + } + if (typeof (block as { text?: unknown }).text !== "string") { + hasNonTextContent = true; + return false; + } + return true; + }) + .map((block) => block.text) + .join(""); + return { text, hasNonTextContent }; +} + /** Client-side defense-in-depth: detect assistant messages whose text is purely NO_REPLY. */ function isAssistantSilentReply(message: unknown): boolean { if (!message || typeof message !== "object") {