diff --git a/src/gateway/chat-display-projection.ts b/src/gateway/chat-display-projection.ts index 745c8d92318..1328d43a856 100644 --- a/src/gateway/chat-display-projection.ts +++ b/src/gateway/chat-display-projection.ts @@ -580,7 +580,7 @@ function extractAssistantTextForSilentCheck(message: unknown): string | undefine return undefined; } const typed = block as { type?: unknown; text?: unknown }; - if (typed.type !== "text" || typeof typed.text !== "string") { + if (!isAssistantTextContentType(typed.type) || typeof typed.text !== "string") { return undefined; } texts.push(typed.text); @@ -588,6 +588,10 @@ function extractAssistantTextForSilentCheck(message: unknown): string | undefine return texts.length > 0 ? texts.join("\n") : undefined; } +function isAssistantTextContentType(type: unknown): boolean { + return type === "text" || type === "input_text" || type === "output_text"; +} + function hasAssistantNonTextContent(message: unknown): boolean { if (!message || typeof message !== "object") { return false; @@ -597,7 +601,10 @@ function hasAssistantNonTextContent(message: unknown): boolean { return false; } return content.some( - (block) => block && typeof block === "object" && (block as { type?: unknown }).type !== "text", + (block) => + block && + typeof block === "object" && + !isAssistantTextContentType((block as { type?: unknown }).type), ); } @@ -619,7 +626,11 @@ function hasAssistantMixedToolVisibleText(message: unknown): boolean { if (isToolHistoryBlockType(entry.type)) { hasToolHistoryBlock = true; } - if (entry.type === "text" && typeof entry.text === "string" && entry.text.trim()) { + if ( + isAssistantTextContentType(entry.type) && + typeof entry.text === "string" && + entry.text.trim() + ) { hasText = true; } } @@ -1644,7 +1655,7 @@ function projectEmptyAssistantErrorMessages( } const type = (block as { type?: unknown }).type; return ( - type !== "text" && + !isAssistantTextContentType(type) && type !== "thinking" && type !== "reasoning" && type !== "redacted_thinking" @@ -1665,7 +1676,7 @@ function projectEmptyAssistantErrorMessages( continue; } const entry = block as { type?: unknown; text?: unknown }; - if (entry.type === "text" && typeof entry.text === "string") { + if (isAssistantTextContentType(entry.type) && typeof entry.text === "string") { visibleTexts.push(entry.text); } } diff --git a/src/gateway/chat-sanitize.test.ts b/src/gateway/chat-sanitize.test.ts index ebba82afbab..49d9c14dd55 100644 --- a/src/gateway/chat-sanitize.test.ts +++ b/src/gateway/chat-sanitize.test.ts @@ -25,6 +25,39 @@ describe("stripEnvelopeFromMessage", () => { expect(result.content?.[0]?.text).toBe("hi"); }); + test("strips role-appropriate Responses text blocks", () => { + const user = stripEnvelopeFromMessage({ + role: "user", + content: [{ type: "input_text", text: "hello\n[message_id: abc123]" }], + }) as { content?: Array<{ text?: string }> }; + const assistant = stripEnvelopeFromMessage({ + role: "assistant", + content: [ + { + type: "output_text", + text: 'Conversation info (untrusted metadata):\n```json\n{"message_id":"123"}\n```\n\nAssistant body', + }, + ], + }) as { content?: Array<{ text?: string }> }; + + expect(user.content?.[0]?.text).toBe("hello"); + expect(assistant.content?.[0]?.text).toBe("Assistant body"); + }); + + test("strips internal metadata from assistant input_text blocks", () => { + const assistant = stripEnvelopeFromMessage({ + role: "assistant", + content: [ + { + type: "input_text", + text: 'Conversation info (untrusted metadata):\n```json\n{"message_id":"123"}\n```\n\nAssistant body', + }, + ], + }) as { content?: Array<{ text?: string }> }; + + expect(assistant.content?.[0]?.text).toBe("Assistant body"); + }); + test("does not strip inline message_id text that is part of a line", () => { const input = { role: "user", diff --git a/src/gateway/chat-sanitize.ts b/src/gateway/chat-sanitize.ts index 22437ff52a7..628dda8b387 100644 --- a/src/gateway/chat-sanitize.ts +++ b/src/gateway/chat-sanitize.ts @@ -47,15 +47,20 @@ function extractMessageSenderLabel(entry: Record): string | nul // inbound envelopes while assistant/tool content may carry internal metadata. function stripEnvelopeFromContentWithRole( content: unknown[], - stripUserEnvelope: boolean, + role: string, ): { content: unknown[]; changed: boolean } { + const stripUserEnvelope = role === "user"; let changed = false; const next = content.map((item) => { if (!item || typeof item !== "object") { return item; } const entry = item as Record; - if (entry.type !== "text" || typeof entry.text !== "string") { + const isRoleTextBlock = + entry.type === "text" || + (role === "user" && entry.type === "input_text") || + (role === "assistant" && (entry.type === "input_text" || entry.type === "output_text")); + if (!isRoleTextBlock || typeof entry.text !== "string") { return item; } const stripped = stripUserEnvelope @@ -99,7 +104,7 @@ export function stripEnvelopeFromMessage(message: unknown): unknown { changed = true; } } else if (Array.isArray(entry.content)) { - const updated = stripEnvelopeFromContentWithRole(entry.content, stripUserEnvelope); + const updated = stripEnvelopeFromContentWithRole(entry.content, role); if (updated.changed) { next.content = updated.content; changed = true; diff --git a/src/gateway/server-methods/server-methods.test.ts b/src/gateway/server-methods/server-methods.test.ts index 44ba7792da1..f9f51c18ac3 100644 --- a/src/gateway/server-methods/server-methods.test.ts +++ b/src/gateway/server-methods/server-methods.test.ts @@ -941,6 +941,43 @@ describe("projectRecentChatDisplayMessages", () => { ]); }); + it.each([ + ["output_text", ""], + ["output_text", "NO_REPLY"], + ["input_text", ""], + ["input_text", "NO_REPLY"], + ])("projects hidden %s assistant errors %j as a generic safe failure", (type, text) => { + const result = projectRecentChatDisplayMessages([ + { + role: "assistant", + content: [{ type, text }], + stopReason: "error", + errorMessage: "Connection error.", + timestamp: 1, + }, + ]); + + expect(result[0]?.content).toEqual([ + { type: "text", text: "The agent run failed before producing a reply." }, + ]); + }); + + it("preserves visible output_text from a failed assistant turn", () => { + const result = projectRecentChatDisplayMessages([ + { + role: "assistant", + content: [{ type: "output_text", text: "A partial reply before the run failed." }], + stopReason: "error", + errorMessage: "Connection error.", + timestamp: 1, + }, + ]); + + expect(result[0]?.content).toEqual([ + { type: "output_text", text: "A partial reply before the run failed." }, + ]); + }); + it("projects thinking-only assistant errors as a generic safe failure", () => { const result = projectRecentChatDisplayMessages([ { diff --git a/src/gateway/server.chat.gateway-server-chat.test.ts b/src/gateway/server.chat.gateway-server-chat.test.ts index 18a85bc7144..4fd028286a2 100644 --- a/src/gateway/server.chat.gateway-server-chat.test.ts +++ b/src/gateway/server.chat.gateway-server-chat.test.ts @@ -702,6 +702,36 @@ describe("gateway server chat", () => { expect(textValues).toEqual(["hello", "real reply", "real text field reply", "NO_REPLY"]); }); + test("chat.history hides assistant control replies in Responses output blocks", async () => { + const historyMessages = await loadChatHistoryWithMessages([ + { + role: "assistant", + content: [{ type: "output_text", text: "NO_REPLY" }], + timestamp: 1, + }, + { + role: "assistant", + content: [{ type: "output_text", text: "visible response" }], + timestamp: 2, + }, + { + role: "assistant", + content: [{ type: "input_text", text: "NO_REPLY" }], + timestamp: 3, + }, + { + role: "assistant", + content: [{ type: "input_text", text: "visible assistant input" }], + timestamp: 4, + }, + ]); + + expect(collectHistoryTextValues(historyMessages)).toEqual([ + "visible response", + "visible assistant input", + ]); + }); + test("chat.history mirrors current-session message tool sends before NO_REPLY", async () => { const replyText = "Here, love. Eva, not Evo."; const historyMessages = await loadChatHistoryWithMessages([ diff --git a/src/shared/chat-message-content.test.ts b/src/shared/chat-message-content.test.ts index d3498f3ea33..479cecca9aa 100644 --- a/src/shared/chat-message-content.test.ts +++ b/src/shared/chat-message-content.test.ts @@ -138,6 +138,24 @@ describe("extractAssistantVisibleText", () => { ).toBe("Legacy answer"); }); + it("extracts persisted Responses output_text blocks as assistant-visible text", () => { + expect( + extractAssistantVisibleText({ + role: "assistant", + content: [{ type: "output_text", text: "Persisted assistant answer" }], + }), + ).toBe("Persisted assistant answer"); + }); + + it("extracts persisted Responses assistant input_text blocks", () => { + expect( + extractAssistantVisibleText({ + role: "assistant", + content: [{ type: "input_text", text: "Persisted assistant input" }], + }), + ).toBe("Persisted assistant input"); + }); + it("does not mix unphased legacy text into final_answer output", () => { expect( extractAssistantVisibleText({ diff --git a/src/shared/chat-message-content.ts b/src/shared/chat-message-content.ts index 2746601b42f..1d610e8e8fb 100644 --- a/src/shared/chat-message-content.ts +++ b/src/shared/chat-message-content.ts @@ -23,6 +23,10 @@ export function extractFirstTextBlock(message: unknown): string | undefined { export type AssistantPhase = "commentary" | "final_answer"; +function isAssistantTextContentBlockType(value: unknown): boolean { + return value === "text" || value === "input_text" || value === "output_text"; +} + /** Narrows unknown phase metadata to assistant text phases that affect visibility. */ export function normalizeAssistantPhase(value: unknown): AssistantPhase | undefined { return value === "commentary" || value === "final_answer" ? value : undefined; @@ -73,7 +77,7 @@ export function resolveAssistantMessagePhase(message: unknown): AssistantPhase | continue; } const record = block as { type?: unknown; textSignature?: unknown }; - if (record.type !== "text") { + if (!isAssistantTextContentBlockType(record.type)) { continue; } const phase = parseAssistantTextSignature(record.textSignature)?.phase; @@ -156,7 +160,7 @@ export function extractAssistantTextForPhase( return false; } const record = block as { type?: unknown; textSignature?: unknown }; - if (record.type !== "text") { + if (!isAssistantTextContentBlockType(record.type)) { return false; } return Boolean(parseAssistantTextSignature(record.textSignature)?.phase); @@ -173,7 +177,7 @@ export function extractAssistantTextForPhase( return null; } const record = block as { type?: unknown; text?: unknown; textSignature?: unknown }; - if (record.type !== "text" || typeof record.text !== "string") { + if (!isAssistantTextContentBlockType(record.type) || typeof record.text !== "string") { return null; } const signature = parseAssistantTextSignature(record.textSignature); diff --git a/ui/src/ui/chat/message-extract.test.ts b/ui/src/ui/chat/message-extract.test.ts index 8538ffd555a..3615968b3a6 100644 --- a/ui/src/ui/chat/message-extract.test.ts +++ b/ui/src/ui/chat/message-extract.test.ts @@ -44,6 +44,36 @@ describe("extractTextCached", () => { expect(extractTextCached(message)).toBe("Final user answer"); }); + it("extracts text from persisted Responses content blocks", () => { + expect( + extractText({ + role: "user", + content: [{ type: "input_text", text: "Persisted user question" }], + }), + ).toBe("Persisted user question"); + expect( + extractText({ + role: "assistant", + content: [{ type: "output_text", text: "Persisted assistant answer" }], + }), + ).toBe("Persisted assistant answer"); + }); + + it("accepts assistant Responses input blocks but ignores user output blocks", () => { + expect( + extractText({ + role: "user", + content: [{ type: "output_text", text: "Assistant-only block" }], + }), + ).toBeNull(); + expect( + extractText({ + role: "assistant", + content: [{ type: "input_text", text: "User-only block" }], + }), + ).toBe("User-only block"); + }); + it("prefers final_answer assistant text over commentary text", () => { const message = { role: "assistant", diff --git a/ui/src/ui/chat/message-extract.ts b/ui/src/ui/chat/message-extract.ts index 64790371a3a..cef618ea84d 100644 --- a/ui/src/ui/chat/message-extract.ts +++ b/ui/src/ui/chat/message-extract.ts @@ -9,6 +9,14 @@ import { stripThinkingTags } from "../strip-thinking-tags.ts"; const textCache = new WeakMap(); const thinkingCache = new WeakMap(); +function isTextContentBlockType(value: unknown, role: string): boolean { + return ( + value === "text" || + (role === "user" && value === "input_text") || + (role === "assistant" && (value === "input_text" || value === "output_text")) + ); +} + function processMessageText(text: string, role: string): string { const shouldStripInboundMetadata = normalizeLowercaseStringOrEmpty(role) === "user"; const withoutInternalContext = stripInternalRuntimeContext(text); @@ -90,6 +98,7 @@ export function extractThinkingCached(message: unknown): string | null { export function extractRawText(message: unknown): string | null { const m = message as Record; + const role = normalizeLowercaseStringOrEmpty(m.role); const content = m.content; if (typeof content === "string") { return content; @@ -98,7 +107,7 @@ export function extractRawText(message: unknown): string | null { const parts = content .map((p) => { const item = p as Record; - if (item.type === "text" && typeof item.text === "string") { + if (isTextContentBlockType(item.type, role) && typeof item.text === "string") { return item.text; } return null; diff --git a/ui/src/ui/chat/message-normalizer.test.ts b/ui/src/ui/chat/message-normalizer.test.ts index da5f98a5239..be2e841a9f6 100644 --- a/ui/src/ui/chat/message-normalizer.test.ts +++ b/ui/src/ui/chat/message-normalizer.test.ts @@ -90,6 +90,41 @@ describe("message-normalizer", () => { }); }); + it("normalizes persisted Responses text blocks as renderable text", () => { + const user = normalizeMessage({ + role: "user", + content: [{ type: "input_text", text: "Persisted user question" }], + }); + const assistant = normalizeMessage({ + role: "assistant", + content: [{ type: "output_text", text: "Persisted assistant answer" }], + }); + + expect(user.content).toEqual([ + { + type: "text", + text: "Persisted user question", + name: undefined, + args: undefined, + }, + ]); + expect(assistant.content).toEqual([{ type: "text", text: "Persisted assistant answer" }]); + }); + + it("accepts assistant Responses input blocks but rejects user output blocks", () => { + const user = normalizeMessage({ + role: "user", + content: [{ type: "output_text", text: "Assistant-only block" }], + }); + const assistant = normalizeMessage({ + role: "assistant", + content: [{ type: "input_text", text: "User-only block" }], + }); + + expect(user.content).not.toContainEqual({ type: "text", text: "Assistant-only block" }); + expect(assistant.content).toContainEqual({ type: "text", text: "User-only block" }); + }); + it("normalizes structured base64 audio content blocks as renderable attachments", () => { const result = normalizeMessage({ role: "assistant", diff --git a/ui/src/ui/chat/message-normalizer.ts b/ui/src/ui/chat/message-normalizer.ts index 10650de3db3..a19e7315070 100644 --- a/ui/src/ui/chat/message-normalizer.ts +++ b/ui/src/ui/chat/message-normalizer.ts @@ -15,6 +15,18 @@ import { parseInlineDirectives } from "../../../../src/utils/directive-tags.js"; import type { NormalizedMessage, MessageContentItem } from "../types/chat-types.ts"; export { isToolResultMessage, normalizeRoleForGrouping } from "./role-normalizer.ts"; +function isTextContentBlock( + item: Record, + role: string, +): item is Record & { text: string } { + return ( + typeof item.text === "string" && + (item.type === "text" || + (role === "user" && item.type === "input_text") || + (role === "assistant" && (item.type === "input_text" || item.type === "output_text"))) + ); +} + function coerceCanvasPreview( value: unknown, ): @@ -406,15 +418,25 @@ export function normalizeMessage(message: unknown): NormalizedMessage { }, ]; } - if (item.type === "text" && typeof item.text === "string" && isAssistantMessage) { - const expanded = expandTextContent(item.text); - audioAsVoice = audioAsVoice || expanded.audioAsVoice; - if (expanded.replyTarget?.kind === "id") { - replyTarget = expanded.replyTarget; - } else if (expanded.replyTarget?.kind === "current" && replyTarget === null) { - replyTarget = expanded.replyTarget; + if (isTextContentBlock(item, role)) { + if (isAssistantMessage) { + const expanded = expandTextContent(item.text); + audioAsVoice = audioAsVoice || expanded.audioAsVoice; + if (expanded.replyTarget?.kind === "id") { + replyTarget = expanded.replyTarget; + } else if (expanded.replyTarget?.kind === "current" && replyTarget === null) { + replyTarget = expanded.replyTarget; + } + return expanded.content; } - return expanded.content; + return [ + { + type: "text" as const, + text: item.text, + name: undefined, + args: undefined, + }, + ]; } return [ { diff --git a/ui/src/ui/e2e/chat-flow.e2e.test.ts b/ui/src/ui/e2e/chat-flow.e2e.test.ts index 66920fab1a6..00033aa4dfc 100644 --- a/ui/src/ui/e2e/chat-flow.e2e.test.ts +++ b/ui/src/ui/e2e/chat-flow.e2e.test.ts @@ -117,6 +117,25 @@ async function closeOpenBrowserContexts(): Promise { await Promise.all([...openBrowserContexts].map((context) => closeBrowserContext(context))); } +async function visibleChatBubbleTexts(page: Page): Promise { + return page.locator(".chat-thread").evaluate((element) => { + const thread = element as HTMLElement; + const viewport = thread.getBoundingClientRect(); + return Array.from(thread.querySelectorAll(".chat-bubble")) + .filter((candidate) => { + const rect = candidate.getBoundingClientRect(); + return ( + rect.height > 0 && + rect.width > 0 && + rect.bottom > viewport.top && + rect.top < viewport.bottom + ); + }) + .map((candidate) => candidate.textContent?.trim() ?? "") + .filter(Boolean); + }); +} + async function controlUiEventPayloads( page: Page, event: string, @@ -878,6 +897,115 @@ describeControlUiE2e("Control UI mocked Gateway E2E", () => { } }); + it("shows persisted user messages after opening History and scrolling mixed history", async () => { + const context = await newBrowserContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + const baseTs = Date.now() - 100_000; + const currentSessionMessages = [ + { + content: [{ text: "Current session placeholder", type: "text" }], + role: "assistant", + timestamp: baseTs - 1, + }, + ]; + const historyMessages = Array.from({ length: 70 }, (_, index) => ({ + content: [ + { + text: `${index % 2 === 0 ? "User history question" : "Assistant history answer"} ${index}\n${"history detail line\n".repeat(4)}`, + type: index % 2 === 0 ? "input_text" : "output_text", + }, + ], + role: index % 2 === 0 ? "user" : "assistant", + timestamp: baseTs + index, + })); + const gateway = await installMockGateway(page, { + historyMessages: currentSessionMessages, + methodResponses: { + "chat.history": { + cases: [ + { + match: { sessionKey: "agent:main:session-b" }, + response: { + messages: historyMessages, + sessionId: "control-ui-e2e-history-session-b", + thinkingLevel: null, + }, + }, + { + match: { sessionKey: "agent:main:session-a" }, + response: { + messages: currentSessionMessages, + sessionId: "control-ui-e2e-history-session-a", + thinkingLevel: null, + }, + }, + ], + }, + "sessions.list": chatSessionListResponse(), + }, + sessionKey: "agent:main:session-a", + }); + + try { + await page.goto(`${server.baseUrl}chat`); + await page.getByText("Current session placeholder").waitFor({ timeout: 10_000 }); + + await page.getByRole("button", { name: "Chat session" }).click(); + await page.getByRole("option", { name: /Session B/ }).click(); + const historyRequest = await gateway.waitForRequest("chat.history"); + expect(requireRecord(historyRequest.params)).toMatchObject({ + sessionKey: "agent:main:session-b", + }); + await page.locator(".chat-thread").getByText("User history question 68").waitFor({ + timeout: 10_000, + }); + await page.locator(".chat-thread").getByText("Assistant history answer 69").waitFor({ + timeout: 10_000, + }); + await expect + .poll( + async () => { + const texts = await visibleChatBubbleTexts(page); + return ( + texts.some((text) => text.includes("User history question 68")) && + texts.some((text) => text.includes("Assistant history answer 69")) + ); + }, + { timeout: 10_000 }, + ) + .toBe(true); + + await waitForChatScrollIdle(page); + await scrollChatThreadToTop(page); + await page.locator(".chat-thread").getByText("User history question 10").waitFor({ + timeout: 10_000, + }); + await scrollChatThreadToTop(page); + await page.locator(".chat-thread").getByText("User history question 0").waitFor({ + timeout: 10_000, + }); + await scrollChatThreadToTop(page); + await expect + .poll( + async () => { + const texts = await visibleChatBubbleTexts(page); + return ( + texts.some((text) => text.includes("User history question 0")) && + texts.some((text) => text.includes("Assistant history answer 1")) + ); + }, + { timeout: 10_000 }, + ) + .toBe(true); + } finally { + await closeBrowserContext(context); + } + }); + it("keeps rejected pre-ACK sends visible and restores the draft", async () => { const context = await newBrowserContext({ locale: "en-US",