fix: prefer final-answer text in web chat previews

This commit is contained in:
Peter Steinberger
2026-04-05 20:27:16 +01:00
parent e92f55302b
commit a4f16f572c
6 changed files with 268 additions and 2 deletions

View File

@@ -628,6 +628,56 @@ describe("readSessionPreviewItemsFromTranscript", () => {
expect(result).toHaveLength(1);
expect(result[0]?.text).toBe("A B");
});
test("prefers final_answer text for assistant preview items", () => {
const sessionId = "preview-final-answer";
const lines = [
JSON.stringify({
message: {
role: "assistant",
content: [
{
type: "text",
text: "thinking like caveman",
textSignature: JSON.stringify({ v: 1, id: "msg_commentary", phase: "commentary" }),
},
{
type: "text",
text: "Actual final answer",
textSignature: JSON.stringify({ v: 1, id: "msg_final", phase: "final_answer" }),
},
],
},
}),
];
writeTranscriptLines(sessionId, lines);
const result = readPreview(sessionId, 1, 120);
expect(result).toHaveLength(1);
expect(result[0]?.text).toBe("Actual final answer");
});
test("hides commentary-only assistant preview items", () => {
const sessionId = "preview-commentary-only";
const lines = [
JSON.stringify({
message: {
role: "assistant",
content: [
{
type: "text",
text: "thinking like caveman",
textSignature: JSON.stringify({ v: 1, id: "msg_commentary", phase: "commentary" }),
},
],
},
}),
];
writeTranscriptLines(sessionId, lines);
const result = readPreview(sessionId, 1, 120);
expect(result).toHaveLength(0);
});
});
describe("readLatestSessionUsageFromTranscript", () => {

View File

@@ -2,6 +2,7 @@ import fs from "node:fs";
import { deriveSessionTotalTokens, hasNonzeroUsage, normalizeUsage } from "../agents/usage.js";
import { jsonUtf8Bytes } from "../infra/json-utf8-bytes.js";
import { hasInterSessionUserProvenance } from "../sessions/input-provenance.js";
import { extractAssistantVisibleText } from "../shared/chat-message-content.js";
import { stripInlineDirectiveTagsForDisplay } from "../utils/directive-tags.js";
import { extractToolCallNames, hasToolCall } from "../utils/transcript-tools.js";
import { stripEnvelope } from "./chat-sanitize.js";
@@ -630,6 +631,15 @@ function truncatePreviewText(text: string, maxChars: number): string {
}
function extractPreviewText(message: TranscriptPreviewMessage): string | null {
const role = typeof message.role === "string" ? message.role.trim().toLowerCase() : "";
if (role === "assistant") {
const assistantText = extractAssistantVisibleText(message);
if (assistantText) {
const normalized = stripInlineDirectiveTagsForDisplay(assistantText).text.trim();
return normalized ? normalized : null;
}
return null;
}
if (typeof message.content === "string") {
const normalized = stripInlineDirectiveTagsForDisplay(message.content).text.trim();
return normalized ? normalized : null;

View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { extractFirstTextBlock } from "./chat-message-content.js";
import { extractAssistantVisibleText, extractFirstTextBlock } from "./chat-message-content.js";
describe("shared/chat-message-content", () => {
it("extracts the first text block from array content", () => {
@@ -47,3 +47,66 @@ describe("shared/chat-message-content", () => {
expect(extractFirstTextBlock({ content: [{ text: 1 }, { text: "later" }] })).toBeUndefined();
});
});
describe("extractAssistantVisibleText", () => {
it("prefers final_answer text over commentary text", () => {
expect(
extractAssistantVisibleText({
role: "assistant",
content: [
{
type: "text",
text: "thinking like caveman",
textSignature: JSON.stringify({ v: 1, id: "msg_commentary", phase: "commentary" }),
},
{
type: "text",
text: "Actual final answer",
textSignature: JSON.stringify({ v: 1, id: "msg_final", phase: "final_answer" }),
},
],
}),
).toBe("Actual final answer");
});
it("does not fall back to commentary-only text", () => {
expect(
extractAssistantVisibleText({
role: "assistant",
content: [
{
type: "text",
text: "thinking like caveman",
textSignature: JSON.stringify({ v: 1, id: "msg_commentary", phase: "commentary" }),
},
],
}),
).toBeUndefined();
});
it("falls back to unphased legacy text", () => {
expect(
extractAssistantVisibleText({
role: "assistant",
content: [{ type: "text", text: "Legacy answer" }],
}),
).toBe("Legacy answer");
});
it("does not mix unphased legacy text into final_answer output", () => {
expect(
extractAssistantVisibleText({
role: "assistant",
phase: "final_answer",
content: [
{ type: "text", text: "Legacy answer" },
{
type: "text",
text: "Actual final answer",
textSignature: JSON.stringify({ v: 1, id: "msg_final", phase: "final_answer" }),
},
],
}),
).toBe("Actual final answer");
});
});

View File

@@ -16,3 +16,109 @@ export function extractFirstTextBlock(message: unknown): string | undefined {
const text = (first as { text?: unknown }).text;
return typeof text === "string" ? text : undefined;
}
type AssistantPhase = "commentary" | "final_answer";
function normalizeAssistantPhase(value: unknown): AssistantPhase | undefined {
return value === "commentary" || value === "final_answer" ? value : undefined;
}
function parseAssistantTextSignature(
value: unknown,
): { id?: string; phase?: AssistantPhase } | null {
if (typeof value !== "string" || value.trim().length === 0) {
return null;
}
if (!value.startsWith("{")) {
return { id: value };
}
try {
const parsed = JSON.parse(value) as { id?: unknown; phase?: unknown; v?: unknown };
if (parsed.v !== 1) {
return null;
}
return {
...(typeof parsed.id === "string" ? { id: parsed.id } : {}),
...(normalizeAssistantPhase(parsed.phase)
? { phase: normalizeAssistantPhase(parsed.phase) }
: {}),
};
} catch {
return null;
}
}
function extractAssistantTextForPhase(
message: unknown,
phase?: AssistantPhase,
): string | undefined {
if (!message || typeof message !== "object") {
return undefined;
}
const entry = message as { text?: unknown; content?: unknown; phase?: unknown };
const messagePhase = normalizeAssistantPhase(entry.phase);
const shouldIncludeContent = (resolvedPhase?: AssistantPhase) => {
if (phase) {
return resolvedPhase === phase;
}
return resolvedPhase === undefined;
};
if (typeof entry.text === "string") {
const normalized = entry.text.trim();
return shouldIncludeContent(messagePhase) && normalized ? normalized : undefined;
}
if (typeof entry.content === "string") {
const normalized = entry.content.trim();
return shouldIncludeContent(messagePhase) && normalized ? normalized : undefined;
}
if (!Array.isArray(entry.content)) {
return undefined;
}
const hasExplicitPhasedTextBlocks = entry.content.some((block) => {
if (!block || typeof block !== "object") {
return false;
}
const record = block as { type?: unknown; textSignature?: unknown };
if (record.type !== "text") {
return false;
}
return Boolean(parseAssistantTextSignature(record.textSignature)?.phase);
});
const parts = entry.content
.map((block) => {
if (!block || typeof block !== "object") {
return null;
}
const record = block as { type?: unknown; text?: unknown; textSignature?: unknown };
if (record.type !== "text" || typeof record.text !== "string") {
return null;
}
const signature = parseAssistantTextSignature(record.textSignature);
const resolvedPhase =
signature?.phase ?? (hasExplicitPhasedTextBlocks ? undefined : messagePhase);
if (!shouldIncludeContent(resolvedPhase)) {
return null;
}
const normalized = record.text.trim();
return normalized || null;
})
.filter((value): value is string => typeof value === "string");
if (parts.length === 0) {
return undefined;
}
return parts.join("\n");
}
export function extractAssistantVisibleText(message: unknown): string | undefined {
const finalAnswerText = extractAssistantTextForPhase(message, "final_answer");
if (finalAnswerText) {
return finalAnswerText;
}
return extractAssistantTextForPhase(message);
}

View File

@@ -42,6 +42,41 @@ describe("extractTextCached", () => {
expect(extractText(message)).toBe("Final user answer");
expect(extractTextCached(message)).toBe("Final user answer");
});
it("prefers final_answer assistant text over commentary text", () => {
const message = {
role: "assistant",
content: [
{
type: "text",
text: "thinking like caveman",
textSignature: JSON.stringify({ v: 1, id: "msg_commentary", phase: "commentary" }),
},
{
type: "text",
text: "Actual final answer",
textSignature: JSON.stringify({ v: 1, id: "msg_final", phase: "final_answer" }),
},
],
};
expect(extractText(message)).toBe("Actual final answer");
expect(extractTextCached(message)).toBe("Actual final answer");
});
it("does not render commentary-only assistant text", () => {
const message = {
role: "assistant",
content: [
{
type: "text",
text: "thinking like caveman",
textSignature: JSON.stringify({ v: 1, id: "msg_commentary", phase: "commentary" }),
},
],
};
expect(extractText(message)).toBeNull();
expect(extractTextCached(message)).toBeNull();
});
});
describe("extractThinkingCached", () => {

View File

@@ -1,5 +1,6 @@
import { stripInboundMetadata } from "../../../../src/auto-reply/reply/strip-inbound-meta.js";
import { stripEnvelope } from "../../../../src/shared/chat-envelope.js";
import { extractAssistantVisibleText as extractSharedAssistantVisibleText } from "../../../../src/shared/chat-message-content.js";
import { stripThinkingTags } from "../format.ts";
const textCache = new WeakMap<object, string | null>();
@@ -18,7 +19,8 @@ function processMessageText(text: string, role: string): string {
export function extractText(message: unknown): string | null {
const m = message as Record<string, unknown>;
const role = typeof m.role === "string" ? m.role : "";
const raw = extractRawText(message);
const raw =
role === "assistant" ? extractSharedAssistantVisibleText(message) : extractRawText(message);
if (!raw) {
return null;
}