diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index a6d55ad5be11..72372329ca8d 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -129,8 +129,8 @@ c5ff317bb7957d0870cb806e7989c1d48877e3bed583c173ce2f8e9cfe57ee36 module/status- 537047854c21ad20ea0572f8019503cbda3bbafece8194a28c30b0639bde2fde module/string-coerce-runtime 1b5b9a4532db991fce807ba736b550de27aa5f9de2e1f508a8f4f09cae76ea29 module/telegram-account 110944726884fca94f38c9c329b5950629438b9a719f4782c4beeade8bd67746 module/temp-path -16f2d67adc0abed91cb5231dfec298715a43e032edd1e10b10d00a6b0973fcc3 module/text-chunking -1df5a33be5dbf611e301c01c6b8bcf502c3101aae10cd14a79f1bbb58b116497 module/text-runtime +0e230313a7a27a915a7aa399c0afd94c4c3e10d485c7b513e757f41bfc362dbe module/text-chunking +d65c9e55599db94565e1942fe3b595cca0208fabf5ba8b60385a1e60ecd01684 module/text-runtime a94c9ff59cc361b04f8731a47d722c4bc88cfabf942221369a8cb67bdcda7249 module/tool-plugin 9d6ab352913a573b226e054e1dc8c6d088493aea9954950c65585923b5b6895a module/tool-send 541df9dea799f25e83ea483d481ecebc5b91c016effab593c54d3efe3ee6517b module/web-media diff --git a/packages/markdown-core/src/construct-fallbacks.test.ts b/packages/markdown-core/src/construct-fallbacks.test.ts new file mode 100644 index 000000000000..0ec47b7ca8f6 --- /dev/null +++ b/packages/markdown-core/src/construct-fallbacks.test.ts @@ -0,0 +1,295 @@ +import { describe, expect, it } from "vitest"; +import { applyConstructFallbacks } from "./construct-fallbacks.js"; +import type { + ConstructSupport, + FormatCapabilityProfile, + FormatConstruct, +} from "./format-capabilities.js"; +import { markdownToIR, sliceMarkdownIR, type MarkdownIR } from "./ir.js"; + +const ALL_NATIVE = { + mechanism: "markdown", + constructs: { + bold: "native", + italic: "native", + underline: "native", + strikethrough: "native", + spoiler: "native", + codeInline: "native", + codeBlock: "native", + codeLanguage: "native", + linkLabel: "native", + heading: "native", + bulletList: "native", + orderedList: "native", + taskList: "native", + table: "native", + blockquote: "native", + image: "native", + mention: "native", + }, + chunk: { limit: 4_000, unit: "chars" }, +} satisfies FormatCapabilityProfile; + +function withSupport( + construct: FormatConstruct, + support: ConstructSupport, +): FormatCapabilityProfile { + return { + ...ALL_NATIVE, + constructs: { ...ALL_NATIVE.constructs, [construct]: support }, + }; +} + +function markdownToTaskIR(markdown: string): MarkdownIR { + return markdownToIR(markdown, { enableTaskLists: true }); +} + +type FallbackCase = { + name: string; + construct: FormatConstruct; + ir: MarkdownIR; + expected: Record; +}; + +const TASK_LIST_IR = markdownToTaskIR("- [x] done"); + +const CASES: FallbackCase[] = [ + { + name: "heading", + construct: "heading", + ir: { text: "Title", styles: [{ start: 0, end: 5, style: "heading_2" }], links: [] }, + expected: { + native: ["Title", ["heading_2"], 0], + fallback: ["Title", ["bold"], 0], + strip: ["Title", [], 0], + }, + }, + { + name: "labeled link", + construct: "linkLabel", + ir: { text: "docs", styles: [], links: [{ start: 0, end: 4, href: "https://example.com" }] }, + expected: { + native: ["docs", [], 1], + fallback: ["docs (https://example.com)", [], 0], + strip: ["docs", [], 0], + }, + }, + { + name: "spoiler", + construct: "spoiler", + ir: { text: "secret", styles: [{ start: 0, end: 6, style: "spoiler" }], links: [] }, + expected: { + native: ["secret", ["spoiler"], 0], + fallback: ["secret", [], 0], + strip: ["secret", [], 0], + }, + }, + { + name: "task list", + construct: "taskList", + ir: TASK_LIST_IR, + expected: { + native: ["• [x] done", [], 0], + fallback: ["[x] done", [], 0], + strip: ["• done", [], 0], + }, + }, + { + name: "code language", + construct: "codeLanguage", + ir: { + text: "const x = 1;", + styles: [{ start: 0, end: 12, style: "code_block", language: "ts" }], + links: [], + }, + expected: { + native: ["const x = 1;", ["code_block:ts"], 0], + fallback: ["const x = 1;", ["code_block"], 0], + strip: ["const x = 1;", ["code_block"], 0], + }, + }, + { + name: "underline", + construct: "underline", + ir: { text: "under", styles: [{ start: 0, end: 5, style: "underline" }], links: [] }, + expected: { + native: ["under", ["underline"], 0], + fallback: ["under", [], 0], + strip: ["under", [], 0], + }, + }, +]; + +describe("applyConstructFallbacks", () => { + for (const testCase of CASES) { + for (const support of ["native", "fallback", "strip"] as const) { + it(`${testCase.name}: ${support}`, () => { + const actual = applyConstructFallbacks( + testCase.ir, + withSupport(testCase.construct, support), + ); + expect( + [ + actual.text, + actual.styles.map((span) => + span.language ? `${span.style}:${span.language}` : span.style, + ), + actual.links.length, + ], + `${testCase.name}: ${support}`, + ).toEqual(testCase.expected[support]); + }); + } + } + + it("keeps link suffixes outside surrounding styles", () => { + const ir: MarkdownIR = { + text: "see docs now", + styles: [{ start: 0, end: 12, style: "bold" }], + links: [{ start: 4, end: 8, href: "https://example.com" }], + }; + + expect(applyConstructFallbacks(ir, withSupport("linkLabel", "fallback"))).toEqual({ + text: "see docs (https://example.com) now", + styles: [ + { start: 0, end: 8, style: "bold" }, + { start: 30, end: 34, style: "bold" }, + ], + links: [], + }); + }); + + it("merges heading fallback with authored bold and respects stripped bold", () => { + const ir: MarkdownIR = { + text: "Title", + styles: [ + { start: 0, end: 5, style: "heading_1" }, + { start: 0, end: 5, style: "bold" }, + ], + links: [], + }; + expect(applyConstructFallbacks(ir, withSupport("heading", "fallback")).styles).toEqual([ + { start: 0, end: 5, style: "bold" }, + ]); + expect( + applyConstructFallbacks(ir, { + ...withSupport("heading", "fallback"), + constructs: { + ...ALL_NATIVE.constructs, + heading: "fallback", + bold: "strip", + }, + }).styles, + ).toEqual([]); + }); + + it("does not infer list constructs from literal code or escaped markers", () => { + const ir = markdownToTaskIR( + "```\n• [x] command\n1. step\n```\n\n- \\[x] literal\n\n-\n [x] indented code", + ); + const profile = { + ...withSupport("taskList", "fallback"), + constructs: { + ...ALL_NATIVE.constructs, + taskList: "fallback", + orderedList: "strip", + }, + } satisfies FormatCapabilityProfile; + + expect(applyConstructFallbacks(ir, profile).text).toBe( + "• [x] command\n1. step\n\n• [x] literal\n\n• [x] indented code\n", + ); + }); + + it("preserves task-list provenance through IR slicing", () => { + const ir = markdownToTaskIR("intro\n\n- [x] done"); + const task = sliceMarkdownIR(ir, ir.text.indexOf("•"), ir.text.length); + expect(applyConstructFallbacks(task, withSupport("taskList", "fallback")).text).toBe( + "[x] done", + ); + const markerOnly = sliceMarkdownIR(ir, ir.text.indexOf("[x]"), ir.text.length); + expect(applyConstructFallbacks(markerOnly, withSupport("taskList", "strip")).text).toBe("done"); + }); + + it("recognizes task items nested inside blockquotes", () => { + for (const markdown of ["> - [x] done", "- > - [x] done"]) { + const ir = markdownToTaskIR(markdown); + expect( + applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text, + markdown, + ).toContain("[x] done"); + expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).not.toContain( + "[x]", + ); + } + }); + + it("recognizes task items nested on the same source line", () => { + const ir = markdownToTaskIR("- - [x] done"); + expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).toBe("• \n • done"); + }); + + it("recognizes multiline task-list markers", () => { + const cases = [ + { markdown: "-\n [x] done", expected: "• done" }, + { markdown: "- [x]\n done", expected: "• \ndone" }, + { markdown: "> -\n> [x] done", expected: "• done" }, + ]; + for (const { markdown, expected } of cases) { + const ir = markdownToTaskIR(markdown); + expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text, markdown).toBe( + expected, + ); + } + }); + + it("does not consume list separators when stripping an empty task", () => { + const ir = markdownToTaskIR("- [x] \n- next"); + expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).toBe("• \n• next"); + }); + + it("preserves task markers that collide with shortcut reference links", () => { + const ir = markdownToTaskIR("- [x] done\n\n[x]: https://example.com"); + expect(applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text).toBe("[x] done"); + expect(ir.links).toEqual([]); + }); + + it("does not protect checkbox-like links after an earlier list-item block", () => { + const markdown = "- ---\n\n [x] done\n\n[x]: https://example.com"; + expect(markdownToIR(markdown, { enableTaskLists: true })).toEqual(markdownToIR(markdown)); + }); + + it("preserves list provenance through structural clones", () => { + const ir = structuredClone(markdownToTaskIR("- [x] done")); + expect(applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text).toBe("[x] done"); + }); + + it("clips task-list provenance at partial marker slice boundaries", () => { + const ir = markdownToTaskIR("- [x] done"); + const prefixOnly = sliceMarkdownIR(ir, 0, 2); + expect(applyConstructFallbacks(prefixOnly, withSupport("taskList", "fallback")).text).toBe(""); + const insideListMarker = sliceMarkdownIR(ir, 1, ir.text.length); + const stripListAndTask = { + ...withSupport("taskList", "strip"), + constructs: { ...ALL_NATIVE.constructs, taskList: "strip", bulletList: "strip" }, + } satisfies FormatCapabilityProfile; + expect(applyConstructFallbacks(insideListMarker, stripListAndTask).text).toBe("done"); + for (const start of [3, 4]) { + const sliced = sliceMarkdownIR(ir, start, ir.text.length); + const stripped = applyConstructFallbacks(sliced, withSupport("taskList", "strip")); + expect(stripped.text, `start=${start}`).toBe("done"); + } + }); + + it("clips trailing empty list-marker provenance during finalization", () => { + const cases = [ + { markdown: "-", construct: "bulletList" as const }, + { markdown: "1.", construct: "orderedList" as const }, + ]; + for (const { markdown, construct } of cases) { + const ir = markdownToIR(markdown); + expect(applyConstructFallbacks(ir, withSupport(construct, "strip")).text, markdown).toBe(""); + } + }); +}); diff --git a/packages/markdown-core/src/construct-fallbacks.ts b/packages/markdown-core/src/construct-fallbacks.ts new file mode 100644 index 000000000000..66bef96bd1f9 --- /dev/null +++ b/packages/markdown-core/src/construct-fallbacks.ts @@ -0,0 +1,174 @@ +import type { FormatCapabilityProfile, FormatConstruct } from "./format-capabilities.js"; +import { + copyMarkdownLinkSpan, + createStyleSpan, + mergeAnnotationSpans, + mergeStyleSpans, + type MarkdownLinkSpan, + type MarkdownStyle, + type MarkdownStyleSpan, +} from "./ir-spans.js"; +import { sliceMarkdownIR, type MarkdownIR } from "./ir.js"; + +type TextEdit = { start: number; end: number; text: string }; + +const STYLE_CONSTRUCTS: Partial> = { + bold: "bold", + italic: "italic", + underline: "underline", + strikethrough: "strikethrough", + spoiler: "spoiler", + code: "codeInline", + code_block: "codeBlock", + blockquote: "blockquote", +}; + +function isHeading(style: MarkdownStyle): boolean { + return style.startsWith("heading_"); +} + +function projectStyles( + styles: MarkdownStyleSpan[], + profile: FormatCapabilityProfile, +): MarkdownStyleSpan[] { + const projected: MarkdownStyleSpan[] = []; + let synthesizedHeading = false; + for (const span of styles) { + if (isHeading(span.style)) { + if (profile.constructs.heading === "native") { + projected.push(span); + } else if ( + profile.constructs.heading === "fallback" && + profile.constructs.bold === "native" + ) { + projected.push(createStyleSpan({ ...span, style: "bold" })); + synthesizedHeading = true; + } + continue; + } + const construct = STYLE_CONSTRUCTS[span.style]; + if (construct && profile.constructs[construct] !== "native") { + continue; + } + if (span.style === "code_block" && profile.constructs.codeLanguage !== "native") { + projected.push(createStyleSpan({ start: span.start, end: span.end, style: span.style })); + } else { + projected.push(span); + } + } + return synthesizedHeading ? mergeStyleSpans(projected) : projected; +} + +function collectLinkFallbacks( + ir: MarkdownIR, + profile: FormatCapabilityProfile, +): { links: MarkdownLinkSpan[]; edits: TextEdit[] } { + if (profile.constructs.linkLabel === "native") { + return { links: ir.links, edits: [] }; + } + if (profile.constructs.linkLabel === "strip") { + return { links: [], edits: [] }; + } + return { + links: [], + edits: ir.links.flatMap((link) => { + const href = link.href.trim(); + const label = ir.text.slice(link.start, link.end).trim(); + const comparableHref = href.startsWith("mailto:") ? href.slice("mailto:".length) : href; + return href && label && label !== href && label !== comparableHref + ? [{ start: link.end, end: link.end, text: ` (${href})` }] + : []; + }), + }; +} + +function collectListFallbacks(ir: MarkdownIR, profile: FormatCapabilityProfile): TextEdit[] { + const edits: TextEdit[] = []; + for (const item of ir.listItems ?? []) { + if (item.task) { + if (profile.constructs.taskList === "fallback" && item.listMarker) { + edits.push({ ...item.listMarker, text: "" }); + } else if (profile.constructs.taskList === "strip" && item.taskMarker) { + edits.push({ ...item.taskMarker, text: "" }); + } + } + if (item.listMarker && item.kind === "bullet" && profile.constructs.bulletList === "strip") { + edits.push({ ...item.listMarker, text: "" }); + } + if (item.listMarker && item.kind === "ordered" && profile.constructs.orderedList === "strip") { + edits.push({ ...item.listMarker, text: "" }); + } + } + return edits; +} + +function appendSlice(target: MarkdownIR, source: MarkdownIR): void { + const offset = target.text.length; + target.text += source.text; + target.styles.push( + ...source.styles.map((span) => + createStyleSpan({ + ...span, + start: offset + span.start, + end: offset + span.end, + }), + ), + ); + target.links.push( + ...source.links.map((link) => + copyMarkdownLinkSpan(link, { + start: offset + link.start, + end: offset + link.end, + }), + ), + ); + const annotations = source.annotations?.map((annotation) => ({ + ...annotation, + start: offset + annotation.start, + end: offset + annotation.end, + })); + if (annotations?.length) { + (target.annotations ??= []).push(...annotations); + } +} + +function applyTextEdits(ir: MarkdownIR, edits: TextEdit[]): MarkdownIR { + if (edits.length === 0) { + return ir; + } + const ordered = edits + .toSorted((a, b) => a.start - b.start || a.end - b.end) + .filter((edit, index, all) => { + const previous = all[index - 1]; + return !previous || edit.start !== previous.start || edit.end !== previous.end; + }); + const result: MarkdownIR = { text: "", styles: [], links: [] }; + let cursor = 0; + for (const edit of ordered) { + appendSlice(result, sliceMarkdownIR(ir, cursor, edit.start)); + result.text += edit.text; + cursor = edit.end; + } + appendSlice(result, sliceMarkdownIR(ir, cursor, ir.text.length)); + result.styles = mergeStyleSpans(result.styles); + if (result.annotations) { + result.annotations = mergeAnnotationSpans(result.annotations); + } + return result; +} + +/** Applies target-declared semantic fallbacks before a mechanism-specific renderer runs. */ +export function applyConstructFallbacks( + ir: MarkdownIR, + profile: FormatCapabilityProfile, +): MarkdownIR { + // Tables stay in convertMarkdownTables; images and mentions are already plain + // text in this flat IR, so those constructs have no shared fallback work here. + const styled: MarkdownIR = { + ...ir, + styles: projectStyles(ir.styles, profile), + }; + const listProjected = applyTextEdits(styled, collectListFallbacks(styled, profile)); + const linkProjection = collectLinkFallbacks(listProjected, profile); + return applyTextEdits({ ...listProjected, links: linkProjection.links }, linkProjection.edits); +} diff --git a/packages/markdown-core/src/format-capabilities.ts b/packages/markdown-core/src/format-capabilities.ts new file mode 100644 index 000000000000..59829167e38e --- /dev/null +++ b/packages/markdown-core/src/format-capabilities.ts @@ -0,0 +1,31 @@ +export type FormatConstruct = + | "bold" + | "italic" + | "underline" + | "strikethrough" + | "spoiler" + | "codeInline" + | "codeBlock" + | "codeLanguage" + | "linkLabel" + | "heading" + | "bulletList" + | "orderedList" + | "taskList" + | "table" + | "blockquote" + | "image" + | "mention"; + +export type ConstructSupport = "native" | "fallback" | "strip"; + +/** Static formatting capabilities declared by an outbound channel. */ +export type FormatCapabilityProfile = { + mechanism: "markdown" | "html" | "ranges" | "blocks" | "plain"; + constructs: Record; + chunk: { + limit: number; + unit: "chars" | "utf16" | "bytes"; + hardCap?: number; + }; +}; diff --git a/packages/markdown-core/src/index.ts b/packages/markdown-core/src/index.ts index 00478e6a4ce4..205745e6bab1 100644 --- a/packages/markdown-core/src/index.ts +++ b/packages/markdown-core/src/index.ts @@ -3,10 +3,13 @@ export * from "./chunk-text.js"; export * from "./code-spans.js"; export * from "./fences.js"; export * from "./frontmatter.js"; +export * from "./format-capabilities.js"; +export * from "./construct-fallbacks.js"; export * from "./html-tags.js"; export * from "./ir.js"; export * from "./render-aware-chunking.js"; export * from "./render-attributed.js"; +export * from "./render-plain.js"; export * from "./render.js"; export * from "./tables.js"; export * from "./types.js"; diff --git a/packages/markdown-core/src/ir-spans.ts b/packages/markdown-core/src/ir-spans.ts index 38d8b31a6a1d..9dd758a4173d 100644 --- a/packages/markdown-core/src/ir-spans.ts +++ b/packages/markdown-core/src/ir-spans.ts @@ -6,6 +6,7 @@ import type { export type MarkdownStyle = | "bold" | "italic" + | "underline" | "strikethrough" | "code" | "code_block" diff --git a/packages/markdown-core/src/ir.ts b/packages/markdown-core/src/ir.ts index b57c5bcc1dd7..e90c096507b2 100644 --- a/packages/markdown-core/src/ir.ts +++ b/packages/markdown-core/src/ir.ts @@ -1,6 +1,9 @@ // Markdown Core module implements ir behavior. import MarkdownIt from "markdown-it"; import markdownItCjkFriendly from "markdown-it-cjk-friendly"; +import { HTML_TAG_RE } from "markdown-it/lib/common/html_re.mjs"; +import type StateCore from "markdown-it/lib/rules_core/state_core.mjs"; +import type StateInline from "markdown-it/lib/rules_inline/state_inline.mjs"; import { visibleWidth } from "../../terminal-core/src/ansi.js"; import { ASSISTANT_TRANSCRIPT_ROLE_NODE_TYPE, @@ -9,6 +12,7 @@ import { type AssistantTranscriptRoleTokenMeta, } from "./assistant-transcript.js"; import { chunkText } from "./chunk-text.js"; +import { tokenizeHtmlTags } from "./html-tags.js"; import { appendAssistantTranscriptRoleImage, appendAssistantTranscriptRoleText, @@ -68,6 +72,14 @@ type MarkdownToken = { map?: [number, number] | null; markup?: string; meta?: unknown; + taskListMarker?: boolean; +}; + +type MarkdownListItemMarker = { + kind: "bullet" | "ordered"; + listMarker?: { start: number; end: number }; + task?: true; + taskMarker?: { start: number; end: number }; }; export type MarkdownIR = { @@ -75,6 +87,7 @@ export type MarkdownIR = { styles: MarkdownStyleSpan[]; links: MarkdownLinkSpan[]; annotations?: MarkdownAnnotationSpan[]; + listItems?: MarkdownListItemMarker[]; }; type MarkdownTableAlignment = "left" | "center" | "right"; @@ -135,6 +148,8 @@ type RenderState = RenderTarget & { horizontalRuleText: string; preserveSourceBlockSpacing: boolean; headingLineEnd: number | undefined; + listItems: MarkdownListItemMarker[]; + openListItems: MarkdownListItemMarker[]; }; export type MarkdownParseOptions = { @@ -142,6 +157,10 @@ export type MarkdownParseOptions = { assistantTranscriptRoleHeaders?: boolean; linkify?: boolean; enableSpoilers?: boolean; + /** Parse authored HTML / tags into underline spans. */ + enableHtmlUnderline?: boolean; + /** Preserve task-list checkboxes as semantic list markers. */ + enableTaskLists?: boolean; headingStyle?: "none" | "bold" | "rich"; blockquotePrefix?: string; autolink?: boolean; @@ -178,6 +197,12 @@ function createMarkdownIt(options: MarkdownParseOptions): MarkdownIt { }); md.use(markdownItCjkFriendly); md.use(markdownItAssistantTranscriptRoles); + if (options.enableTaskLists) { + md.core.ruler.before("inline", "markdown_core_task_lists", protectTaskListMarkers); + } + if (options.enableHtmlUnderline) { + md.inline.ruler.before("html_inline", "markdown_core_html_underline", parseHtmlUnderline); + } if (options.enableSpoilers) { // Spoiler delimiters can surround a line-leading role header. Normalize // them before semantic detection so later rendering cannot expose a role @@ -198,6 +223,64 @@ function createMarkdownIt(options: MarkdownParseOptions): MarkdownIt { return md; } +function protectTaskListMarkers(state: StateCore): void { + const stack: Array<{ contentStarted: boolean }> = []; + for (const token of state.tokens as MarkdownToken[]) { + if (token.type === "list_item_open") { + stack.push({ contentStarted: false }); + continue; + } + if (token.type === "list_item_close") { + stack.pop(); + continue; + } + const item = stack.at(-1); + if (!item || item.contentStarted) { + continue; + } + if (token.type === "inline") { + item.contentStarted = true; + if (/^\[[ xX]\](?:[ \t]|\n|$)/u.test(token.content ?? "")) { + token.taskListMarker = true; + token.content = `\\${token.content}`; + } + continue; + } + if (token.type !== "paragraph_open") { + item.contentStarted = true; + } + } +} + +function parseHtmlUnderline(state: StateInline, silent: boolean): boolean { + if (state.src.charCodeAt(state.pos) !== 0x3c) { + return false; + } + const raw = HTML_TAG_RE.exec(state.src.slice(state.pos))?.[0]; + if (!raw) { + return false; + } + const tag = tokenizeHtmlTags(raw).next().value; + const underlineTag = + tag && tag.start === 0 && (tag.name === "u" || tag.name === "ins") ? tag : undefined; + if (!silent) { + const token = state.push( + !underlineTag || underlineTag.selfClosing + ? "text" + : underlineTag.closing + ? "underline_close" + : "underline_open", + "", + 0, + ); + if (!underlineTag || underlineTag.selfClosing) { + token.content = raw; + } + } + state.pos += raw.length; + return true; +} + function getAttr(token: MarkdownToken, name: string): string | null { if (token.attrGet) { return token.attrGet(name); @@ -387,16 +470,35 @@ function appendNestedListSeparator(state: RenderState) { } } -function appendListPrefix(state: RenderState) { +function appendListPrefix(state: RenderState, isTask: boolean): MarkdownListItemMarker | undefined { const stack = state.env.listStack; const top = stack[stack.length - 1]; if (!top) { - return; + return undefined; } top.index += 1; const indent = " ".repeat(Math.max(0, stack.length - 1)); const prefix = top.type === "ordered" ? `${top.index}. ` : "• "; - state.text += `${indent}${prefix}`; + state.text += indent; + const start = state.text.length; + state.text += prefix; + return { + kind: top.type, + listMarker: { start, end: state.text.length }, + ...(isTask ? { task: true as const } : {}), + }; +} + +function recordTaskMarker(state: RenderState, content: string): void { + const item = state.openListItems.at(-1); + if (!item?.task || item.taskMarker) { + return; + } + const marker = /^\[[ xX]\][ \t]?/u.exec(content)?.[0]; + if (marker) { + const start = resolveRenderTarget(state).text.length; + item.taskMarker = { start, end: start + marker.length }; + } } function renderInlineCode(state: RenderState, content: string) { @@ -770,8 +872,15 @@ function renderTokens(tokens: MarkdownToken[], state: RenderState): void { } break; case "text": + recordTaskMarker(state, token.content ?? ""); appendText(state, token.content ?? ""); break; + case "underline_open": + openStyle(state, "underline"); + break; + case "underline_close": + closeStyle(state, "underline"); + break; case ASSISTANT_TRANSCRIPT_ROLE_NODE_TYPE: { const meta = (token.meta as AssistantTranscriptRoleTokenMeta | undefined) ?.assistantTranscriptRoleHeader; @@ -906,15 +1015,32 @@ function renderTokens(tokens: MarkdownToken[], state: RenderState): void { appendTopLevelListSeparator(state); } break; - case "list_item_open": - appendListPrefix(state); + case "list_item_open": { + const leadingInline = + tokens[tokenIndex + 1]?.type === "paragraph_open" ? tokens[tokenIndex + 2] : undefined; + const isTask = leadingInline?.type === "inline" && leadingInline.taskListMarker === true; + const item = appendListPrefix(state, isTask); + if (item) { + state.openListItems.push(item); + } break; - case "list_item_close": + } + case "list_item_close": { + const item = state.openListItems.pop(); + if (item) { + state.listItems.push({ + kind: item.kind, + ...(item.listMarker ? { listMarker: item.listMarker } : {}), + ...(item.task ? { task: true } : {}), + ...(item.taskMarker ? { taskMarker: item.taskMarker } : {}), + }); + } // Avoid double newlines (nested list's last item already added newline) if (!state.text.endsWith("\n")) { state.text += "\n"; } break; + } case "code_block": case "fence": renderCodeBlock( @@ -1026,14 +1152,40 @@ function closeRemainingStyles(target: RenderTarget) { target.openStyles = []; } +function sliceListMarker( + marker: { start: number; end: number }, + start: number, + end: number, +): { start: number; end: number } | undefined { + const sliceStart = Math.max(marker.start, start); + const sliceEnd = Math.min(marker.end, end); + return sliceEnd > sliceStart ? { start: sliceStart - start, end: sliceEnd - start } : undefined; +} + export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): MarkdownIR { const annotations = sliceAnnotationSpans(ir.annotations ?? [], start, end); - return { + const listItems = (ir.listItems ?? []).flatMap((item) => { + const listMarker = item.listMarker ? sliceListMarker(item.listMarker, start, end) : undefined; + const taskMarker = item.taskMarker ? sliceListMarker(item.taskMarker, start, end) : undefined; + return listMarker || taskMarker + ? [ + { + kind: item.kind, + ...(listMarker ? { listMarker } : {}), + ...(item.task ? { task: true as const } : {}), + ...(taskMarker ? { taskMarker } : {}), + }, + ] + : []; + }); + const sliced: MarkdownIR = { text: ir.text.slice(start, end), styles: sliceStyleSpans(ir.styles, start, end), links: sliceLinkSpans(ir.links, start, end), ...(annotations.length > 0 ? { annotations } : {}), + ...(listItems.length > 0 ? { listItems } : {}), }; + return sliced; } export function markdownToIR(markdown: string, options: MarkdownParseOptions = {}): MarkdownIR { @@ -1072,6 +1224,8 @@ export function markdownToIRWithMeta( horizontalRuleText: options.horizontalRuleText ?? "───", preserveSourceBlockSpacing: options.preserveSourceBlockSpacing ?? false, headingLineEnd: undefined, + listItems: [], + openListItems: [], }; renderTokens(tokens as MarkdownToken[], state); @@ -1092,14 +1246,34 @@ export function markdownToIRWithMeta( const finalText = finalLength === state.text.length ? state.text : state.text.slice(0, finalLength); const annotations = mergeAnnotationSpans(clampAnnotationSpans(state.annotations, finalLength)); + const listItems = state.listItems.flatMap((item) => { + const listMarker = item.listMarker + ? sliceListMarker(item.listMarker, 0, finalLength) + : undefined; + const taskMarker = item.taskMarker + ? sliceListMarker(item.taskMarker, 0, finalLength) + : undefined; + return listMarker || taskMarker + ? [ + { + kind: item.kind, + ...(listMarker ? { listMarker } : {}), + ...(item.task ? { task: true as const } : {}), + ...(taskMarker ? { taskMarker } : {}), + }, + ] + : []; + }); + const ir: MarkdownIR = { + text: finalText, + styles: mergeStyleSpans(clampStyleSpans(state.styles, finalLength)), + links: clampLinkSpans(state.links, finalLength), + ...(annotations.length > 0 ? { annotations } : {}), + ...(listItems.length > 0 ? { listItems } : {}), + }; return { - ir: { - text: finalText, - styles: mergeStyleSpans(clampStyleSpans(state.styles, finalLength)), - links: clampLinkSpans(state.links, finalLength), - ...(annotations.length > 0 ? { annotations } : {}), - }, + ir, hasTables: state.hasTables, tables: state.collectedTables.map((table) => Object.assign({}, table, { @@ -1132,13 +1306,9 @@ export function chunkMarkdownIR(ir: MarkdownIR, limit: number): MarkdownIR[] { } const start = cursor; const end = Math.min(ir.text.length, start + chunk.length); - const annotations = sliceAnnotationSpans(ir.annotations ?? [], start, end); - results.push({ - text: chunk, - styles: sliceStyleSpans(ir.styles, start, end), - links: sliceLinkSpans(ir.links, start, end), - ...(annotations.length > 0 ? { annotations } : {}), - }); + const sliced = sliceMarkdownIR(ir, start, end); + sliced.text = chunk; + results.push(sliced); cursor = end; }); diff --git a/packages/markdown-core/src/ir.underline.test.ts b/packages/markdown-core/src/ir.underline.test.ts new file mode 100644 index 000000000000..18245ceab1b6 --- /dev/null +++ b/packages/markdown-core/src/ir.underline.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { markdownToIR } from "./ir.js"; + +describe("markdownToIR authored HTML underline", () => { + it("emits underline spans only for authored u and ins tags when enabled", () => { + const ir = markdownToIR("under nested and __bold__", { + enableHtmlUnderline: true, + }); + + expect(ir.text).toBe("under nested and bold"); + expect(ir.styles).toEqual([ + { start: 0, end: 12, style: "underline" }, + { start: 17, end: 21, style: "bold" }, + ]); + }); + + it("preserves raw HTML bytes unless underline parsing is explicitly enabled", () => { + expect(markdownToIR("under").text).toBe("under"); + }); + + it("does not enable HTML block parsing for unrelated tags", () => { + const ir = markdownToIR("
\n**bold**\n
", { enableHtmlUnderline: true }); + expect(ir.text).toBe("
\nbold\n
"); + expect(ir.styles).toEqual([{ start: 6, end: 10, style: "bold" }]); + }); + + it("keeps entity-decoded and escaped underline tags literal", () => { + for (const input of ["<u>text</u>", "\\text\\"]) { + const ir = markdownToIR(input, { enableHtmlUnderline: true }); + expect(ir.text, input).toBe("text"); + expect(ir.styles, input).toEqual([]); + } + }); + + it("does not parse underline-shaped text inside other HTML lexemes", () => { + for (const input of ['x', "x"]) { + const ir = markdownToIR(input, { enableHtmlUnderline: true }); + expect(ir.text, input).toBe(input); + expect(ir.styles, input).toEqual([]); + } + }); +}); diff --git a/packages/markdown-core/src/render-attributed.ts b/packages/markdown-core/src/render-attributed.ts index e16a3b2484ad..1b6c005c4cc7 100644 --- a/packages/markdown-core/src/render-attributed.ts +++ b/packages/markdown-core/src/render-attributed.ts @@ -1,3 +1,5 @@ +import { applyConstructFallbacks } from "./construct-fallbacks.js"; +import type { FormatCapabilityProfile } from "./format-capabilities.js"; import { isAutoLinkedMarkdownLink, type MarkdownAnnotationSpan } from "./ir-spans.js"; import type { MarkdownIR, MarkdownLinkSpan, MarkdownStyle } from "./ir.js"; @@ -24,14 +26,16 @@ export type AttributedRenderOptions = { export function renderMarkdownWithAttributedRanges( ir: MarkdownIR, options: AttributedRenderOptions, + profile?: FormatCapabilityProfile, ): { text: string; ranges: AttributedRange[] } { - const text = ir.text ?? ""; + const projected = profile ? applyConstructFallbacks(ir, profile) : ir; + const text = projected.text ?? ""; const insertions: Array<{ pos: number; length: number }> = []; let rendered = text; if (options.renderLink) { rendered = ""; let cursor = 0; - for (const link of [...ir.links].toSorted((a, b) => a.start - b.start)) { + for (const link of [...projected.links].toSorted((a, b) => a.start - b.start)) { if (link.start < cursor) { continue; } @@ -48,11 +52,11 @@ export function renderMarkdownWithAttributedRanges( } rendered = options.trimEnd ? rendered.trimEnd() : rendered; - const spans = ir.styles.flatMap((span) => { + const spans = projected.styles.flatMap((span) => { const style = options.styleMap[span.style]; return style === undefined ? [] : [{ start: span.start, end: span.end, style }]; }); - for (const annotation of ir.annotations ?? []) { + for (const annotation of projected.annotations ?? []) { const style = options.annotationStyleMap?.[annotation.type]; if (style !== undefined) { spans.push({ start: annotation.start, end: annotation.end, style }); diff --git a/packages/markdown-core/src/render-plain.ts b/packages/markdown-core/src/render-plain.ts new file mode 100644 index 000000000000..6b26304439d1 --- /dev/null +++ b/packages/markdown-core/src/render-plain.ts @@ -0,0 +1,39 @@ +import { applyConstructFallbacks } from "./construct-fallbacks.js"; +import type { FormatCapabilityProfile } from "./format-capabilities.js"; +import type { MarkdownIR } from "./ir.js"; + +export type PlainRenderOptions = { + linkStyle?: "label" | "label-and-url"; +}; + +/** Projects Markdown IR to plain text, optionally applying channel capability fallbacks. */ +export function renderMarkdownAsPlainText( + ir: MarkdownIR, + options: PlainRenderOptions = {}, + profile?: FormatCapabilityProfile, +): string { + const effectiveProfile = + profile && options.linkStyle === "label" + ? { ...profile, constructs: { ...profile.constructs, linkLabel: "strip" as const } } + : profile; + const projected = effectiveProfile ? applyConstructFallbacks(ir, effectiveProfile) : ir; + if ((options.linkStyle ?? "label-and-url") === "label" || projected.links.length === 0) { + return projected.text; + } + let output = ""; + let cursor = 0; + for (const link of [...projected.links].toSorted((a, b) => a.start - b.start)) { + if (link.start < cursor) { + continue; + } + output += projected.text.slice(cursor, link.end); + const href = link.href.trim(); + const label = projected.text.slice(link.start, link.end).trim(); + const comparableHref = href.startsWith("mailto:") ? href.slice("mailto:".length) : href; + if (href && label && label !== href && label !== comparableHref) { + output += ` (${href})`; + } + cursor = link.end; + } + return output + projected.text.slice(cursor); +} diff --git a/packages/markdown-core/src/render.capabilities.test.ts b/packages/markdown-core/src/render.capabilities.test.ts new file mode 100644 index 000000000000..8e7dd90d7f69 --- /dev/null +++ b/packages/markdown-core/src/render.capabilities.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import type { FormatCapabilityProfile } from "./format-capabilities.js"; +import { markdownToIR } from "./ir.js"; +import { renderMarkdownWithAttributedRanges } from "./render-attributed.js"; +import { renderMarkdownAsPlainText } from "./render-plain.js"; +import { renderMarkdownWithMarkers } from "./render.js"; + +const ALL_NATIVE = { + mechanism: "markdown", + constructs: { + bold: "native", + italic: "native", + underline: "native", + strikethrough: "native", + spoiler: "native", + codeInline: "native", + codeBlock: "native", + codeLanguage: "native", + linkLabel: "native", + heading: "native", + bulletList: "native", + orderedList: "native", + taskList: "native", + table: "native", + blockquote: "native", + image: "native", + mention: "native", + }, + chunk: { limit: 4_000, unit: "chars" }, +} satisfies FormatCapabilityProfile; + +describe("format capability driver plumbing", () => { + const ir = markdownToIR("**See [docs](https://example.com)**", { headingStyle: "rich" }); + + it("keeps marker rendering byte-identical for an all-native optional profile", () => { + const options = { + styleMarkers: { bold: { open: "", close: "" } }, + escapeText: (text: string) => text, + buildLink: (link: { start: number; end: number; href: string }) => ({ + start: link.start, + end: link.end, + open: "", + close: "", + }), + }; + expect(renderMarkdownWithMarkers(ir, options, ALL_NATIVE)).toBe( + renderMarkdownWithMarkers(ir, options), + ); + }); + + it("preserves caller-constructed style spans for an all-native profile", () => { + const customIr = { + text: "same", + styles: [ + { start: 0, end: 2, style: "bold" as const }, + { start: 2, end: 4, style: "bold" as const }, + ], + links: [], + }; + const options = { + styleMarkers: { bold: { open: "*", close: "*" } }, + escapeText: (text: string) => text, + }; + expect(renderMarkdownWithMarkers(customIr, options, ALL_NATIVE)).toBe( + renderMarkdownWithMarkers(customIr, options), + ); + }); + + it("keeps attributed rendering byte-identical for an all-native optional profile", () => { + const options = { styleMap: { bold: "strong" as const }, renderLink: () => " (url)" }; + expect(renderMarkdownWithAttributedRanges(ir, options, ALL_NATIVE)).toEqual( + renderMarkdownWithAttributedRanges(ir, options), + ); + }); + + it("keeps plain projection byte-identical for an all-native optional profile", () => { + expect(renderMarkdownAsPlainText(ir, {}, ALL_NATIVE)).toBe(renderMarkdownAsPlainText(ir)); + }); + + it("keeps explicit label-only link projection above profile fallback", () => { + const profile = { + ...ALL_NATIVE, + constructs: { ...ALL_NATIVE.constructs, linkLabel: "fallback" as const }, + }; + expect(renderMarkdownAsPlainText(ir, { linkStyle: "label" }, profile)).toBe("See docs"); + }); +}); diff --git a/packages/markdown-core/src/render.ts b/packages/markdown-core/src/render.ts index 5ab7a30ae206..c1026b5344cb 100644 --- a/packages/markdown-core/src/render.ts +++ b/packages/markdown-core/src/render.ts @@ -1,3 +1,5 @@ +import { applyConstructFallbacks } from "./construct-fallbacks.js"; +import type { FormatCapabilityProfile } from "./format-capabilities.js"; import { copyMarkdownLinkSpan, isAutoLinkedMarkdownLink, @@ -63,6 +65,7 @@ const STYLE_ORDER: MarkdownStyle[] = [ "heading_6", "bold", "italic", + "underline", "strikethrough", "spoiler", ]; @@ -191,8 +194,13 @@ function sortAnnotationSpans(spans: MarkdownAnnotationSpan[]): MarkdownAnnotatio } /** Renders Markdown IR by nesting configured style markers and optional link markers. */ -export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions): string { - const text = ir.text ?? ""; +export function renderMarkdownWithMarkers( + ir: MarkdownIR, + options: RenderOptions, + profile?: FormatCapabilityProfile, +): string { + const projected = profile ? applyConstructFallbacks(ir, profile) : ir; + const text = projected.text ?? ""; if (!text) { return ""; } @@ -200,7 +208,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions const styleMarkers = options.styleMarkers; const annotationMarkers = options.annotationMarkers ?? {}; const annotated = sortAnnotationSpans( - (ir.annotations ?? []).filter((span) => Boolean(annotationMarkers[span.type])), + (projected.annotations ?? []).filter((span) => Boolean(annotationMarkers[span.type])), ); const dominantAnnotations = annotated.filter( (span) => annotationMarkers[span.type]?.suppressNestedFormatting === true, @@ -210,7 +218,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions ...new Set(annotated.flatMap((span) => [span.start, span.end])), ].toSorted((a, b) => a - b); const styled = sortStyleSpans( - ir.styles + projected.styles .filter((span) => Boolean(styleMarkers[span.style])) .flatMap((span) => { if (STRUCTURAL_STYLES.has(span.style)) { @@ -266,7 +274,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions const linkStarts = new Map(); if (options.buildLink) { - const links = ir.links.flatMap((span) => + const links = projected.links.flatMap((span) => subtractRanges(span, dominantAnnotationRanges) .flatMap((piece) => splitAtBoundaries(piece, annotationBoundaries)) .map((piece) => diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 4003976d2965..35ce911edb62 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -152,7 +152,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +2: generic inbound-root and SCP-host schema validators. // +2: attributed-range renderer and its options contract. // +1: agent-harness transcript visibility projector. - 4690, + // +1: outbound formatting capability profile. + 4691, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/plugin-sdk/format-capabilities.test.ts b/src/plugin-sdk/format-capabilities.test.ts new file mode 100644 index 000000000000..dd9b0d47e40f --- /dev/null +++ b/src/plugin-sdk/format-capabilities.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import type { FormatCapabilityProfile } from "./text-chunking.js"; + +describe("FormatCapabilityProfile", () => { + it("preserves literal inference with satisfies", () => { + const profile = { + mechanism: "ranges", + constructs: { + bold: "native", + italic: "native", + underline: "strip", + strikethrough: "native", + spoiler: "fallback", + codeInline: "native", + codeBlock: "native", + codeLanguage: "strip", + linkLabel: "fallback", + heading: "fallback", + bulletList: "fallback", + orderedList: "fallback", + taskList: "fallback", + table: "strip", + blockquote: "fallback", + image: "strip", + mention: "native", + }, + chunk: { limit: 2_048, unit: "bytes", hardCap: 65_536 }, + } satisfies FormatCapabilityProfile; + + expectTypeOf(profile.mechanism).toEqualTypeOf<"ranges">(); + expectTypeOf(profile.constructs.underline).toEqualTypeOf<"strip">(); + expectTypeOf(profile.chunk.unit).toEqualTypeOf<"bytes">(); + expect(profile.chunk.hardCap).toBe(65_536); + }); +}); diff --git a/src/plugin-sdk/text-chunking.ts b/src/plugin-sdk/text-chunking.ts index 0bb0f94c64c8..64ba9b5f1aa7 100644 --- a/src/plugin-sdk/text-chunking.ts +++ b/src/plugin-sdk/text-chunking.ts @@ -9,6 +9,8 @@ export { } from "../../packages/markdown-core/src/chunk-text.js"; /** Quote-aware HTML tag tokens for exact post-render projections. */ export { tokenizeHtmlTags } from "../../packages/markdown-core/src/html-tags.js"; +/** Static outbound formatting capabilities declared by a channel plugin. */ +export type { FormatCapabilityProfile } from "../../packages/markdown-core/src/format-capabilities.js"; /** * Splits outbound channel text into chunks no longer than the requested limit. diff --git a/src/shared/text/strip-markdown.ts b/src/shared/text/strip-markdown.ts index 073155f17b6d..52046a18f553 100644 --- a/src/shared/text/strip-markdown.ts +++ b/src/shared/text/strip-markdown.ts @@ -1,4 +1,6 @@ import { findAssistantTranscriptRoleHeaderSpans } from "../../../packages/markdown-core/src/assistant-transcript-headers.js"; +import { applyConstructFallbacks } from "../../../packages/markdown-core/src/construct-fallbacks.js"; +import type { FormatCapabilityProfile } from "../../../packages/markdown-core/src/format-capabilities.js"; import { markdownToIR, type MarkdownIR } from "../../../packages/markdown-core/src/ir.js"; type StripMarkdownOptions = { @@ -22,8 +24,7 @@ function collectLinkInsertions( options: StripMarkdownOptions, ): PlainTextInsertion[] { const insertions: PlainTextInsertion[] = []; - const linkStyle = options.linkStyle ?? "label-and-url"; - if (linkStyle === "label-and-url") { + if ((options.linkStyle ?? "label-and-url") === "label-and-url") { for (const link of ir.links) { const href = link.href.trim(); const label = ir.text.slice(link.start, link.end).trim(); @@ -106,7 +107,11 @@ function cleanSpeechText(text: string): string { } /** Parse Markdown, then protect role headers exposed by the final plain-text projection. */ -export function stripMarkdown(text: string, options: StripMarkdownOptions = {}): string { +export function stripMarkdown( + text: string, + options: StripMarkdownOptions = {}, + profile?: FormatCapabilityProfile, +): string { // The IR parser preserves links when role annotations are enabled so this // plain-text projection can still append explicit destinations. Direct rich // renderers suppress overlapping active links later at their own boundary. @@ -114,6 +119,8 @@ export function stripMarkdown(text: string, options: StripMarkdownOptions = {}): assistantTranscriptRoleHeaders: options.assistantTranscriptRoleHeaders, autolink: false, blockquotePrefix: "", + enableHtmlUnderline: profile !== undefined, + enableTaskLists: profile !== undefined, headingStyle: "none", horizontalRuleText: "", linkify: false, @@ -123,9 +130,14 @@ export function stripMarkdown(text: string, options: StripMarkdownOptions = {}): // Detect against the exact leading boundary transports receive. String.trim // removes Unicode whitespace that the transcript header grammar intentionally // does not treat as Markdown indentation. - const plainText = applyPlainTextInsertions(ir.text, [ - ...collectLinkInsertions(ir, options), - ...collectParsedAssistantTranscriptRoleInsertions(ir, options), + const effectiveProfile = + profile && options.linkStyle === "label" + ? { ...profile, constructs: { ...profile.constructs, linkLabel: "strip" as const } } + : profile; + const projectedIr = effectiveProfile ? applyConstructFallbacks(ir, effectiveProfile) : ir; + const plainText = applyPlainTextInsertions(projectedIr.text, [ + ...collectLinkInsertions(projectedIr, options), + ...collectParsedAssistantTranscriptRoleInsertions(projectedIr, options), ]).trim(); const projected = applyPlainTextInsertions( plainText, diff --git a/src/tts/prepare-text.test.ts b/src/tts/prepare-text.test.ts index cf321e9438f7..9487b9ddacca 100644 --- a/src/tts/prepare-text.test.ts +++ b/src/tts/prepare-text.test.ts @@ -1,7 +1,32 @@ // TTS prepare text tests cover text cleanup before speech synthesis. import { describe, expect, it } from "vitest"; +import type { FormatCapabilityProfile } from "../../packages/markdown-core/src/format-capabilities.js"; import { stripMarkdown } from "../shared/text/strip-markdown.js"; +const PLAIN_PROFILE = { + mechanism: "plain", + constructs: { + bold: "strip", + italic: "strip", + underline: "strip", + strikethrough: "strip", + spoiler: "strip", + codeInline: "strip", + codeBlock: "strip", + codeLanguage: "strip", + linkLabel: "fallback", + heading: "strip", + bulletList: "native", + orderedList: "native", + taskList: "fallback", + table: "strip", + blockquote: "strip", + image: "strip", + mention: "strip", + }, + chunk: { limit: 1_600, unit: "chars" }, +} satisfies FormatCapabilityProfile; + /** * Tests that stripMarkdown (used in the TTS pipeline via maybeApplyTtsToPayload) * produces clean text suitable for speech synthesis. @@ -46,6 +71,26 @@ describe("TTS text preparation – stripMarkdown", () => { ).toBe("Read the download"); }); + it("keeps role-header prefixes aligned after labeled link expansion", () => { + expect( + stripMarkdown("[docs](https://example.com)\nuser[Thu] hello", { + assistantTranscriptRoleHeaders: true, + }), + ).toBe("docs (https://example.com)\n[assistant-authored transcript] user[Thu] hello"); + }); + + it("applies profile-aware task and authored-HTML fallbacks before projection", () => { + expect(stripMarkdown("- [x] done\n\nunder", {}, PLAIN_PROFILE)).toBe( + "[x] done\n\nunder", + ); + }); + + it("keeps explicit label-only links above profile fallback", () => { + expect( + stripMarkdown("Read [docs](https://example.com)", { linkStyle: "label" }, PLAIN_PROFILE), + ).toBe("Read docs"); + }); + it("handles a typical LLM reply with mixed markdown", () => { const input = `## Heading with **bold** and *italic*