refactor(markdown): add capability-driven fallbacks (#112985)

* refactor(markdown): add capability-driven fallbacks

* fix(markdown): type task-list core rule state

* style(markdown): make list-prefix return explicit

* fix(markdown): honor label-only plain links

* fix(markdown): retain clipped list provenance
This commit is contained in:
Peter Steinberger
2026-07-23 05:47:48 -04:00
committed by GitHub
parent b190e5e3ea
commit b5232bc4dd
17 changed files with 987 additions and 38 deletions

View File

@@ -129,8 +129,8 @@ c5ff317bb7957d0870cb806e7989c1d48877e3bed583c173ce2f8e9cfe57ee36 module/status-
537047854c21ad20ea0572f8019503cbda3bbafece8194a28c30b0639bde2fde module/string-coerce-runtime
1b5b9a4532db991fce807ba736b550de27aa5f9de2e1f508a8f4f09cae76ea29 module/telegram-account
110944726884fca94f38c9c329b5950629438b9a719f4782c4beeade8bd67746 module/temp-path
16f2d67adc0abed91cb5231dfec298715a43e032edd1e10b10d00a6b0973fcc3 module/text-chunking
1df5a33be5dbf611e301c01c6b8bcf502c3101aae10cd14a79f1bbb58b116497 module/text-runtime
0e230313a7a27a915a7aa399c0afd94c4c3e10d485c7b513e757f41bfc362dbe module/text-chunking
d65c9e55599db94565e1942fe3b595cca0208fabf5ba8b60385a1e60ecd01684 module/text-runtime
a94c9ff59cc361b04f8731a47d722c4bc88cfabf942221369a8cb67bdcda7249 module/tool-plugin
9d6ab352913a573b226e054e1dc8c6d088493aea9954950c65585923b5b6895a module/tool-send
541df9dea799f25e83ea483d481ecebc5b91c016effab593c54d3efe3ee6517b module/web-media

View File

@@ -0,0 +1,295 @@
import { describe, expect, it } from "vitest";
import { applyConstructFallbacks } from "./construct-fallbacks.js";
import type {
ConstructSupport,
FormatCapabilityProfile,
FormatConstruct,
} from "./format-capabilities.js";
import { markdownToIR, sliceMarkdownIR, type MarkdownIR } from "./ir.js";
const ALL_NATIVE = {
mechanism: "markdown",
constructs: {
bold: "native",
italic: "native",
underline: "native",
strikethrough: "native",
spoiler: "native",
codeInline: "native",
codeBlock: "native",
codeLanguage: "native",
linkLabel: "native",
heading: "native",
bulletList: "native",
orderedList: "native",
taskList: "native",
table: "native",
blockquote: "native",
image: "native",
mention: "native",
},
chunk: { limit: 4_000, unit: "chars" },
} satisfies FormatCapabilityProfile;
function withSupport(
construct: FormatConstruct,
support: ConstructSupport,
): FormatCapabilityProfile {
return {
...ALL_NATIVE,
constructs: { ...ALL_NATIVE.constructs, [construct]: support },
};
}
function markdownToTaskIR(markdown: string): MarkdownIR {
return markdownToIR(markdown, { enableTaskLists: true });
}
type FallbackCase = {
name: string;
construct: FormatConstruct;
ir: MarkdownIR;
expected: Record<ConstructSupport, [text: string, styles: string[], links: number]>;
};
const TASK_LIST_IR = markdownToTaskIR("- [x] done");
const CASES: FallbackCase[] = [
{
name: "heading",
construct: "heading",
ir: { text: "Title", styles: [{ start: 0, end: 5, style: "heading_2" }], links: [] },
expected: {
native: ["Title", ["heading_2"], 0],
fallback: ["Title", ["bold"], 0],
strip: ["Title", [], 0],
},
},
{
name: "labeled link",
construct: "linkLabel",
ir: { text: "docs", styles: [], links: [{ start: 0, end: 4, href: "https://example.com" }] },
expected: {
native: ["docs", [], 1],
fallback: ["docs (https://example.com)", [], 0],
strip: ["docs", [], 0],
},
},
{
name: "spoiler",
construct: "spoiler",
ir: { text: "secret", styles: [{ start: 0, end: 6, style: "spoiler" }], links: [] },
expected: {
native: ["secret", ["spoiler"], 0],
fallback: ["secret", [], 0],
strip: ["secret", [], 0],
},
},
{
name: "task list",
construct: "taskList",
ir: TASK_LIST_IR,
expected: {
native: ["• [x] done", [], 0],
fallback: ["[x] done", [], 0],
strip: ["• done", [], 0],
},
},
{
name: "code language",
construct: "codeLanguage",
ir: {
text: "const x = 1;",
styles: [{ start: 0, end: 12, style: "code_block", language: "ts" }],
links: [],
},
expected: {
native: ["const x = 1;", ["code_block:ts"], 0],
fallback: ["const x = 1;", ["code_block"], 0],
strip: ["const x = 1;", ["code_block"], 0],
},
},
{
name: "underline",
construct: "underline",
ir: { text: "under", styles: [{ start: 0, end: 5, style: "underline" }], links: [] },
expected: {
native: ["under", ["underline"], 0],
fallback: ["under", [], 0],
strip: ["under", [], 0],
},
},
];
describe("applyConstructFallbacks", () => {
for (const testCase of CASES) {
for (const support of ["native", "fallback", "strip"] as const) {
it(`${testCase.name}: ${support}`, () => {
const actual = applyConstructFallbacks(
testCase.ir,
withSupport(testCase.construct, support),
);
expect(
[
actual.text,
actual.styles.map((span) =>
span.language ? `${span.style}:${span.language}` : span.style,
),
actual.links.length,
],
`${testCase.name}: ${support}`,
).toEqual(testCase.expected[support]);
});
}
}
it("keeps link suffixes outside surrounding styles", () => {
const ir: MarkdownIR = {
text: "see docs now",
styles: [{ start: 0, end: 12, style: "bold" }],
links: [{ start: 4, end: 8, href: "https://example.com" }],
};
expect(applyConstructFallbacks(ir, withSupport("linkLabel", "fallback"))).toEqual({
text: "see docs (https://example.com) now",
styles: [
{ start: 0, end: 8, style: "bold" },
{ start: 30, end: 34, style: "bold" },
],
links: [],
});
});
it("merges heading fallback with authored bold and respects stripped bold", () => {
const ir: MarkdownIR = {
text: "Title",
styles: [
{ start: 0, end: 5, style: "heading_1" },
{ start: 0, end: 5, style: "bold" },
],
links: [],
};
expect(applyConstructFallbacks(ir, withSupport("heading", "fallback")).styles).toEqual([
{ start: 0, end: 5, style: "bold" },
]);
expect(
applyConstructFallbacks(ir, {
...withSupport("heading", "fallback"),
constructs: {
...ALL_NATIVE.constructs,
heading: "fallback",
bold: "strip",
},
}).styles,
).toEqual([]);
});
it("does not infer list constructs from literal code or escaped markers", () => {
const ir = markdownToTaskIR(
"```\n• [x] command\n1. step\n```\n\n- \\[x] literal\n\n-\n [x] indented code",
);
const profile = {
...withSupport("taskList", "fallback"),
constructs: {
...ALL_NATIVE.constructs,
taskList: "fallback",
orderedList: "strip",
},
} satisfies FormatCapabilityProfile;
expect(applyConstructFallbacks(ir, profile).text).toBe(
"• [x] command\n1. step\n\n• [x] literal\n\n• [x] indented code\n",
);
});
it("preserves task-list provenance through IR slicing", () => {
const ir = markdownToTaskIR("intro\n\n- [x] done");
const task = sliceMarkdownIR(ir, ir.text.indexOf("•"), ir.text.length);
expect(applyConstructFallbacks(task, withSupport("taskList", "fallback")).text).toBe(
"[x] done",
);
const markerOnly = sliceMarkdownIR(ir, ir.text.indexOf("[x]"), ir.text.length);
expect(applyConstructFallbacks(markerOnly, withSupport("taskList", "strip")).text).toBe("done");
});
it("recognizes task items nested inside blockquotes", () => {
for (const markdown of ["> - [x] done", "- > - [x] done"]) {
const ir = markdownToTaskIR(markdown);
expect(
applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text,
markdown,
).toContain("[x] done");
expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).not.toContain(
"[x]",
);
}
});
it("recognizes task items nested on the same source line", () => {
const ir = markdownToTaskIR("- - [x] done");
expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).toBe("• \n • done");
});
it("recognizes multiline task-list markers", () => {
const cases = [
{ markdown: "-\n [x] done", expected: "• done" },
{ markdown: "- [x]\n done", expected: "• \ndone" },
{ markdown: "> -\n> [x] done", expected: "• done" },
];
for (const { markdown, expected } of cases) {
const ir = markdownToTaskIR(markdown);
expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text, markdown).toBe(
expected,
);
}
});
it("does not consume list separators when stripping an empty task", () => {
const ir = markdownToTaskIR("- [x] \n- next");
expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).toBe("• \n• next");
});
it("preserves task markers that collide with shortcut reference links", () => {
const ir = markdownToTaskIR("- [x] done\n\n[x]: https://example.com");
expect(applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text).toBe("[x] done");
expect(ir.links).toEqual([]);
});
it("does not protect checkbox-like links after an earlier list-item block", () => {
const markdown = "- ---\n\n [x] done\n\n[x]: https://example.com";
expect(markdownToIR(markdown, { enableTaskLists: true })).toEqual(markdownToIR(markdown));
});
it("preserves list provenance through structural clones", () => {
const ir = structuredClone(markdownToTaskIR("- [x] done"));
expect(applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text).toBe("[x] done");
});
it("clips task-list provenance at partial marker slice boundaries", () => {
const ir = markdownToTaskIR("- [x] done");
const prefixOnly = sliceMarkdownIR(ir, 0, 2);
expect(applyConstructFallbacks(prefixOnly, withSupport("taskList", "fallback")).text).toBe("");
const insideListMarker = sliceMarkdownIR(ir, 1, ir.text.length);
const stripListAndTask = {
...withSupport("taskList", "strip"),
constructs: { ...ALL_NATIVE.constructs, taskList: "strip", bulletList: "strip" },
} satisfies FormatCapabilityProfile;
expect(applyConstructFallbacks(insideListMarker, stripListAndTask).text).toBe("done");
for (const start of [3, 4]) {
const sliced = sliceMarkdownIR(ir, start, ir.text.length);
const stripped = applyConstructFallbacks(sliced, withSupport("taskList", "strip"));
expect(stripped.text, `start=${start}`).toBe("done");
}
});
it("clips trailing empty list-marker provenance during finalization", () => {
const cases = [
{ markdown: "-", construct: "bulletList" as const },
{ markdown: "1.", construct: "orderedList" as const },
];
for (const { markdown, construct } of cases) {
const ir = markdownToIR(markdown);
expect(applyConstructFallbacks(ir, withSupport(construct, "strip")).text, markdown).toBe("");
}
});
});

View File

@@ -0,0 +1,174 @@
import type { FormatCapabilityProfile, FormatConstruct } from "./format-capabilities.js";
import {
copyMarkdownLinkSpan,
createStyleSpan,
mergeAnnotationSpans,
mergeStyleSpans,
type MarkdownLinkSpan,
type MarkdownStyle,
type MarkdownStyleSpan,
} from "./ir-spans.js";
import { sliceMarkdownIR, type MarkdownIR } from "./ir.js";
type TextEdit = { start: number; end: number; text: string };
const STYLE_CONSTRUCTS: Partial<Record<MarkdownStyle, FormatConstruct>> = {
bold: "bold",
italic: "italic",
underline: "underline",
strikethrough: "strikethrough",
spoiler: "spoiler",
code: "codeInline",
code_block: "codeBlock",
blockquote: "blockquote",
};
function isHeading(style: MarkdownStyle): boolean {
return style.startsWith("heading_");
}
function projectStyles(
styles: MarkdownStyleSpan[],
profile: FormatCapabilityProfile,
): MarkdownStyleSpan[] {
const projected: MarkdownStyleSpan[] = [];
let synthesizedHeading = false;
for (const span of styles) {
if (isHeading(span.style)) {
if (profile.constructs.heading === "native") {
projected.push(span);
} else if (
profile.constructs.heading === "fallback" &&
profile.constructs.bold === "native"
) {
projected.push(createStyleSpan({ ...span, style: "bold" }));
synthesizedHeading = true;
}
continue;
}
const construct = STYLE_CONSTRUCTS[span.style];
if (construct && profile.constructs[construct] !== "native") {
continue;
}
if (span.style === "code_block" && profile.constructs.codeLanguage !== "native") {
projected.push(createStyleSpan({ start: span.start, end: span.end, style: span.style }));
} else {
projected.push(span);
}
}
return synthesizedHeading ? mergeStyleSpans(projected) : projected;
}
function collectLinkFallbacks(
ir: MarkdownIR,
profile: FormatCapabilityProfile,
): { links: MarkdownLinkSpan[]; edits: TextEdit[] } {
if (profile.constructs.linkLabel === "native") {
return { links: ir.links, edits: [] };
}
if (profile.constructs.linkLabel === "strip") {
return { links: [], edits: [] };
}
return {
links: [],
edits: ir.links.flatMap((link) => {
const href = link.href.trim();
const label = ir.text.slice(link.start, link.end).trim();
const comparableHref = href.startsWith("mailto:") ? href.slice("mailto:".length) : href;
return href && label && label !== href && label !== comparableHref
? [{ start: link.end, end: link.end, text: ` (${href})` }]
: [];
}),
};
}
function collectListFallbacks(ir: MarkdownIR, profile: FormatCapabilityProfile): TextEdit[] {
const edits: TextEdit[] = [];
for (const item of ir.listItems ?? []) {
if (item.task) {
if (profile.constructs.taskList === "fallback" && item.listMarker) {
edits.push({ ...item.listMarker, text: "" });
} else if (profile.constructs.taskList === "strip" && item.taskMarker) {
edits.push({ ...item.taskMarker, text: "" });
}
}
if (item.listMarker && item.kind === "bullet" && profile.constructs.bulletList === "strip") {
edits.push({ ...item.listMarker, text: "" });
}
if (item.listMarker && item.kind === "ordered" && profile.constructs.orderedList === "strip") {
edits.push({ ...item.listMarker, text: "" });
}
}
return edits;
}
function appendSlice(target: MarkdownIR, source: MarkdownIR): void {
const offset = target.text.length;
target.text += source.text;
target.styles.push(
...source.styles.map((span) =>
createStyleSpan({
...span,
start: offset + span.start,
end: offset + span.end,
}),
),
);
target.links.push(
...source.links.map((link) =>
copyMarkdownLinkSpan(link, {
start: offset + link.start,
end: offset + link.end,
}),
),
);
const annotations = source.annotations?.map((annotation) => ({
...annotation,
start: offset + annotation.start,
end: offset + annotation.end,
}));
if (annotations?.length) {
(target.annotations ??= []).push(...annotations);
}
}
function applyTextEdits(ir: MarkdownIR, edits: TextEdit[]): MarkdownIR {
if (edits.length === 0) {
return ir;
}
const ordered = edits
.toSorted((a, b) => a.start - b.start || a.end - b.end)
.filter((edit, index, all) => {
const previous = all[index - 1];
return !previous || edit.start !== previous.start || edit.end !== previous.end;
});
const result: MarkdownIR = { text: "", styles: [], links: [] };
let cursor = 0;
for (const edit of ordered) {
appendSlice(result, sliceMarkdownIR(ir, cursor, edit.start));
result.text += edit.text;
cursor = edit.end;
}
appendSlice(result, sliceMarkdownIR(ir, cursor, ir.text.length));
result.styles = mergeStyleSpans(result.styles);
if (result.annotations) {
result.annotations = mergeAnnotationSpans(result.annotations);
}
return result;
}
/** Applies target-declared semantic fallbacks before a mechanism-specific renderer runs. */
export function applyConstructFallbacks(
ir: MarkdownIR,
profile: FormatCapabilityProfile,
): MarkdownIR {
// Tables stay in convertMarkdownTables; images and mentions are already plain
// text in this flat IR, so those constructs have no shared fallback work here.
const styled: MarkdownIR = {
...ir,
styles: projectStyles(ir.styles, profile),
};
const listProjected = applyTextEdits(styled, collectListFallbacks(styled, profile));
const linkProjection = collectLinkFallbacks(listProjected, profile);
return applyTextEdits({ ...listProjected, links: linkProjection.links }, linkProjection.edits);
}

View File

@@ -0,0 +1,31 @@
export type FormatConstruct =
| "bold"
| "italic"
| "underline"
| "strikethrough"
| "spoiler"
| "codeInline"
| "codeBlock"
| "codeLanguage"
| "linkLabel"
| "heading"
| "bulletList"
| "orderedList"
| "taskList"
| "table"
| "blockquote"
| "image"
| "mention";
export type ConstructSupport = "native" | "fallback" | "strip";
/** Static formatting capabilities declared by an outbound channel. */
export type FormatCapabilityProfile = {
mechanism: "markdown" | "html" | "ranges" | "blocks" | "plain";
constructs: Record<FormatConstruct, ConstructSupport>;
chunk: {
limit: number;
unit: "chars" | "utf16" | "bytes";
hardCap?: number;
};
};

View File

@@ -3,10 +3,13 @@ export * from "./chunk-text.js";
export * from "./code-spans.js";
export * from "./fences.js";
export * from "./frontmatter.js";
export * from "./format-capabilities.js";
export * from "./construct-fallbacks.js";
export * from "./html-tags.js";
export * from "./ir.js";
export * from "./render-aware-chunking.js";
export * from "./render-attributed.js";
export * from "./render-plain.js";
export * from "./render.js";
export * from "./tables.js";
export * from "./types.js";

View File

@@ -6,6 +6,7 @@ import type {
export type MarkdownStyle =
| "bold"
| "italic"
| "underline"
| "strikethrough"
| "code"
| "code_block"

View File

@@ -1,6 +1,9 @@
// Markdown Core module implements ir behavior.
import MarkdownIt from "markdown-it";
import markdownItCjkFriendly from "markdown-it-cjk-friendly";
import { HTML_TAG_RE } from "markdown-it/lib/common/html_re.mjs";
import type StateCore from "markdown-it/lib/rules_core/state_core.mjs";
import type StateInline from "markdown-it/lib/rules_inline/state_inline.mjs";
import { visibleWidth } from "../../terminal-core/src/ansi.js";
import {
ASSISTANT_TRANSCRIPT_ROLE_NODE_TYPE,
@@ -9,6 +12,7 @@ import {
type AssistantTranscriptRoleTokenMeta,
} from "./assistant-transcript.js";
import { chunkText } from "./chunk-text.js";
import { tokenizeHtmlTags } from "./html-tags.js";
import {
appendAssistantTranscriptRoleImage,
appendAssistantTranscriptRoleText,
@@ -68,6 +72,14 @@ type MarkdownToken = {
map?: [number, number] | null;
markup?: string;
meta?: unknown;
taskListMarker?: boolean;
};
type MarkdownListItemMarker = {
kind: "bullet" | "ordered";
listMarker?: { start: number; end: number };
task?: true;
taskMarker?: { start: number; end: number };
};
export type MarkdownIR = {
@@ -75,6 +87,7 @@ export type MarkdownIR = {
styles: MarkdownStyleSpan[];
links: MarkdownLinkSpan[];
annotations?: MarkdownAnnotationSpan[];
listItems?: MarkdownListItemMarker[];
};
type MarkdownTableAlignment = "left" | "center" | "right";
@@ -135,6 +148,8 @@ type RenderState = RenderTarget & {
horizontalRuleText: string;
preserveSourceBlockSpacing: boolean;
headingLineEnd: number | undefined;
listItems: MarkdownListItemMarker[];
openListItems: MarkdownListItemMarker[];
};
export type MarkdownParseOptions = {
@@ -142,6 +157,10 @@ export type MarkdownParseOptions = {
assistantTranscriptRoleHeaders?: boolean;
linkify?: boolean;
enableSpoilers?: boolean;
/** Parse authored HTML <u>/<ins> tags into underline spans. */
enableHtmlUnderline?: boolean;
/** Preserve task-list checkboxes as semantic list markers. */
enableTaskLists?: boolean;
headingStyle?: "none" | "bold" | "rich";
blockquotePrefix?: string;
autolink?: boolean;
@@ -178,6 +197,12 @@ function createMarkdownIt(options: MarkdownParseOptions): MarkdownIt {
});
md.use(markdownItCjkFriendly);
md.use(markdownItAssistantTranscriptRoles);
if (options.enableTaskLists) {
md.core.ruler.before("inline", "markdown_core_task_lists", protectTaskListMarkers);
}
if (options.enableHtmlUnderline) {
md.inline.ruler.before("html_inline", "markdown_core_html_underline", parseHtmlUnderline);
}
if (options.enableSpoilers) {
// Spoiler delimiters can surround a line-leading role header. Normalize
// them before semantic detection so later rendering cannot expose a role
@@ -198,6 +223,64 @@ function createMarkdownIt(options: MarkdownParseOptions): MarkdownIt {
return md;
}
function protectTaskListMarkers(state: StateCore): void {
const stack: Array<{ contentStarted: boolean }> = [];
for (const token of state.tokens as MarkdownToken[]) {
if (token.type === "list_item_open") {
stack.push({ contentStarted: false });
continue;
}
if (token.type === "list_item_close") {
stack.pop();
continue;
}
const item = stack.at(-1);
if (!item || item.contentStarted) {
continue;
}
if (token.type === "inline") {
item.contentStarted = true;
if (/^\[[ xX]\](?:[ \t]|\n|$)/u.test(token.content ?? "")) {
token.taskListMarker = true;
token.content = `\\${token.content}`;
}
continue;
}
if (token.type !== "paragraph_open") {
item.contentStarted = true;
}
}
}
function parseHtmlUnderline(state: StateInline, silent: boolean): boolean {
if (state.src.charCodeAt(state.pos) !== 0x3c) {
return false;
}
const raw = HTML_TAG_RE.exec(state.src.slice(state.pos))?.[0];
if (!raw) {
return false;
}
const tag = tokenizeHtmlTags(raw).next().value;
const underlineTag =
tag && tag.start === 0 && (tag.name === "u" || tag.name === "ins") ? tag : undefined;
if (!silent) {
const token = state.push(
!underlineTag || underlineTag.selfClosing
? "text"
: underlineTag.closing
? "underline_close"
: "underline_open",
"",
0,
);
if (!underlineTag || underlineTag.selfClosing) {
token.content = raw;
}
}
state.pos += raw.length;
return true;
}
function getAttr(token: MarkdownToken, name: string): string | null {
if (token.attrGet) {
return token.attrGet(name);
@@ -387,16 +470,35 @@ function appendNestedListSeparator(state: RenderState) {
}
}
function appendListPrefix(state: RenderState) {
function appendListPrefix(state: RenderState, isTask: boolean): MarkdownListItemMarker | undefined {
const stack = state.env.listStack;
const top = stack[stack.length - 1];
if (!top) {
return;
return undefined;
}
top.index += 1;
const indent = " ".repeat(Math.max(0, stack.length - 1));
const prefix = top.type === "ordered" ? `${top.index}. ` : "• ";
state.text += `${indent}${prefix}`;
state.text += indent;
const start = state.text.length;
state.text += prefix;
return {
kind: top.type,
listMarker: { start, end: state.text.length },
...(isTask ? { task: true as const } : {}),
};
}
function recordTaskMarker(state: RenderState, content: string): void {
const item = state.openListItems.at(-1);
if (!item?.task || item.taskMarker) {
return;
}
const marker = /^\[[ xX]\][ \t]?/u.exec(content)?.[0];
if (marker) {
const start = resolveRenderTarget(state).text.length;
item.taskMarker = { start, end: start + marker.length };
}
}
function renderInlineCode(state: RenderState, content: string) {
@@ -770,8 +872,15 @@ function renderTokens(tokens: MarkdownToken[], state: RenderState): void {
}
break;
case "text":
recordTaskMarker(state, token.content ?? "");
appendText(state, token.content ?? "");
break;
case "underline_open":
openStyle(state, "underline");
break;
case "underline_close":
closeStyle(state, "underline");
break;
case ASSISTANT_TRANSCRIPT_ROLE_NODE_TYPE: {
const meta = (token.meta as AssistantTranscriptRoleTokenMeta | undefined)
?.assistantTranscriptRoleHeader;
@@ -906,15 +1015,32 @@ function renderTokens(tokens: MarkdownToken[], state: RenderState): void {
appendTopLevelListSeparator(state);
}
break;
case "list_item_open":
appendListPrefix(state);
case "list_item_open": {
const leadingInline =
tokens[tokenIndex + 1]?.type === "paragraph_open" ? tokens[tokenIndex + 2] : undefined;
const isTask = leadingInline?.type === "inline" && leadingInline.taskListMarker === true;
const item = appendListPrefix(state, isTask);
if (item) {
state.openListItems.push(item);
}
break;
case "list_item_close":
}
case "list_item_close": {
const item = state.openListItems.pop();
if (item) {
state.listItems.push({
kind: item.kind,
...(item.listMarker ? { listMarker: item.listMarker } : {}),
...(item.task ? { task: true } : {}),
...(item.taskMarker ? { taskMarker: item.taskMarker } : {}),
});
}
// Avoid double newlines (nested list's last item already added newline)
if (!state.text.endsWith("\n")) {
state.text += "\n";
}
break;
}
case "code_block":
case "fence":
renderCodeBlock(
@@ -1026,14 +1152,40 @@ function closeRemainingStyles(target: RenderTarget) {
target.openStyles = [];
}
function sliceListMarker(
marker: { start: number; end: number },
start: number,
end: number,
): { start: number; end: number } | undefined {
const sliceStart = Math.max(marker.start, start);
const sliceEnd = Math.min(marker.end, end);
return sliceEnd > sliceStart ? { start: sliceStart - start, end: sliceEnd - start } : undefined;
}
export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): MarkdownIR {
const annotations = sliceAnnotationSpans(ir.annotations ?? [], start, end);
return {
const listItems = (ir.listItems ?? []).flatMap((item) => {
const listMarker = item.listMarker ? sliceListMarker(item.listMarker, start, end) : undefined;
const taskMarker = item.taskMarker ? sliceListMarker(item.taskMarker, start, end) : undefined;
return listMarker || taskMarker
? [
{
kind: item.kind,
...(listMarker ? { listMarker } : {}),
...(item.task ? { task: true as const } : {}),
...(taskMarker ? { taskMarker } : {}),
},
]
: [];
});
const sliced: MarkdownIR = {
text: ir.text.slice(start, end),
styles: sliceStyleSpans(ir.styles, start, end),
links: sliceLinkSpans(ir.links, start, end),
...(annotations.length > 0 ? { annotations } : {}),
...(listItems.length > 0 ? { listItems } : {}),
};
return sliced;
}
export function markdownToIR(markdown: string, options: MarkdownParseOptions = {}): MarkdownIR {
@@ -1072,6 +1224,8 @@ export function markdownToIRWithMeta(
horizontalRuleText: options.horizontalRuleText ?? "───",
preserveSourceBlockSpacing: options.preserveSourceBlockSpacing ?? false,
headingLineEnd: undefined,
listItems: [],
openListItems: [],
};
renderTokens(tokens as MarkdownToken[], state);
@@ -1092,14 +1246,34 @@ export function markdownToIRWithMeta(
const finalText =
finalLength === state.text.length ? state.text : state.text.slice(0, finalLength);
const annotations = mergeAnnotationSpans(clampAnnotationSpans(state.annotations, finalLength));
const listItems = state.listItems.flatMap((item) => {
const listMarker = item.listMarker
? sliceListMarker(item.listMarker, 0, finalLength)
: undefined;
const taskMarker = item.taskMarker
? sliceListMarker(item.taskMarker, 0, finalLength)
: undefined;
return listMarker || taskMarker
? [
{
kind: item.kind,
...(listMarker ? { listMarker } : {}),
...(item.task ? { task: true as const } : {}),
...(taskMarker ? { taskMarker } : {}),
},
]
: [];
});
const ir: MarkdownIR = {
text: finalText,
styles: mergeStyleSpans(clampStyleSpans(state.styles, finalLength)),
links: clampLinkSpans(state.links, finalLength),
...(annotations.length > 0 ? { annotations } : {}),
...(listItems.length > 0 ? { listItems } : {}),
};
return {
ir: {
text: finalText,
styles: mergeStyleSpans(clampStyleSpans(state.styles, finalLength)),
links: clampLinkSpans(state.links, finalLength),
...(annotations.length > 0 ? { annotations } : {}),
},
ir,
hasTables: state.hasTables,
tables: state.collectedTables.map((table) =>
Object.assign({}, table, {
@@ -1132,13 +1306,9 @@ export function chunkMarkdownIR(ir: MarkdownIR, limit: number): MarkdownIR[] {
}
const start = cursor;
const end = Math.min(ir.text.length, start + chunk.length);
const annotations = sliceAnnotationSpans(ir.annotations ?? [], start, end);
results.push({
text: chunk,
styles: sliceStyleSpans(ir.styles, start, end),
links: sliceLinkSpans(ir.links, start, end),
...(annotations.length > 0 ? { annotations } : {}),
});
const sliced = sliceMarkdownIR(ir, start, end);
sliced.text = chunk;
results.push(sliced);
cursor = end;
});

View File

@@ -0,0 +1,42 @@
import { describe, expect, it } from "vitest";
import { markdownToIR } from "./ir.js";
describe("markdownToIR authored HTML underline", () => {
it("emits underline spans only for authored u and ins tags when enabled", () => {
const ir = markdownToIR("<u>under <ins>nested</ins></u> and __bold__", {
enableHtmlUnderline: true,
});
expect(ir.text).toBe("under nested and bold");
expect(ir.styles).toEqual([
{ start: 0, end: 12, style: "underline" },
{ start: 17, end: 21, style: "bold" },
]);
});
it("preserves raw HTML bytes unless underline parsing is explicitly enabled", () => {
expect(markdownToIR("<u>under</u>").text).toBe("<u>under</u>");
});
it("does not enable HTML block parsing for unrelated tags", () => {
const ir = markdownToIR("<div>\n**bold**\n</div>", { enableHtmlUnderline: true });
expect(ir.text).toBe("<div>\nbold\n</div>");
expect(ir.styles).toEqual([{ start: 6, end: 10, style: "bold" }]);
});
it("keeps entity-decoded and escaped underline tags literal", () => {
for (const input of ["&lt;u&gt;text&lt;/u&gt;", "\\<u>text\\</u>"]) {
const ir = markdownToIR(input, { enableHtmlUnderline: true });
expect(ir.text, input).toBe("<u>text</u>");
expect(ir.styles, input).toEqual([]);
}
});
it("does not parse underline-shaped text inside other HTML lexemes", () => {
for (const input of ['<span title="<u>">x</span>', "<!-- <u> -->x"]) {
const ir = markdownToIR(input, { enableHtmlUnderline: true });
expect(ir.text, input).toBe(input);
expect(ir.styles, input).toEqual([]);
}
});
});

View File

@@ -1,3 +1,5 @@
import { applyConstructFallbacks } from "./construct-fallbacks.js";
import type { FormatCapabilityProfile } from "./format-capabilities.js";
import { isAutoLinkedMarkdownLink, type MarkdownAnnotationSpan } from "./ir-spans.js";
import type { MarkdownIR, MarkdownLinkSpan, MarkdownStyle } from "./ir.js";
@@ -24,14 +26,16 @@ export type AttributedRenderOptions<TStyle extends string> = {
export function renderMarkdownWithAttributedRanges<TStyle extends string>(
ir: MarkdownIR,
options: AttributedRenderOptions<TStyle>,
profile?: FormatCapabilityProfile,
): { text: string; ranges: AttributedRange<TStyle>[] } {
const text = ir.text ?? "";
const projected = profile ? applyConstructFallbacks(ir, profile) : ir;
const text = projected.text ?? "";
const insertions: Array<{ pos: number; length: number }> = [];
let rendered = text;
if (options.renderLink) {
rendered = "";
let cursor = 0;
for (const link of [...ir.links].toSorted((a, b) => a.start - b.start)) {
for (const link of [...projected.links].toSorted((a, b) => a.start - b.start)) {
if (link.start < cursor) {
continue;
}
@@ -48,11 +52,11 @@ export function renderMarkdownWithAttributedRanges<TStyle extends string>(
}
rendered = options.trimEnd ? rendered.trimEnd() : rendered;
const spans = ir.styles.flatMap((span) => {
const spans = projected.styles.flatMap((span) => {
const style = options.styleMap[span.style];
return style === undefined ? [] : [{ start: span.start, end: span.end, style }];
});
for (const annotation of ir.annotations ?? []) {
for (const annotation of projected.annotations ?? []) {
const style = options.annotationStyleMap?.[annotation.type];
if (style !== undefined) {
spans.push({ start: annotation.start, end: annotation.end, style });

View File

@@ -0,0 +1,39 @@
import { applyConstructFallbacks } from "./construct-fallbacks.js";
import type { FormatCapabilityProfile } from "./format-capabilities.js";
import type { MarkdownIR } from "./ir.js";
export type PlainRenderOptions = {
linkStyle?: "label" | "label-and-url";
};
/** Projects Markdown IR to plain text, optionally applying channel capability fallbacks. */
export function renderMarkdownAsPlainText(
ir: MarkdownIR,
options: PlainRenderOptions = {},
profile?: FormatCapabilityProfile,
): string {
const effectiveProfile =
profile && options.linkStyle === "label"
? { ...profile, constructs: { ...profile.constructs, linkLabel: "strip" as const } }
: profile;
const projected = effectiveProfile ? applyConstructFallbacks(ir, effectiveProfile) : ir;
if ((options.linkStyle ?? "label-and-url") === "label" || projected.links.length === 0) {
return projected.text;
}
let output = "";
let cursor = 0;
for (const link of [...projected.links].toSorted((a, b) => a.start - b.start)) {
if (link.start < cursor) {
continue;
}
output += projected.text.slice(cursor, link.end);
const href = link.href.trim();
const label = projected.text.slice(link.start, link.end).trim();
const comparableHref = href.startsWith("mailto:") ? href.slice("mailto:".length) : href;
if (href && label && label !== href && label !== comparableHref) {
output += ` (${href})`;
}
cursor = link.end;
}
return output + projected.text.slice(cursor);
}

View File

@@ -0,0 +1,87 @@
import { describe, expect, it } from "vitest";
import type { FormatCapabilityProfile } from "./format-capabilities.js";
import { markdownToIR } from "./ir.js";
import { renderMarkdownWithAttributedRanges } from "./render-attributed.js";
import { renderMarkdownAsPlainText } from "./render-plain.js";
import { renderMarkdownWithMarkers } from "./render.js";
const ALL_NATIVE = {
mechanism: "markdown",
constructs: {
bold: "native",
italic: "native",
underline: "native",
strikethrough: "native",
spoiler: "native",
codeInline: "native",
codeBlock: "native",
codeLanguage: "native",
linkLabel: "native",
heading: "native",
bulletList: "native",
orderedList: "native",
taskList: "native",
table: "native",
blockquote: "native",
image: "native",
mention: "native",
},
chunk: { limit: 4_000, unit: "chars" },
} satisfies FormatCapabilityProfile;
describe("format capability driver plumbing", () => {
const ir = markdownToIR("**See [docs](https://example.com)**", { headingStyle: "rich" });
it("keeps marker rendering byte-identical for an all-native optional profile", () => {
const options = {
styleMarkers: { bold: { open: "<b>", close: "</b>" } },
escapeText: (text: string) => text,
buildLink: (link: { start: number; end: number; href: string }) => ({
start: link.start,
end: link.end,
open: "<a>",
close: "</a>",
}),
};
expect(renderMarkdownWithMarkers(ir, options, ALL_NATIVE)).toBe(
renderMarkdownWithMarkers(ir, options),
);
});
it("preserves caller-constructed style spans for an all-native profile", () => {
const customIr = {
text: "same",
styles: [
{ start: 0, end: 2, style: "bold" as const },
{ start: 2, end: 4, style: "bold" as const },
],
links: [],
};
const options = {
styleMarkers: { bold: { open: "*", close: "*" } },
escapeText: (text: string) => text,
};
expect(renderMarkdownWithMarkers(customIr, options, ALL_NATIVE)).toBe(
renderMarkdownWithMarkers(customIr, options),
);
});
it("keeps attributed rendering byte-identical for an all-native optional profile", () => {
const options = { styleMap: { bold: "strong" as const }, renderLink: () => " (url)" };
expect(renderMarkdownWithAttributedRanges(ir, options, ALL_NATIVE)).toEqual(
renderMarkdownWithAttributedRanges(ir, options),
);
});
it("keeps plain projection byte-identical for an all-native optional profile", () => {
expect(renderMarkdownAsPlainText(ir, {}, ALL_NATIVE)).toBe(renderMarkdownAsPlainText(ir));
});
it("keeps explicit label-only link projection above profile fallback", () => {
const profile = {
...ALL_NATIVE,
constructs: { ...ALL_NATIVE.constructs, linkLabel: "fallback" as const },
};
expect(renderMarkdownAsPlainText(ir, { linkStyle: "label" }, profile)).toBe("See docs");
});
});

View File

@@ -1,3 +1,5 @@
import { applyConstructFallbacks } from "./construct-fallbacks.js";
import type { FormatCapabilityProfile } from "./format-capabilities.js";
import {
copyMarkdownLinkSpan,
isAutoLinkedMarkdownLink,
@@ -63,6 +65,7 @@ const STYLE_ORDER: MarkdownStyle[] = [
"heading_6",
"bold",
"italic",
"underline",
"strikethrough",
"spoiler",
];
@@ -191,8 +194,13 @@ function sortAnnotationSpans(spans: MarkdownAnnotationSpan[]): MarkdownAnnotatio
}
/** Renders Markdown IR by nesting configured style markers and optional link markers. */
export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions): string {
const text = ir.text ?? "";
export function renderMarkdownWithMarkers(
ir: MarkdownIR,
options: RenderOptions,
profile?: FormatCapabilityProfile,
): string {
const projected = profile ? applyConstructFallbacks(ir, profile) : ir;
const text = projected.text ?? "";
if (!text) {
return "";
}
@@ -200,7 +208,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions
const styleMarkers = options.styleMarkers;
const annotationMarkers = options.annotationMarkers ?? {};
const annotated = sortAnnotationSpans(
(ir.annotations ?? []).filter((span) => Boolean(annotationMarkers[span.type])),
(projected.annotations ?? []).filter((span) => Boolean(annotationMarkers[span.type])),
);
const dominantAnnotations = annotated.filter(
(span) => annotationMarkers[span.type]?.suppressNestedFormatting === true,
@@ -210,7 +218,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions
...new Set(annotated.flatMap((span) => [span.start, span.end])),
].toSorted((a, b) => a - b);
const styled = sortStyleSpans(
ir.styles
projected.styles
.filter((span) => Boolean(styleMarkers[span.style]))
.flatMap((span) => {
if (STRUCTURAL_STYLES.has(span.style)) {
@@ -266,7 +274,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions
const linkStarts = new Map<number, RenderLink[]>();
if (options.buildLink) {
const links = ir.links.flatMap((span) =>
const links = projected.links.flatMap((span) =>
subtractRanges(span, dominantAnnotationRanges)
.flatMap((piece) => splitAtBoundaries(piece, annotationBoundaries))
.map((piece) =>

View File

@@ -152,7 +152,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
// +2: generic inbound-root and SCP-host schema validators.
// +2: attributed-range renderer and its options contract.
// +1: agent-harness transcript visibility projector.
4690,
// +1: outbound formatting capability profile.
4691,
env,
),
publicFunctionExports: readPluginSdkSurfaceBudgetEnv(

View File

@@ -0,0 +1,35 @@
import { describe, expect, expectTypeOf, it } from "vitest";
import type { FormatCapabilityProfile } from "./text-chunking.js";
describe("FormatCapabilityProfile", () => {
it("preserves literal inference with satisfies", () => {
const profile = {
mechanism: "ranges",
constructs: {
bold: "native",
italic: "native",
underline: "strip",
strikethrough: "native",
spoiler: "fallback",
codeInline: "native",
codeBlock: "native",
codeLanguage: "strip",
linkLabel: "fallback",
heading: "fallback",
bulletList: "fallback",
orderedList: "fallback",
taskList: "fallback",
table: "strip",
blockquote: "fallback",
image: "strip",
mention: "native",
},
chunk: { limit: 2_048, unit: "bytes", hardCap: 65_536 },
} satisfies FormatCapabilityProfile;
expectTypeOf(profile.mechanism).toEqualTypeOf<"ranges">();
expectTypeOf(profile.constructs.underline).toEqualTypeOf<"strip">();
expectTypeOf(profile.chunk.unit).toEqualTypeOf<"bytes">();
expect(profile.chunk.hardCap).toBe(65_536);
});
});

View File

@@ -9,6 +9,8 @@ export {
} from "../../packages/markdown-core/src/chunk-text.js";
/** Quote-aware HTML tag tokens for exact post-render projections. */
export { tokenizeHtmlTags } from "../../packages/markdown-core/src/html-tags.js";
/** Static outbound formatting capabilities declared by a channel plugin. */
export type { FormatCapabilityProfile } from "../../packages/markdown-core/src/format-capabilities.js";
/**
* Splits outbound channel text into chunks no longer than the requested limit.

View File

@@ -1,4 +1,6 @@
import { findAssistantTranscriptRoleHeaderSpans } from "../../../packages/markdown-core/src/assistant-transcript-headers.js";
import { applyConstructFallbacks } from "../../../packages/markdown-core/src/construct-fallbacks.js";
import type { FormatCapabilityProfile } from "../../../packages/markdown-core/src/format-capabilities.js";
import { markdownToIR, type MarkdownIR } from "../../../packages/markdown-core/src/ir.js";
type StripMarkdownOptions = {
@@ -22,8 +24,7 @@ function collectLinkInsertions(
options: StripMarkdownOptions,
): PlainTextInsertion[] {
const insertions: PlainTextInsertion[] = [];
const linkStyle = options.linkStyle ?? "label-and-url";
if (linkStyle === "label-and-url") {
if ((options.linkStyle ?? "label-and-url") === "label-and-url") {
for (const link of ir.links) {
const href = link.href.trim();
const label = ir.text.slice(link.start, link.end).trim();
@@ -106,7 +107,11 @@ function cleanSpeechText(text: string): string {
}
/** Parse Markdown, then protect role headers exposed by the final plain-text projection. */
export function stripMarkdown(text: string, options: StripMarkdownOptions = {}): string {
export function stripMarkdown(
text: string,
options: StripMarkdownOptions = {},
profile?: FormatCapabilityProfile,
): string {
// The IR parser preserves links when role annotations are enabled so this
// plain-text projection can still append explicit destinations. Direct rich
// renderers suppress overlapping active links later at their own boundary.
@@ -114,6 +119,8 @@ export function stripMarkdown(text: string, options: StripMarkdownOptions = {}):
assistantTranscriptRoleHeaders: options.assistantTranscriptRoleHeaders,
autolink: false,
blockquotePrefix: "",
enableHtmlUnderline: profile !== undefined,
enableTaskLists: profile !== undefined,
headingStyle: "none",
horizontalRuleText: "",
linkify: false,
@@ -123,9 +130,14 @@ export function stripMarkdown(text: string, options: StripMarkdownOptions = {}):
// Detect against the exact leading boundary transports receive. String.trim
// removes Unicode whitespace that the transcript header grammar intentionally
// does not treat as Markdown indentation.
const plainText = applyPlainTextInsertions(ir.text, [
...collectLinkInsertions(ir, options),
...collectParsedAssistantTranscriptRoleInsertions(ir, options),
const effectiveProfile =
profile && options.linkStyle === "label"
? { ...profile, constructs: { ...profile.constructs, linkLabel: "strip" as const } }
: profile;
const projectedIr = effectiveProfile ? applyConstructFallbacks(ir, effectiveProfile) : ir;
const plainText = applyPlainTextInsertions(projectedIr.text, [
...collectLinkInsertions(projectedIr, options),
...collectParsedAssistantTranscriptRoleInsertions(projectedIr, options),
]).trim();
const projected = applyPlainTextInsertions(
plainText,

View File

@@ -1,7 +1,32 @@
// TTS prepare text tests cover text cleanup before speech synthesis.
import { describe, expect, it } from "vitest";
import type { FormatCapabilityProfile } from "../../packages/markdown-core/src/format-capabilities.js";
import { stripMarkdown } from "../shared/text/strip-markdown.js";
const PLAIN_PROFILE = {
mechanism: "plain",
constructs: {
bold: "strip",
italic: "strip",
underline: "strip",
strikethrough: "strip",
spoiler: "strip",
codeInline: "strip",
codeBlock: "strip",
codeLanguage: "strip",
linkLabel: "fallback",
heading: "strip",
bulletList: "native",
orderedList: "native",
taskList: "fallback",
table: "strip",
blockquote: "strip",
image: "strip",
mention: "strip",
},
chunk: { limit: 1_600, unit: "chars" },
} satisfies FormatCapabilityProfile;
/**
* Tests that stripMarkdown (used in the TTS pipeline via maybeApplyTtsToPayload)
* produces clean text suitable for speech synthesis.
@@ -46,6 +71,26 @@ describe("TTS text preparation stripMarkdown", () => {
).toBe("Read the download");
});
it("keeps role-header prefixes aligned after labeled link expansion", () => {
expect(
stripMarkdown("[docs](https://example.com)\nuser[Thu] hello", {
assistantTranscriptRoleHeaders: true,
}),
).toBe("docs (https://example.com)\n[assistant-authored transcript] user[Thu] hello");
});
it("applies profile-aware task and authored-HTML fallbacks before projection", () => {
expect(stripMarkdown("- [x] done\n\n<u>under</u>", {}, PLAIN_PROFILE)).toBe(
"[x] done\n\nunder",
);
});
it("keeps explicit label-only links above profile fallback", () => {
expect(
stripMarkdown("Read [docs](https://example.com)", { linkStyle: "label" }, PLAIN_PROFILE),
).toBe("Read docs");
});
it("handles a typical LLM reply with mixed markdown", () => {
const input = `## Heading with **bold** and *italic*