mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-02 08:21:35 +00:00
refactor(markdown): add capability-driven fallbacks (#112985)
* refactor(markdown): add capability-driven fallbacks * fix(markdown): type task-list core rule state * style(markdown): make list-prefix return explicit * fix(markdown): honor label-only plain links * fix(markdown): retain clipped list provenance
This commit is contained in:
committed by
GitHub
parent
b190e5e3ea
commit
b5232bc4dd
@@ -129,8 +129,8 @@ c5ff317bb7957d0870cb806e7989c1d48877e3bed583c173ce2f8e9cfe57ee36 module/status-
|
||||
537047854c21ad20ea0572f8019503cbda3bbafece8194a28c30b0639bde2fde module/string-coerce-runtime
|
||||
1b5b9a4532db991fce807ba736b550de27aa5f9de2e1f508a8f4f09cae76ea29 module/telegram-account
|
||||
110944726884fca94f38c9c329b5950629438b9a719f4782c4beeade8bd67746 module/temp-path
|
||||
16f2d67adc0abed91cb5231dfec298715a43e032edd1e10b10d00a6b0973fcc3 module/text-chunking
|
||||
1df5a33be5dbf611e301c01c6b8bcf502c3101aae10cd14a79f1bbb58b116497 module/text-runtime
|
||||
0e230313a7a27a915a7aa399c0afd94c4c3e10d485c7b513e757f41bfc362dbe module/text-chunking
|
||||
d65c9e55599db94565e1942fe3b595cca0208fabf5ba8b60385a1e60ecd01684 module/text-runtime
|
||||
a94c9ff59cc361b04f8731a47d722c4bc88cfabf942221369a8cb67bdcda7249 module/tool-plugin
|
||||
9d6ab352913a573b226e054e1dc8c6d088493aea9954950c65585923b5b6895a module/tool-send
|
||||
541df9dea799f25e83ea483d481ecebc5b91c016effab593c54d3efe3ee6517b module/web-media
|
||||
|
||||
295
packages/markdown-core/src/construct-fallbacks.test.ts
Normal file
295
packages/markdown-core/src/construct-fallbacks.test.ts
Normal file
@@ -0,0 +1,295 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyConstructFallbacks } from "./construct-fallbacks.js";
|
||||
import type {
|
||||
ConstructSupport,
|
||||
FormatCapabilityProfile,
|
||||
FormatConstruct,
|
||||
} from "./format-capabilities.js";
|
||||
import { markdownToIR, sliceMarkdownIR, type MarkdownIR } from "./ir.js";
|
||||
|
||||
const ALL_NATIVE = {
|
||||
mechanism: "markdown",
|
||||
constructs: {
|
||||
bold: "native",
|
||||
italic: "native",
|
||||
underline: "native",
|
||||
strikethrough: "native",
|
||||
spoiler: "native",
|
||||
codeInline: "native",
|
||||
codeBlock: "native",
|
||||
codeLanguage: "native",
|
||||
linkLabel: "native",
|
||||
heading: "native",
|
||||
bulletList: "native",
|
||||
orderedList: "native",
|
||||
taskList: "native",
|
||||
table: "native",
|
||||
blockquote: "native",
|
||||
image: "native",
|
||||
mention: "native",
|
||||
},
|
||||
chunk: { limit: 4_000, unit: "chars" },
|
||||
} satisfies FormatCapabilityProfile;
|
||||
|
||||
function withSupport(
|
||||
construct: FormatConstruct,
|
||||
support: ConstructSupport,
|
||||
): FormatCapabilityProfile {
|
||||
return {
|
||||
...ALL_NATIVE,
|
||||
constructs: { ...ALL_NATIVE.constructs, [construct]: support },
|
||||
};
|
||||
}
|
||||
|
||||
function markdownToTaskIR(markdown: string): MarkdownIR {
|
||||
return markdownToIR(markdown, { enableTaskLists: true });
|
||||
}
|
||||
|
||||
type FallbackCase = {
|
||||
name: string;
|
||||
construct: FormatConstruct;
|
||||
ir: MarkdownIR;
|
||||
expected: Record<ConstructSupport, [text: string, styles: string[], links: number]>;
|
||||
};
|
||||
|
||||
const TASK_LIST_IR = markdownToTaskIR("- [x] done");
|
||||
|
||||
const CASES: FallbackCase[] = [
|
||||
{
|
||||
name: "heading",
|
||||
construct: "heading",
|
||||
ir: { text: "Title", styles: [{ start: 0, end: 5, style: "heading_2" }], links: [] },
|
||||
expected: {
|
||||
native: ["Title", ["heading_2"], 0],
|
||||
fallback: ["Title", ["bold"], 0],
|
||||
strip: ["Title", [], 0],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "labeled link",
|
||||
construct: "linkLabel",
|
||||
ir: { text: "docs", styles: [], links: [{ start: 0, end: 4, href: "https://example.com" }] },
|
||||
expected: {
|
||||
native: ["docs", [], 1],
|
||||
fallback: ["docs (https://example.com)", [], 0],
|
||||
strip: ["docs", [], 0],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "spoiler",
|
||||
construct: "spoiler",
|
||||
ir: { text: "secret", styles: [{ start: 0, end: 6, style: "spoiler" }], links: [] },
|
||||
expected: {
|
||||
native: ["secret", ["spoiler"], 0],
|
||||
fallback: ["secret", [], 0],
|
||||
strip: ["secret", [], 0],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "task list",
|
||||
construct: "taskList",
|
||||
ir: TASK_LIST_IR,
|
||||
expected: {
|
||||
native: ["• [x] done", [], 0],
|
||||
fallback: ["[x] done", [], 0],
|
||||
strip: ["• done", [], 0],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "code language",
|
||||
construct: "codeLanguage",
|
||||
ir: {
|
||||
text: "const x = 1;",
|
||||
styles: [{ start: 0, end: 12, style: "code_block", language: "ts" }],
|
||||
links: [],
|
||||
},
|
||||
expected: {
|
||||
native: ["const x = 1;", ["code_block:ts"], 0],
|
||||
fallback: ["const x = 1;", ["code_block"], 0],
|
||||
strip: ["const x = 1;", ["code_block"], 0],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "underline",
|
||||
construct: "underline",
|
||||
ir: { text: "under", styles: [{ start: 0, end: 5, style: "underline" }], links: [] },
|
||||
expected: {
|
||||
native: ["under", ["underline"], 0],
|
||||
fallback: ["under", [], 0],
|
||||
strip: ["under", [], 0],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe("applyConstructFallbacks", () => {
|
||||
for (const testCase of CASES) {
|
||||
for (const support of ["native", "fallback", "strip"] as const) {
|
||||
it(`${testCase.name}: ${support}`, () => {
|
||||
const actual = applyConstructFallbacks(
|
||||
testCase.ir,
|
||||
withSupport(testCase.construct, support),
|
||||
);
|
||||
expect(
|
||||
[
|
||||
actual.text,
|
||||
actual.styles.map((span) =>
|
||||
span.language ? `${span.style}:${span.language}` : span.style,
|
||||
),
|
||||
actual.links.length,
|
||||
],
|
||||
`${testCase.name}: ${support}`,
|
||||
).toEqual(testCase.expected[support]);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
it("keeps link suffixes outside surrounding styles", () => {
|
||||
const ir: MarkdownIR = {
|
||||
text: "see docs now",
|
||||
styles: [{ start: 0, end: 12, style: "bold" }],
|
||||
links: [{ start: 4, end: 8, href: "https://example.com" }],
|
||||
};
|
||||
|
||||
expect(applyConstructFallbacks(ir, withSupport("linkLabel", "fallback"))).toEqual({
|
||||
text: "see docs (https://example.com) now",
|
||||
styles: [
|
||||
{ start: 0, end: 8, style: "bold" },
|
||||
{ start: 30, end: 34, style: "bold" },
|
||||
],
|
||||
links: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("merges heading fallback with authored bold and respects stripped bold", () => {
|
||||
const ir: MarkdownIR = {
|
||||
text: "Title",
|
||||
styles: [
|
||||
{ start: 0, end: 5, style: "heading_1" },
|
||||
{ start: 0, end: 5, style: "bold" },
|
||||
],
|
||||
links: [],
|
||||
};
|
||||
expect(applyConstructFallbacks(ir, withSupport("heading", "fallback")).styles).toEqual([
|
||||
{ start: 0, end: 5, style: "bold" },
|
||||
]);
|
||||
expect(
|
||||
applyConstructFallbacks(ir, {
|
||||
...withSupport("heading", "fallback"),
|
||||
constructs: {
|
||||
...ALL_NATIVE.constructs,
|
||||
heading: "fallback",
|
||||
bold: "strip",
|
||||
},
|
||||
}).styles,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not infer list constructs from literal code or escaped markers", () => {
|
||||
const ir = markdownToTaskIR(
|
||||
"```\n• [x] command\n1. step\n```\n\n- \\[x] literal\n\n-\n [x] indented code",
|
||||
);
|
||||
const profile = {
|
||||
...withSupport("taskList", "fallback"),
|
||||
constructs: {
|
||||
...ALL_NATIVE.constructs,
|
||||
taskList: "fallback",
|
||||
orderedList: "strip",
|
||||
},
|
||||
} satisfies FormatCapabilityProfile;
|
||||
|
||||
expect(applyConstructFallbacks(ir, profile).text).toBe(
|
||||
"• [x] command\n1. step\n\n• [x] literal\n\n• [x] indented code\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves task-list provenance through IR slicing", () => {
|
||||
const ir = markdownToTaskIR("intro\n\n- [x] done");
|
||||
const task = sliceMarkdownIR(ir, ir.text.indexOf("•"), ir.text.length);
|
||||
expect(applyConstructFallbacks(task, withSupport("taskList", "fallback")).text).toBe(
|
||||
"[x] done",
|
||||
);
|
||||
const markerOnly = sliceMarkdownIR(ir, ir.text.indexOf("[x]"), ir.text.length);
|
||||
expect(applyConstructFallbacks(markerOnly, withSupport("taskList", "strip")).text).toBe("done");
|
||||
});
|
||||
|
||||
it("recognizes task items nested inside blockquotes", () => {
|
||||
for (const markdown of ["> - [x] done", "- > - [x] done"]) {
|
||||
const ir = markdownToTaskIR(markdown);
|
||||
expect(
|
||||
applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text,
|
||||
markdown,
|
||||
).toContain("[x] done");
|
||||
expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).not.toContain(
|
||||
"[x]",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("recognizes task items nested on the same source line", () => {
|
||||
const ir = markdownToTaskIR("- - [x] done");
|
||||
expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).toBe("• \n • done");
|
||||
});
|
||||
|
||||
it("recognizes multiline task-list markers", () => {
|
||||
const cases = [
|
||||
{ markdown: "-\n [x] done", expected: "• done" },
|
||||
{ markdown: "- [x]\n done", expected: "• \ndone" },
|
||||
{ markdown: "> -\n> [x] done", expected: "• done" },
|
||||
];
|
||||
for (const { markdown, expected } of cases) {
|
||||
const ir = markdownToTaskIR(markdown);
|
||||
expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text, markdown).toBe(
|
||||
expected,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not consume list separators when stripping an empty task", () => {
|
||||
const ir = markdownToTaskIR("- [x] \n- next");
|
||||
expect(applyConstructFallbacks(ir, withSupport("taskList", "strip")).text).toBe("• \n• next");
|
||||
});
|
||||
|
||||
it("preserves task markers that collide with shortcut reference links", () => {
|
||||
const ir = markdownToTaskIR("- [x] done\n\n[x]: https://example.com");
|
||||
expect(applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text).toBe("[x] done");
|
||||
expect(ir.links).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not protect checkbox-like links after an earlier list-item block", () => {
|
||||
const markdown = "- ---\n\n [x] done\n\n[x]: https://example.com";
|
||||
expect(markdownToIR(markdown, { enableTaskLists: true })).toEqual(markdownToIR(markdown));
|
||||
});
|
||||
|
||||
it("preserves list provenance through structural clones", () => {
|
||||
const ir = structuredClone(markdownToTaskIR("- [x] done"));
|
||||
expect(applyConstructFallbacks(ir, withSupport("taskList", "fallback")).text).toBe("[x] done");
|
||||
});
|
||||
|
||||
it("clips task-list provenance at partial marker slice boundaries", () => {
|
||||
const ir = markdownToTaskIR("- [x] done");
|
||||
const prefixOnly = sliceMarkdownIR(ir, 0, 2);
|
||||
expect(applyConstructFallbacks(prefixOnly, withSupport("taskList", "fallback")).text).toBe("");
|
||||
const insideListMarker = sliceMarkdownIR(ir, 1, ir.text.length);
|
||||
const stripListAndTask = {
|
||||
...withSupport("taskList", "strip"),
|
||||
constructs: { ...ALL_NATIVE.constructs, taskList: "strip", bulletList: "strip" },
|
||||
} satisfies FormatCapabilityProfile;
|
||||
expect(applyConstructFallbacks(insideListMarker, stripListAndTask).text).toBe("done");
|
||||
for (const start of [3, 4]) {
|
||||
const sliced = sliceMarkdownIR(ir, start, ir.text.length);
|
||||
const stripped = applyConstructFallbacks(sliced, withSupport("taskList", "strip"));
|
||||
expect(stripped.text, `start=${start}`).toBe("done");
|
||||
}
|
||||
});
|
||||
|
||||
it("clips trailing empty list-marker provenance during finalization", () => {
|
||||
const cases = [
|
||||
{ markdown: "-", construct: "bulletList" as const },
|
||||
{ markdown: "1.", construct: "orderedList" as const },
|
||||
];
|
||||
for (const { markdown, construct } of cases) {
|
||||
const ir = markdownToIR(markdown);
|
||||
expect(applyConstructFallbacks(ir, withSupport(construct, "strip")).text, markdown).toBe("");
|
||||
}
|
||||
});
|
||||
});
|
||||
174
packages/markdown-core/src/construct-fallbacks.ts
Normal file
174
packages/markdown-core/src/construct-fallbacks.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
import type { FormatCapabilityProfile, FormatConstruct } from "./format-capabilities.js";
|
||||
import {
|
||||
copyMarkdownLinkSpan,
|
||||
createStyleSpan,
|
||||
mergeAnnotationSpans,
|
||||
mergeStyleSpans,
|
||||
type MarkdownLinkSpan,
|
||||
type MarkdownStyle,
|
||||
type MarkdownStyleSpan,
|
||||
} from "./ir-spans.js";
|
||||
import { sliceMarkdownIR, type MarkdownIR } from "./ir.js";
|
||||
|
||||
type TextEdit = { start: number; end: number; text: string };
|
||||
|
||||
const STYLE_CONSTRUCTS: Partial<Record<MarkdownStyle, FormatConstruct>> = {
|
||||
bold: "bold",
|
||||
italic: "italic",
|
||||
underline: "underline",
|
||||
strikethrough: "strikethrough",
|
||||
spoiler: "spoiler",
|
||||
code: "codeInline",
|
||||
code_block: "codeBlock",
|
||||
blockquote: "blockquote",
|
||||
};
|
||||
|
||||
function isHeading(style: MarkdownStyle): boolean {
|
||||
return style.startsWith("heading_");
|
||||
}
|
||||
|
||||
function projectStyles(
|
||||
styles: MarkdownStyleSpan[],
|
||||
profile: FormatCapabilityProfile,
|
||||
): MarkdownStyleSpan[] {
|
||||
const projected: MarkdownStyleSpan[] = [];
|
||||
let synthesizedHeading = false;
|
||||
for (const span of styles) {
|
||||
if (isHeading(span.style)) {
|
||||
if (profile.constructs.heading === "native") {
|
||||
projected.push(span);
|
||||
} else if (
|
||||
profile.constructs.heading === "fallback" &&
|
||||
profile.constructs.bold === "native"
|
||||
) {
|
||||
projected.push(createStyleSpan({ ...span, style: "bold" }));
|
||||
synthesizedHeading = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const construct = STYLE_CONSTRUCTS[span.style];
|
||||
if (construct && profile.constructs[construct] !== "native") {
|
||||
continue;
|
||||
}
|
||||
if (span.style === "code_block" && profile.constructs.codeLanguage !== "native") {
|
||||
projected.push(createStyleSpan({ start: span.start, end: span.end, style: span.style }));
|
||||
} else {
|
||||
projected.push(span);
|
||||
}
|
||||
}
|
||||
return synthesizedHeading ? mergeStyleSpans(projected) : projected;
|
||||
}
|
||||
|
||||
function collectLinkFallbacks(
|
||||
ir: MarkdownIR,
|
||||
profile: FormatCapabilityProfile,
|
||||
): { links: MarkdownLinkSpan[]; edits: TextEdit[] } {
|
||||
if (profile.constructs.linkLabel === "native") {
|
||||
return { links: ir.links, edits: [] };
|
||||
}
|
||||
if (profile.constructs.linkLabel === "strip") {
|
||||
return { links: [], edits: [] };
|
||||
}
|
||||
return {
|
||||
links: [],
|
||||
edits: ir.links.flatMap((link) => {
|
||||
const href = link.href.trim();
|
||||
const label = ir.text.slice(link.start, link.end).trim();
|
||||
const comparableHref = href.startsWith("mailto:") ? href.slice("mailto:".length) : href;
|
||||
return href && label && label !== href && label !== comparableHref
|
||||
? [{ start: link.end, end: link.end, text: ` (${href})` }]
|
||||
: [];
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function collectListFallbacks(ir: MarkdownIR, profile: FormatCapabilityProfile): TextEdit[] {
|
||||
const edits: TextEdit[] = [];
|
||||
for (const item of ir.listItems ?? []) {
|
||||
if (item.task) {
|
||||
if (profile.constructs.taskList === "fallback" && item.listMarker) {
|
||||
edits.push({ ...item.listMarker, text: "" });
|
||||
} else if (profile.constructs.taskList === "strip" && item.taskMarker) {
|
||||
edits.push({ ...item.taskMarker, text: "" });
|
||||
}
|
||||
}
|
||||
if (item.listMarker && item.kind === "bullet" && profile.constructs.bulletList === "strip") {
|
||||
edits.push({ ...item.listMarker, text: "" });
|
||||
}
|
||||
if (item.listMarker && item.kind === "ordered" && profile.constructs.orderedList === "strip") {
|
||||
edits.push({ ...item.listMarker, text: "" });
|
||||
}
|
||||
}
|
||||
return edits;
|
||||
}
|
||||
|
||||
function appendSlice(target: MarkdownIR, source: MarkdownIR): void {
|
||||
const offset = target.text.length;
|
||||
target.text += source.text;
|
||||
target.styles.push(
|
||||
...source.styles.map((span) =>
|
||||
createStyleSpan({
|
||||
...span,
|
||||
start: offset + span.start,
|
||||
end: offset + span.end,
|
||||
}),
|
||||
),
|
||||
);
|
||||
target.links.push(
|
||||
...source.links.map((link) =>
|
||||
copyMarkdownLinkSpan(link, {
|
||||
start: offset + link.start,
|
||||
end: offset + link.end,
|
||||
}),
|
||||
),
|
||||
);
|
||||
const annotations = source.annotations?.map((annotation) => ({
|
||||
...annotation,
|
||||
start: offset + annotation.start,
|
||||
end: offset + annotation.end,
|
||||
}));
|
||||
if (annotations?.length) {
|
||||
(target.annotations ??= []).push(...annotations);
|
||||
}
|
||||
}
|
||||
|
||||
function applyTextEdits(ir: MarkdownIR, edits: TextEdit[]): MarkdownIR {
|
||||
if (edits.length === 0) {
|
||||
return ir;
|
||||
}
|
||||
const ordered = edits
|
||||
.toSorted((a, b) => a.start - b.start || a.end - b.end)
|
||||
.filter((edit, index, all) => {
|
||||
const previous = all[index - 1];
|
||||
return !previous || edit.start !== previous.start || edit.end !== previous.end;
|
||||
});
|
||||
const result: MarkdownIR = { text: "", styles: [], links: [] };
|
||||
let cursor = 0;
|
||||
for (const edit of ordered) {
|
||||
appendSlice(result, sliceMarkdownIR(ir, cursor, edit.start));
|
||||
result.text += edit.text;
|
||||
cursor = edit.end;
|
||||
}
|
||||
appendSlice(result, sliceMarkdownIR(ir, cursor, ir.text.length));
|
||||
result.styles = mergeStyleSpans(result.styles);
|
||||
if (result.annotations) {
|
||||
result.annotations = mergeAnnotationSpans(result.annotations);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Applies target-declared semantic fallbacks before a mechanism-specific renderer runs. */
|
||||
export function applyConstructFallbacks(
|
||||
ir: MarkdownIR,
|
||||
profile: FormatCapabilityProfile,
|
||||
): MarkdownIR {
|
||||
// Tables stay in convertMarkdownTables; images and mentions are already plain
|
||||
// text in this flat IR, so those constructs have no shared fallback work here.
|
||||
const styled: MarkdownIR = {
|
||||
...ir,
|
||||
styles: projectStyles(ir.styles, profile),
|
||||
};
|
||||
const listProjected = applyTextEdits(styled, collectListFallbacks(styled, profile));
|
||||
const linkProjection = collectLinkFallbacks(listProjected, profile);
|
||||
return applyTextEdits({ ...listProjected, links: linkProjection.links }, linkProjection.edits);
|
||||
}
|
||||
31
packages/markdown-core/src/format-capabilities.ts
Normal file
31
packages/markdown-core/src/format-capabilities.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
export type FormatConstruct =
|
||||
| "bold"
|
||||
| "italic"
|
||||
| "underline"
|
||||
| "strikethrough"
|
||||
| "spoiler"
|
||||
| "codeInline"
|
||||
| "codeBlock"
|
||||
| "codeLanguage"
|
||||
| "linkLabel"
|
||||
| "heading"
|
||||
| "bulletList"
|
||||
| "orderedList"
|
||||
| "taskList"
|
||||
| "table"
|
||||
| "blockquote"
|
||||
| "image"
|
||||
| "mention";
|
||||
|
||||
export type ConstructSupport = "native" | "fallback" | "strip";
|
||||
|
||||
/** Static formatting capabilities declared by an outbound channel. */
|
||||
export type FormatCapabilityProfile = {
|
||||
mechanism: "markdown" | "html" | "ranges" | "blocks" | "plain";
|
||||
constructs: Record<FormatConstruct, ConstructSupport>;
|
||||
chunk: {
|
||||
limit: number;
|
||||
unit: "chars" | "utf16" | "bytes";
|
||||
hardCap?: number;
|
||||
};
|
||||
};
|
||||
@@ -3,10 +3,13 @@ export * from "./chunk-text.js";
|
||||
export * from "./code-spans.js";
|
||||
export * from "./fences.js";
|
||||
export * from "./frontmatter.js";
|
||||
export * from "./format-capabilities.js";
|
||||
export * from "./construct-fallbacks.js";
|
||||
export * from "./html-tags.js";
|
||||
export * from "./ir.js";
|
||||
export * from "./render-aware-chunking.js";
|
||||
export * from "./render-attributed.js";
|
||||
export * from "./render-plain.js";
|
||||
export * from "./render.js";
|
||||
export * from "./tables.js";
|
||||
export * from "./types.js";
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
export type MarkdownStyle =
|
||||
| "bold"
|
||||
| "italic"
|
||||
| "underline"
|
||||
| "strikethrough"
|
||||
| "code"
|
||||
| "code_block"
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
// Markdown Core module implements ir behavior.
|
||||
import MarkdownIt from "markdown-it";
|
||||
import markdownItCjkFriendly from "markdown-it-cjk-friendly";
|
||||
import { HTML_TAG_RE } from "markdown-it/lib/common/html_re.mjs";
|
||||
import type StateCore from "markdown-it/lib/rules_core/state_core.mjs";
|
||||
import type StateInline from "markdown-it/lib/rules_inline/state_inline.mjs";
|
||||
import { visibleWidth } from "../../terminal-core/src/ansi.js";
|
||||
import {
|
||||
ASSISTANT_TRANSCRIPT_ROLE_NODE_TYPE,
|
||||
@@ -9,6 +12,7 @@ import {
|
||||
type AssistantTranscriptRoleTokenMeta,
|
||||
} from "./assistant-transcript.js";
|
||||
import { chunkText } from "./chunk-text.js";
|
||||
import { tokenizeHtmlTags } from "./html-tags.js";
|
||||
import {
|
||||
appendAssistantTranscriptRoleImage,
|
||||
appendAssistantTranscriptRoleText,
|
||||
@@ -68,6 +72,14 @@ type MarkdownToken = {
|
||||
map?: [number, number] | null;
|
||||
markup?: string;
|
||||
meta?: unknown;
|
||||
taskListMarker?: boolean;
|
||||
};
|
||||
|
||||
type MarkdownListItemMarker = {
|
||||
kind: "bullet" | "ordered";
|
||||
listMarker?: { start: number; end: number };
|
||||
task?: true;
|
||||
taskMarker?: { start: number; end: number };
|
||||
};
|
||||
|
||||
export type MarkdownIR = {
|
||||
@@ -75,6 +87,7 @@ export type MarkdownIR = {
|
||||
styles: MarkdownStyleSpan[];
|
||||
links: MarkdownLinkSpan[];
|
||||
annotations?: MarkdownAnnotationSpan[];
|
||||
listItems?: MarkdownListItemMarker[];
|
||||
};
|
||||
|
||||
type MarkdownTableAlignment = "left" | "center" | "right";
|
||||
@@ -135,6 +148,8 @@ type RenderState = RenderTarget & {
|
||||
horizontalRuleText: string;
|
||||
preserveSourceBlockSpacing: boolean;
|
||||
headingLineEnd: number | undefined;
|
||||
listItems: MarkdownListItemMarker[];
|
||||
openListItems: MarkdownListItemMarker[];
|
||||
};
|
||||
|
||||
export type MarkdownParseOptions = {
|
||||
@@ -142,6 +157,10 @@ export type MarkdownParseOptions = {
|
||||
assistantTranscriptRoleHeaders?: boolean;
|
||||
linkify?: boolean;
|
||||
enableSpoilers?: boolean;
|
||||
/** Parse authored HTML <u>/<ins> tags into underline spans. */
|
||||
enableHtmlUnderline?: boolean;
|
||||
/** Preserve task-list checkboxes as semantic list markers. */
|
||||
enableTaskLists?: boolean;
|
||||
headingStyle?: "none" | "bold" | "rich";
|
||||
blockquotePrefix?: string;
|
||||
autolink?: boolean;
|
||||
@@ -178,6 +197,12 @@ function createMarkdownIt(options: MarkdownParseOptions): MarkdownIt {
|
||||
});
|
||||
md.use(markdownItCjkFriendly);
|
||||
md.use(markdownItAssistantTranscriptRoles);
|
||||
if (options.enableTaskLists) {
|
||||
md.core.ruler.before("inline", "markdown_core_task_lists", protectTaskListMarkers);
|
||||
}
|
||||
if (options.enableHtmlUnderline) {
|
||||
md.inline.ruler.before("html_inline", "markdown_core_html_underline", parseHtmlUnderline);
|
||||
}
|
||||
if (options.enableSpoilers) {
|
||||
// Spoiler delimiters can surround a line-leading role header. Normalize
|
||||
// them before semantic detection so later rendering cannot expose a role
|
||||
@@ -198,6 +223,64 @@ function createMarkdownIt(options: MarkdownParseOptions): MarkdownIt {
|
||||
return md;
|
||||
}
|
||||
|
||||
function protectTaskListMarkers(state: StateCore): void {
|
||||
const stack: Array<{ contentStarted: boolean }> = [];
|
||||
for (const token of state.tokens as MarkdownToken[]) {
|
||||
if (token.type === "list_item_open") {
|
||||
stack.push({ contentStarted: false });
|
||||
continue;
|
||||
}
|
||||
if (token.type === "list_item_close") {
|
||||
stack.pop();
|
||||
continue;
|
||||
}
|
||||
const item = stack.at(-1);
|
||||
if (!item || item.contentStarted) {
|
||||
continue;
|
||||
}
|
||||
if (token.type === "inline") {
|
||||
item.contentStarted = true;
|
||||
if (/^\[[ xX]\](?:[ \t]|\n|$)/u.test(token.content ?? "")) {
|
||||
token.taskListMarker = true;
|
||||
token.content = `\\${token.content}`;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (token.type !== "paragraph_open") {
|
||||
item.contentStarted = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseHtmlUnderline(state: StateInline, silent: boolean): boolean {
|
||||
if (state.src.charCodeAt(state.pos) !== 0x3c) {
|
||||
return false;
|
||||
}
|
||||
const raw = HTML_TAG_RE.exec(state.src.slice(state.pos))?.[0];
|
||||
if (!raw) {
|
||||
return false;
|
||||
}
|
||||
const tag = tokenizeHtmlTags(raw).next().value;
|
||||
const underlineTag =
|
||||
tag && tag.start === 0 && (tag.name === "u" || tag.name === "ins") ? tag : undefined;
|
||||
if (!silent) {
|
||||
const token = state.push(
|
||||
!underlineTag || underlineTag.selfClosing
|
||||
? "text"
|
||||
: underlineTag.closing
|
||||
? "underline_close"
|
||||
: "underline_open",
|
||||
"",
|
||||
0,
|
||||
);
|
||||
if (!underlineTag || underlineTag.selfClosing) {
|
||||
token.content = raw;
|
||||
}
|
||||
}
|
||||
state.pos += raw.length;
|
||||
return true;
|
||||
}
|
||||
|
||||
function getAttr(token: MarkdownToken, name: string): string | null {
|
||||
if (token.attrGet) {
|
||||
return token.attrGet(name);
|
||||
@@ -387,16 +470,35 @@ function appendNestedListSeparator(state: RenderState) {
|
||||
}
|
||||
}
|
||||
|
||||
function appendListPrefix(state: RenderState) {
|
||||
function appendListPrefix(state: RenderState, isTask: boolean): MarkdownListItemMarker | undefined {
|
||||
const stack = state.env.listStack;
|
||||
const top = stack[stack.length - 1];
|
||||
if (!top) {
|
||||
return;
|
||||
return undefined;
|
||||
}
|
||||
top.index += 1;
|
||||
const indent = " ".repeat(Math.max(0, stack.length - 1));
|
||||
const prefix = top.type === "ordered" ? `${top.index}. ` : "• ";
|
||||
state.text += `${indent}${prefix}`;
|
||||
state.text += indent;
|
||||
const start = state.text.length;
|
||||
state.text += prefix;
|
||||
return {
|
||||
kind: top.type,
|
||||
listMarker: { start, end: state.text.length },
|
||||
...(isTask ? { task: true as const } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function recordTaskMarker(state: RenderState, content: string): void {
|
||||
const item = state.openListItems.at(-1);
|
||||
if (!item?.task || item.taskMarker) {
|
||||
return;
|
||||
}
|
||||
const marker = /^\[[ xX]\][ \t]?/u.exec(content)?.[0];
|
||||
if (marker) {
|
||||
const start = resolveRenderTarget(state).text.length;
|
||||
item.taskMarker = { start, end: start + marker.length };
|
||||
}
|
||||
}
|
||||
|
||||
function renderInlineCode(state: RenderState, content: string) {
|
||||
@@ -770,8 +872,15 @@ function renderTokens(tokens: MarkdownToken[], state: RenderState): void {
|
||||
}
|
||||
break;
|
||||
case "text":
|
||||
recordTaskMarker(state, token.content ?? "");
|
||||
appendText(state, token.content ?? "");
|
||||
break;
|
||||
case "underline_open":
|
||||
openStyle(state, "underline");
|
||||
break;
|
||||
case "underline_close":
|
||||
closeStyle(state, "underline");
|
||||
break;
|
||||
case ASSISTANT_TRANSCRIPT_ROLE_NODE_TYPE: {
|
||||
const meta = (token.meta as AssistantTranscriptRoleTokenMeta | undefined)
|
||||
?.assistantTranscriptRoleHeader;
|
||||
@@ -906,15 +1015,32 @@ function renderTokens(tokens: MarkdownToken[], state: RenderState): void {
|
||||
appendTopLevelListSeparator(state);
|
||||
}
|
||||
break;
|
||||
case "list_item_open":
|
||||
appendListPrefix(state);
|
||||
case "list_item_open": {
|
||||
const leadingInline =
|
||||
tokens[tokenIndex + 1]?.type === "paragraph_open" ? tokens[tokenIndex + 2] : undefined;
|
||||
const isTask = leadingInline?.type === "inline" && leadingInline.taskListMarker === true;
|
||||
const item = appendListPrefix(state, isTask);
|
||||
if (item) {
|
||||
state.openListItems.push(item);
|
||||
}
|
||||
break;
|
||||
case "list_item_close":
|
||||
}
|
||||
case "list_item_close": {
|
||||
const item = state.openListItems.pop();
|
||||
if (item) {
|
||||
state.listItems.push({
|
||||
kind: item.kind,
|
||||
...(item.listMarker ? { listMarker: item.listMarker } : {}),
|
||||
...(item.task ? { task: true } : {}),
|
||||
...(item.taskMarker ? { taskMarker: item.taskMarker } : {}),
|
||||
});
|
||||
}
|
||||
// Avoid double newlines (nested list's last item already added newline)
|
||||
if (!state.text.endsWith("\n")) {
|
||||
state.text += "\n";
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "code_block":
|
||||
case "fence":
|
||||
renderCodeBlock(
|
||||
@@ -1026,14 +1152,40 @@ function closeRemainingStyles(target: RenderTarget) {
|
||||
target.openStyles = [];
|
||||
}
|
||||
|
||||
function sliceListMarker(
|
||||
marker: { start: number; end: number },
|
||||
start: number,
|
||||
end: number,
|
||||
): { start: number; end: number } | undefined {
|
||||
const sliceStart = Math.max(marker.start, start);
|
||||
const sliceEnd = Math.min(marker.end, end);
|
||||
return sliceEnd > sliceStart ? { start: sliceStart - start, end: sliceEnd - start } : undefined;
|
||||
}
|
||||
|
||||
export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): MarkdownIR {
|
||||
const annotations = sliceAnnotationSpans(ir.annotations ?? [], start, end);
|
||||
return {
|
||||
const listItems = (ir.listItems ?? []).flatMap((item) => {
|
||||
const listMarker = item.listMarker ? sliceListMarker(item.listMarker, start, end) : undefined;
|
||||
const taskMarker = item.taskMarker ? sliceListMarker(item.taskMarker, start, end) : undefined;
|
||||
return listMarker || taskMarker
|
||||
? [
|
||||
{
|
||||
kind: item.kind,
|
||||
...(listMarker ? { listMarker } : {}),
|
||||
...(item.task ? { task: true as const } : {}),
|
||||
...(taskMarker ? { taskMarker } : {}),
|
||||
},
|
||||
]
|
||||
: [];
|
||||
});
|
||||
const sliced: MarkdownIR = {
|
||||
text: ir.text.slice(start, end),
|
||||
styles: sliceStyleSpans(ir.styles, start, end),
|
||||
links: sliceLinkSpans(ir.links, start, end),
|
||||
...(annotations.length > 0 ? { annotations } : {}),
|
||||
...(listItems.length > 0 ? { listItems } : {}),
|
||||
};
|
||||
return sliced;
|
||||
}
|
||||
|
||||
export function markdownToIR(markdown: string, options: MarkdownParseOptions = {}): MarkdownIR {
|
||||
@@ -1072,6 +1224,8 @@ export function markdownToIRWithMeta(
|
||||
horizontalRuleText: options.horizontalRuleText ?? "───",
|
||||
preserveSourceBlockSpacing: options.preserveSourceBlockSpacing ?? false,
|
||||
headingLineEnd: undefined,
|
||||
listItems: [],
|
||||
openListItems: [],
|
||||
};
|
||||
|
||||
renderTokens(tokens as MarkdownToken[], state);
|
||||
@@ -1092,14 +1246,34 @@ export function markdownToIRWithMeta(
|
||||
const finalText =
|
||||
finalLength === state.text.length ? state.text : state.text.slice(0, finalLength);
|
||||
const annotations = mergeAnnotationSpans(clampAnnotationSpans(state.annotations, finalLength));
|
||||
const listItems = state.listItems.flatMap((item) => {
|
||||
const listMarker = item.listMarker
|
||||
? sliceListMarker(item.listMarker, 0, finalLength)
|
||||
: undefined;
|
||||
const taskMarker = item.taskMarker
|
||||
? sliceListMarker(item.taskMarker, 0, finalLength)
|
||||
: undefined;
|
||||
return listMarker || taskMarker
|
||||
? [
|
||||
{
|
||||
kind: item.kind,
|
||||
...(listMarker ? { listMarker } : {}),
|
||||
...(item.task ? { task: true as const } : {}),
|
||||
...(taskMarker ? { taskMarker } : {}),
|
||||
},
|
||||
]
|
||||
: [];
|
||||
});
|
||||
|
||||
const ir: MarkdownIR = {
|
||||
text: finalText,
|
||||
styles: mergeStyleSpans(clampStyleSpans(state.styles, finalLength)),
|
||||
links: clampLinkSpans(state.links, finalLength),
|
||||
...(annotations.length > 0 ? { annotations } : {}),
|
||||
...(listItems.length > 0 ? { listItems } : {}),
|
||||
};
|
||||
return {
|
||||
ir: {
|
||||
text: finalText,
|
||||
styles: mergeStyleSpans(clampStyleSpans(state.styles, finalLength)),
|
||||
links: clampLinkSpans(state.links, finalLength),
|
||||
...(annotations.length > 0 ? { annotations } : {}),
|
||||
},
|
||||
ir,
|
||||
hasTables: state.hasTables,
|
||||
tables: state.collectedTables.map((table) =>
|
||||
Object.assign({}, table, {
|
||||
@@ -1132,13 +1306,9 @@ export function chunkMarkdownIR(ir: MarkdownIR, limit: number): MarkdownIR[] {
|
||||
}
|
||||
const start = cursor;
|
||||
const end = Math.min(ir.text.length, start + chunk.length);
|
||||
const annotations = sliceAnnotationSpans(ir.annotations ?? [], start, end);
|
||||
results.push({
|
||||
text: chunk,
|
||||
styles: sliceStyleSpans(ir.styles, start, end),
|
||||
links: sliceLinkSpans(ir.links, start, end),
|
||||
...(annotations.length > 0 ? { annotations } : {}),
|
||||
});
|
||||
const sliced = sliceMarkdownIR(ir, start, end);
|
||||
sliced.text = chunk;
|
||||
results.push(sliced);
|
||||
cursor = end;
|
||||
});
|
||||
|
||||
|
||||
42
packages/markdown-core/src/ir.underline.test.ts
Normal file
42
packages/markdown-core/src/ir.underline.test.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { markdownToIR } from "./ir.js";
|
||||
|
||||
describe("markdownToIR authored HTML underline", () => {
|
||||
it("emits underline spans only for authored u and ins tags when enabled", () => {
|
||||
const ir = markdownToIR("<u>under <ins>nested</ins></u> and __bold__", {
|
||||
enableHtmlUnderline: true,
|
||||
});
|
||||
|
||||
expect(ir.text).toBe("under nested and bold");
|
||||
expect(ir.styles).toEqual([
|
||||
{ start: 0, end: 12, style: "underline" },
|
||||
{ start: 17, end: 21, style: "bold" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves raw HTML bytes unless underline parsing is explicitly enabled", () => {
|
||||
expect(markdownToIR("<u>under</u>").text).toBe("<u>under</u>");
|
||||
});
|
||||
|
||||
it("does not enable HTML block parsing for unrelated tags", () => {
|
||||
const ir = markdownToIR("<div>\n**bold**\n</div>", { enableHtmlUnderline: true });
|
||||
expect(ir.text).toBe("<div>\nbold\n</div>");
|
||||
expect(ir.styles).toEqual([{ start: 6, end: 10, style: "bold" }]);
|
||||
});
|
||||
|
||||
it("keeps entity-decoded and escaped underline tags literal", () => {
|
||||
for (const input of ["<u>text</u>", "\\<u>text\\</u>"]) {
|
||||
const ir = markdownToIR(input, { enableHtmlUnderline: true });
|
||||
expect(ir.text, input).toBe("<u>text</u>");
|
||||
expect(ir.styles, input).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not parse underline-shaped text inside other HTML lexemes", () => {
|
||||
for (const input of ['<span title="<u>">x</span>', "<!-- <u> -->x"]) {
|
||||
const ir = markdownToIR(input, { enableHtmlUnderline: true });
|
||||
expect(ir.text, input).toBe(input);
|
||||
expect(ir.styles, input).toEqual([]);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,5 @@
|
||||
import { applyConstructFallbacks } from "./construct-fallbacks.js";
|
||||
import type { FormatCapabilityProfile } from "./format-capabilities.js";
|
||||
import { isAutoLinkedMarkdownLink, type MarkdownAnnotationSpan } from "./ir-spans.js";
|
||||
import type { MarkdownIR, MarkdownLinkSpan, MarkdownStyle } from "./ir.js";
|
||||
|
||||
@@ -24,14 +26,16 @@ export type AttributedRenderOptions<TStyle extends string> = {
|
||||
export function renderMarkdownWithAttributedRanges<TStyle extends string>(
|
||||
ir: MarkdownIR,
|
||||
options: AttributedRenderOptions<TStyle>,
|
||||
profile?: FormatCapabilityProfile,
|
||||
): { text: string; ranges: AttributedRange<TStyle>[] } {
|
||||
const text = ir.text ?? "";
|
||||
const projected = profile ? applyConstructFallbacks(ir, profile) : ir;
|
||||
const text = projected.text ?? "";
|
||||
const insertions: Array<{ pos: number; length: number }> = [];
|
||||
let rendered = text;
|
||||
if (options.renderLink) {
|
||||
rendered = "";
|
||||
let cursor = 0;
|
||||
for (const link of [...ir.links].toSorted((a, b) => a.start - b.start)) {
|
||||
for (const link of [...projected.links].toSorted((a, b) => a.start - b.start)) {
|
||||
if (link.start < cursor) {
|
||||
continue;
|
||||
}
|
||||
@@ -48,11 +52,11 @@ export function renderMarkdownWithAttributedRanges<TStyle extends string>(
|
||||
}
|
||||
rendered = options.trimEnd ? rendered.trimEnd() : rendered;
|
||||
|
||||
const spans = ir.styles.flatMap((span) => {
|
||||
const spans = projected.styles.flatMap((span) => {
|
||||
const style = options.styleMap[span.style];
|
||||
return style === undefined ? [] : [{ start: span.start, end: span.end, style }];
|
||||
});
|
||||
for (const annotation of ir.annotations ?? []) {
|
||||
for (const annotation of projected.annotations ?? []) {
|
||||
const style = options.annotationStyleMap?.[annotation.type];
|
||||
if (style !== undefined) {
|
||||
spans.push({ start: annotation.start, end: annotation.end, style });
|
||||
|
||||
39
packages/markdown-core/src/render-plain.ts
Normal file
39
packages/markdown-core/src/render-plain.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { applyConstructFallbacks } from "./construct-fallbacks.js";
|
||||
import type { FormatCapabilityProfile } from "./format-capabilities.js";
|
||||
import type { MarkdownIR } from "./ir.js";
|
||||
|
||||
export type PlainRenderOptions = {
|
||||
linkStyle?: "label" | "label-and-url";
|
||||
};
|
||||
|
||||
/** Projects Markdown IR to plain text, optionally applying channel capability fallbacks. */
|
||||
export function renderMarkdownAsPlainText(
|
||||
ir: MarkdownIR,
|
||||
options: PlainRenderOptions = {},
|
||||
profile?: FormatCapabilityProfile,
|
||||
): string {
|
||||
const effectiveProfile =
|
||||
profile && options.linkStyle === "label"
|
||||
? { ...profile, constructs: { ...profile.constructs, linkLabel: "strip" as const } }
|
||||
: profile;
|
||||
const projected = effectiveProfile ? applyConstructFallbacks(ir, effectiveProfile) : ir;
|
||||
if ((options.linkStyle ?? "label-and-url") === "label" || projected.links.length === 0) {
|
||||
return projected.text;
|
||||
}
|
||||
let output = "";
|
||||
let cursor = 0;
|
||||
for (const link of [...projected.links].toSorted((a, b) => a.start - b.start)) {
|
||||
if (link.start < cursor) {
|
||||
continue;
|
||||
}
|
||||
output += projected.text.slice(cursor, link.end);
|
||||
const href = link.href.trim();
|
||||
const label = projected.text.slice(link.start, link.end).trim();
|
||||
const comparableHref = href.startsWith("mailto:") ? href.slice("mailto:".length) : href;
|
||||
if (href && label && label !== href && label !== comparableHref) {
|
||||
output += ` (${href})`;
|
||||
}
|
||||
cursor = link.end;
|
||||
}
|
||||
return output + projected.text.slice(cursor);
|
||||
}
|
||||
87
packages/markdown-core/src/render.capabilities.test.ts
Normal file
87
packages/markdown-core/src/render.capabilities.test.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { FormatCapabilityProfile } from "./format-capabilities.js";
|
||||
import { markdownToIR } from "./ir.js";
|
||||
import { renderMarkdownWithAttributedRanges } from "./render-attributed.js";
|
||||
import { renderMarkdownAsPlainText } from "./render-plain.js";
|
||||
import { renderMarkdownWithMarkers } from "./render.js";
|
||||
|
||||
const ALL_NATIVE = {
|
||||
mechanism: "markdown",
|
||||
constructs: {
|
||||
bold: "native",
|
||||
italic: "native",
|
||||
underline: "native",
|
||||
strikethrough: "native",
|
||||
spoiler: "native",
|
||||
codeInline: "native",
|
||||
codeBlock: "native",
|
||||
codeLanguage: "native",
|
||||
linkLabel: "native",
|
||||
heading: "native",
|
||||
bulletList: "native",
|
||||
orderedList: "native",
|
||||
taskList: "native",
|
||||
table: "native",
|
||||
blockquote: "native",
|
||||
image: "native",
|
||||
mention: "native",
|
||||
},
|
||||
chunk: { limit: 4_000, unit: "chars" },
|
||||
} satisfies FormatCapabilityProfile;
|
||||
|
||||
describe("format capability driver plumbing", () => {
|
||||
const ir = markdownToIR("**See [docs](https://example.com)**", { headingStyle: "rich" });
|
||||
|
||||
it("keeps marker rendering byte-identical for an all-native optional profile", () => {
|
||||
const options = {
|
||||
styleMarkers: { bold: { open: "<b>", close: "</b>" } },
|
||||
escapeText: (text: string) => text,
|
||||
buildLink: (link: { start: number; end: number; href: string }) => ({
|
||||
start: link.start,
|
||||
end: link.end,
|
||||
open: "<a>",
|
||||
close: "</a>",
|
||||
}),
|
||||
};
|
||||
expect(renderMarkdownWithMarkers(ir, options, ALL_NATIVE)).toBe(
|
||||
renderMarkdownWithMarkers(ir, options),
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves caller-constructed style spans for an all-native profile", () => {
|
||||
const customIr = {
|
||||
text: "same",
|
||||
styles: [
|
||||
{ start: 0, end: 2, style: "bold" as const },
|
||||
{ start: 2, end: 4, style: "bold" as const },
|
||||
],
|
||||
links: [],
|
||||
};
|
||||
const options = {
|
||||
styleMarkers: { bold: { open: "*", close: "*" } },
|
||||
escapeText: (text: string) => text,
|
||||
};
|
||||
expect(renderMarkdownWithMarkers(customIr, options, ALL_NATIVE)).toBe(
|
||||
renderMarkdownWithMarkers(customIr, options),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps attributed rendering byte-identical for an all-native optional profile", () => {
|
||||
const options = { styleMap: { bold: "strong" as const }, renderLink: () => " (url)" };
|
||||
expect(renderMarkdownWithAttributedRanges(ir, options, ALL_NATIVE)).toEqual(
|
||||
renderMarkdownWithAttributedRanges(ir, options),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps plain projection byte-identical for an all-native optional profile", () => {
|
||||
expect(renderMarkdownAsPlainText(ir, {}, ALL_NATIVE)).toBe(renderMarkdownAsPlainText(ir));
|
||||
});
|
||||
|
||||
it("keeps explicit label-only link projection above profile fallback", () => {
|
||||
const profile = {
|
||||
...ALL_NATIVE,
|
||||
constructs: { ...ALL_NATIVE.constructs, linkLabel: "fallback" as const },
|
||||
};
|
||||
expect(renderMarkdownAsPlainText(ir, { linkStyle: "label" }, profile)).toBe("See docs");
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,5 @@
|
||||
import { applyConstructFallbacks } from "./construct-fallbacks.js";
|
||||
import type { FormatCapabilityProfile } from "./format-capabilities.js";
|
||||
import {
|
||||
copyMarkdownLinkSpan,
|
||||
isAutoLinkedMarkdownLink,
|
||||
@@ -63,6 +65,7 @@ const STYLE_ORDER: MarkdownStyle[] = [
|
||||
"heading_6",
|
||||
"bold",
|
||||
"italic",
|
||||
"underline",
|
||||
"strikethrough",
|
||||
"spoiler",
|
||||
];
|
||||
@@ -191,8 +194,13 @@ function sortAnnotationSpans(spans: MarkdownAnnotationSpan[]): MarkdownAnnotatio
|
||||
}
|
||||
|
||||
/** Renders Markdown IR by nesting configured style markers and optional link markers. */
|
||||
export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions): string {
|
||||
const text = ir.text ?? "";
|
||||
export function renderMarkdownWithMarkers(
|
||||
ir: MarkdownIR,
|
||||
options: RenderOptions,
|
||||
profile?: FormatCapabilityProfile,
|
||||
): string {
|
||||
const projected = profile ? applyConstructFallbacks(ir, profile) : ir;
|
||||
const text = projected.text ?? "";
|
||||
if (!text) {
|
||||
return "";
|
||||
}
|
||||
@@ -200,7 +208,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions
|
||||
const styleMarkers = options.styleMarkers;
|
||||
const annotationMarkers = options.annotationMarkers ?? {};
|
||||
const annotated = sortAnnotationSpans(
|
||||
(ir.annotations ?? []).filter((span) => Boolean(annotationMarkers[span.type])),
|
||||
(projected.annotations ?? []).filter((span) => Boolean(annotationMarkers[span.type])),
|
||||
);
|
||||
const dominantAnnotations = annotated.filter(
|
||||
(span) => annotationMarkers[span.type]?.suppressNestedFormatting === true,
|
||||
@@ -210,7 +218,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions
|
||||
...new Set(annotated.flatMap((span) => [span.start, span.end])),
|
||||
].toSorted((a, b) => a - b);
|
||||
const styled = sortStyleSpans(
|
||||
ir.styles
|
||||
projected.styles
|
||||
.filter((span) => Boolean(styleMarkers[span.style]))
|
||||
.flatMap((span) => {
|
||||
if (STRUCTURAL_STYLES.has(span.style)) {
|
||||
@@ -266,7 +274,7 @@ export function renderMarkdownWithMarkers(ir: MarkdownIR, options: RenderOptions
|
||||
|
||||
const linkStarts = new Map<number, RenderLink[]>();
|
||||
if (options.buildLink) {
|
||||
const links = ir.links.flatMap((span) =>
|
||||
const links = projected.links.flatMap((span) =>
|
||||
subtractRanges(span, dominantAnnotationRanges)
|
||||
.flatMap((piece) => splitAtBoundaries(piece, annotationBoundaries))
|
||||
.map((piece) =>
|
||||
|
||||
@@ -152,7 +152,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
|
||||
// +2: generic inbound-root and SCP-host schema validators.
|
||||
// +2: attributed-range renderer and its options contract.
|
||||
// +1: agent-harness transcript visibility projector.
|
||||
4690,
|
||||
// +1: outbound formatting capability profile.
|
||||
4691,
|
||||
env,
|
||||
),
|
||||
publicFunctionExports: readPluginSdkSurfaceBudgetEnv(
|
||||
|
||||
35
src/plugin-sdk/format-capabilities.test.ts
Normal file
35
src/plugin-sdk/format-capabilities.test.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, expectTypeOf, it } from "vitest";
|
||||
import type { FormatCapabilityProfile } from "./text-chunking.js";
|
||||
|
||||
describe("FormatCapabilityProfile", () => {
|
||||
it("preserves literal inference with satisfies", () => {
|
||||
const profile = {
|
||||
mechanism: "ranges",
|
||||
constructs: {
|
||||
bold: "native",
|
||||
italic: "native",
|
||||
underline: "strip",
|
||||
strikethrough: "native",
|
||||
spoiler: "fallback",
|
||||
codeInline: "native",
|
||||
codeBlock: "native",
|
||||
codeLanguage: "strip",
|
||||
linkLabel: "fallback",
|
||||
heading: "fallback",
|
||||
bulletList: "fallback",
|
||||
orderedList: "fallback",
|
||||
taskList: "fallback",
|
||||
table: "strip",
|
||||
blockquote: "fallback",
|
||||
image: "strip",
|
||||
mention: "native",
|
||||
},
|
||||
chunk: { limit: 2_048, unit: "bytes", hardCap: 65_536 },
|
||||
} satisfies FormatCapabilityProfile;
|
||||
|
||||
expectTypeOf(profile.mechanism).toEqualTypeOf<"ranges">();
|
||||
expectTypeOf(profile.constructs.underline).toEqualTypeOf<"strip">();
|
||||
expectTypeOf(profile.chunk.unit).toEqualTypeOf<"bytes">();
|
||||
expect(profile.chunk.hardCap).toBe(65_536);
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,8 @@ export {
|
||||
} from "../../packages/markdown-core/src/chunk-text.js";
|
||||
/** Quote-aware HTML tag tokens for exact post-render projections. */
|
||||
export { tokenizeHtmlTags } from "../../packages/markdown-core/src/html-tags.js";
|
||||
/** Static outbound formatting capabilities declared by a channel plugin. */
|
||||
export type { FormatCapabilityProfile } from "../../packages/markdown-core/src/format-capabilities.js";
|
||||
|
||||
/**
|
||||
* Splits outbound channel text into chunks no longer than the requested limit.
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { findAssistantTranscriptRoleHeaderSpans } from "../../../packages/markdown-core/src/assistant-transcript-headers.js";
|
||||
import { applyConstructFallbacks } from "../../../packages/markdown-core/src/construct-fallbacks.js";
|
||||
import type { FormatCapabilityProfile } from "../../../packages/markdown-core/src/format-capabilities.js";
|
||||
import { markdownToIR, type MarkdownIR } from "../../../packages/markdown-core/src/ir.js";
|
||||
|
||||
type StripMarkdownOptions = {
|
||||
@@ -22,8 +24,7 @@ function collectLinkInsertions(
|
||||
options: StripMarkdownOptions,
|
||||
): PlainTextInsertion[] {
|
||||
const insertions: PlainTextInsertion[] = [];
|
||||
const linkStyle = options.linkStyle ?? "label-and-url";
|
||||
if (linkStyle === "label-and-url") {
|
||||
if ((options.linkStyle ?? "label-and-url") === "label-and-url") {
|
||||
for (const link of ir.links) {
|
||||
const href = link.href.trim();
|
||||
const label = ir.text.slice(link.start, link.end).trim();
|
||||
@@ -106,7 +107,11 @@ function cleanSpeechText(text: string): string {
|
||||
}
|
||||
|
||||
/** Parse Markdown, then protect role headers exposed by the final plain-text projection. */
|
||||
export function stripMarkdown(text: string, options: StripMarkdownOptions = {}): string {
|
||||
export function stripMarkdown(
|
||||
text: string,
|
||||
options: StripMarkdownOptions = {},
|
||||
profile?: FormatCapabilityProfile,
|
||||
): string {
|
||||
// The IR parser preserves links when role annotations are enabled so this
|
||||
// plain-text projection can still append explicit destinations. Direct rich
|
||||
// renderers suppress overlapping active links later at their own boundary.
|
||||
@@ -114,6 +119,8 @@ export function stripMarkdown(text: string, options: StripMarkdownOptions = {}):
|
||||
assistantTranscriptRoleHeaders: options.assistantTranscriptRoleHeaders,
|
||||
autolink: false,
|
||||
blockquotePrefix: "",
|
||||
enableHtmlUnderline: profile !== undefined,
|
||||
enableTaskLists: profile !== undefined,
|
||||
headingStyle: "none",
|
||||
horizontalRuleText: "",
|
||||
linkify: false,
|
||||
@@ -123,9 +130,14 @@ export function stripMarkdown(text: string, options: StripMarkdownOptions = {}):
|
||||
// Detect against the exact leading boundary transports receive. String.trim
|
||||
// removes Unicode whitespace that the transcript header grammar intentionally
|
||||
// does not treat as Markdown indentation.
|
||||
const plainText = applyPlainTextInsertions(ir.text, [
|
||||
...collectLinkInsertions(ir, options),
|
||||
...collectParsedAssistantTranscriptRoleInsertions(ir, options),
|
||||
const effectiveProfile =
|
||||
profile && options.linkStyle === "label"
|
||||
? { ...profile, constructs: { ...profile.constructs, linkLabel: "strip" as const } }
|
||||
: profile;
|
||||
const projectedIr = effectiveProfile ? applyConstructFallbacks(ir, effectiveProfile) : ir;
|
||||
const plainText = applyPlainTextInsertions(projectedIr.text, [
|
||||
...collectLinkInsertions(projectedIr, options),
|
||||
...collectParsedAssistantTranscriptRoleInsertions(projectedIr, options),
|
||||
]).trim();
|
||||
const projected = applyPlainTextInsertions(
|
||||
plainText,
|
||||
|
||||
@@ -1,7 +1,32 @@
|
||||
// TTS prepare text tests cover text cleanup before speech synthesis.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { FormatCapabilityProfile } from "../../packages/markdown-core/src/format-capabilities.js";
|
||||
import { stripMarkdown } from "../shared/text/strip-markdown.js";
|
||||
|
||||
const PLAIN_PROFILE = {
|
||||
mechanism: "plain",
|
||||
constructs: {
|
||||
bold: "strip",
|
||||
italic: "strip",
|
||||
underline: "strip",
|
||||
strikethrough: "strip",
|
||||
spoiler: "strip",
|
||||
codeInline: "strip",
|
||||
codeBlock: "strip",
|
||||
codeLanguage: "strip",
|
||||
linkLabel: "fallback",
|
||||
heading: "strip",
|
||||
bulletList: "native",
|
||||
orderedList: "native",
|
||||
taskList: "fallback",
|
||||
table: "strip",
|
||||
blockquote: "strip",
|
||||
image: "strip",
|
||||
mention: "strip",
|
||||
},
|
||||
chunk: { limit: 1_600, unit: "chars" },
|
||||
} satisfies FormatCapabilityProfile;
|
||||
|
||||
/**
|
||||
* Tests that stripMarkdown (used in the TTS pipeline via maybeApplyTtsToPayload)
|
||||
* produces clean text suitable for speech synthesis.
|
||||
@@ -46,6 +71,26 @@ describe("TTS text preparation – stripMarkdown", () => {
|
||||
).toBe("Read the download");
|
||||
});
|
||||
|
||||
it("keeps role-header prefixes aligned after labeled link expansion", () => {
|
||||
expect(
|
||||
stripMarkdown("[docs](https://example.com)\nuser[Thu] hello", {
|
||||
assistantTranscriptRoleHeaders: true,
|
||||
}),
|
||||
).toBe("docs (https://example.com)\n[assistant-authored transcript] user[Thu] hello");
|
||||
});
|
||||
|
||||
it("applies profile-aware task and authored-HTML fallbacks before projection", () => {
|
||||
expect(stripMarkdown("- [x] done\n\n<u>under</u>", {}, PLAIN_PROFILE)).toBe(
|
||||
"[x] done\n\nunder",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps explicit label-only links above profile fallback", () => {
|
||||
expect(
|
||||
stripMarkdown("Read [docs](https://example.com)", { linkStyle: "label" }, PLAIN_PROFILE),
|
||||
).toBe("Read docs");
|
||||
});
|
||||
|
||||
it("handles a typical LLM reply with mixed markdown", () => {
|
||||
const input = `## Heading with **bold** and *italic*
|
||||
|
||||
|
||||
Reference in New Issue
Block a user