refactor: extract markdown core package (#88265)

* refactor: extract markdown core package

* refactor: remove old markdown sources

* fix: use source paths for markdown core imports

* fix: clean markdown package dependency ownership

* fix: refresh root shrinkwrap for markdown dependency move
This commit is contained in:
Peter Steinberger
2026-05-30 09:33:24 +02:00
committed by GitHub
parent 0f8ea1d3d9
commit 99ffd714ce
61 changed files with 455 additions and 114 deletions

View File

@@ -180,6 +180,10 @@ const config = {
entry: ["src/index.ts!", "src/ip.ts!"],
project: ["src/**/*.ts!"],
},
"packages/markdown-core": {
entry: ["src/*.ts!"],
project: ["src/**/*.ts!"],
},
"packages/speech-core": {
entry: ["api.ts!", "runtime-api.ts!", "speaker.ts!", "voice-models.ts!"],
project: ["**/*.ts!"],

View File

@@ -102,6 +102,36 @@
"../dist/plugin-sdk/packages/llm-core/src/validation.d.ts"
],
"@openclaw/llm-core/*": ["../dist/plugin-sdk/packages/llm-core/src/*.d.ts"],
"@openclaw/markdown-core": [
"../dist/plugin-sdk/packages/markdown-core/src/index.d.ts"
],
"@openclaw/markdown-core/code-spans": [
"../dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts"
],
"@openclaw/markdown-core/fences": [
"../dist/plugin-sdk/packages/markdown-core/src/fences.d.ts"
],
"@openclaw/markdown-core/frontmatter": [
"../dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts"
],
"@openclaw/markdown-core/ir": [
"../dist/plugin-sdk/packages/markdown-core/src/ir.d.ts"
],
"@openclaw/markdown-core/render": [
"../dist/plugin-sdk/packages/markdown-core/src/render.d.ts"
],
"@openclaw/markdown-core/render-aware-chunking": [
"../dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts"
],
"@openclaw/markdown-core/tables": [
"../dist/plugin-sdk/packages/markdown-core/src/tables.d.ts"
],
"@openclaw/markdown-core/types": [
"../dist/plugin-sdk/packages/markdown-core/src/types.d.ts"
],
"@openclaw/markdown-core/*": [
"../dist/plugin-sdk/packages/markdown-core/src/*.d.ts"
],
"@openclaw/media-generation-core": [
"../dist/plugin-sdk/packages/media-generation-core/src/index.d.ts"
],

View File

@@ -111,6 +111,36 @@
"@openclaw/llm-core/*": [
"../../dist/plugin-sdk/packages/llm-core/src/*.d.ts"
],
"@openclaw/markdown-core": [
"../../dist/plugin-sdk/packages/markdown-core/src/index.d.ts"
],
"@openclaw/markdown-core/code-spans": [
"../../dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts"
],
"@openclaw/markdown-core/fences": [
"../../dist/plugin-sdk/packages/markdown-core/src/fences.d.ts"
],
"@openclaw/markdown-core/frontmatter": [
"../../dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts"
],
"@openclaw/markdown-core/ir": [
"../../dist/plugin-sdk/packages/markdown-core/src/ir.d.ts"
],
"@openclaw/markdown-core/render": [
"../../dist/plugin-sdk/packages/markdown-core/src/render.d.ts"
],
"@openclaw/markdown-core/render-aware-chunking": [
"../../dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts"
],
"@openclaw/markdown-core/tables": [
"../../dist/plugin-sdk/packages/markdown-core/src/tables.d.ts"
],
"@openclaw/markdown-core/types": [
"../../dist/plugin-sdk/packages/markdown-core/src/types.d.ts"
],
"@openclaw/markdown-core/*": [
"../../dist/plugin-sdk/packages/markdown-core/src/*.d.ts"
],
"@openclaw/media-generation-core": [
"../../dist/plugin-sdk/packages/media-generation-core/src/index.d.ts"
],

74
npm-shrinkwrap.json generated
View File

@@ -46,7 +46,6 @@
"jszip": "3.10.1",
"kysely": "0.29.2",
"linkedom": "0.18.12",
"markdown-it": "14.2.0",
"minimatch": "10.2.5",
"node-edge-tts": "1.2.10",
"openai": "6.39.0",
@@ -1053,12 +1052,6 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"license": "Python-2.0"
},
"node_modules/asn1.js": {
"version": "5.4.1",
"resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz",
@@ -2503,25 +2496,6 @@
}
}
},
"node_modules/linkify-it": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.1.tgz",
"integrity": "sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/puzrin"
},
{
"type": "github",
"url": "https://github.com/sponsors/markdown-it"
}
],
"license": "MIT",
"dependencies": {
"uc.micro": "^2.0.0"
}
},
"node_modules/locate-path": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
@@ -2549,33 +2523,6 @@
"node": "20 || >=22"
}
},
"node_modules/markdown-it": {
"version": "14.2.0",
"resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.2.0.tgz",
"integrity": "sha512-1TGiQiJVRQ3NPmZH6sx5Cfnmg6GQm9jvC1ch4TK511NjSJvjzKLzn5pPfZRNZkRPZP0HqCioSndqH8v2nRaWVQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/puzrin"
},
{
"type": "github",
"url": "https://github.com/sponsors/markdown-it"
}
],
"license": "MIT",
"dependencies": {
"argparse": "^2.0.1",
"entities": "^4.4.0",
"linkify-it": "^5.0.1",
"mdurl": "^2.0.0",
"punycode.js": "^2.3.1",
"uc.micro": "^2.1.0"
},
"bin": {
"markdown-it": "bin/markdown-it.mjs"
}
},
"node_modules/marked": {
"version": "15.0.12",
"resolved": "https://registry.npmjs.org/marked/-/marked-15.0.12.tgz",
@@ -2597,12 +2544,6 @@
"node": ">= 0.4"
}
},
"node_modules/mdurl": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
"integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==",
"license": "MIT"
},
"node_modules/media-typer": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz",
@@ -3048,15 +2989,6 @@
"node": ">= 0.10"
}
},
"node_modules/punycode.js": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
"integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/qrcode": {
"version": "1.5.4",
"resolved": "https://registry.npmjs.org/qrcode/-/qrcode-1.5.4.tgz",
@@ -3848,12 +3780,6 @@
"node": ">=14.17"
}
},
"node_modules/uc.micro": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz",
"integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
"license": "MIT"
},
"node_modules/uhyphen": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz",

View File

@@ -1882,7 +1882,6 @@
"jszip": "3.10.1",
"kysely": "0.29.2",
"linkedom": "0.18.12",
"markdown-it": "14.2.0",
"minimatch": "10.2.5",
"node-edge-tts": "1.2.10",
"openai": "6.39.0",

View File

@@ -0,0 +1,62 @@
{
"name": "@openclaw/markdown-core",
"version": "0.0.0-private",
"private": true,
"files": [
"dist"
],
"type": "module",
"main": "./dist/index.mjs",
"types": "./dist/index.d.mts",
"exports": {
".": {
"types": "./dist/index.d.mts",
"import": "./dist/index.mjs",
"default": "./dist/index.mjs"
},
"./code-spans": {
"types": "./dist/code-spans.d.mts",
"import": "./dist/code-spans.mjs",
"default": "./dist/code-spans.mjs"
},
"./fences": {
"types": "./dist/fences.d.mts",
"import": "./dist/fences.mjs",
"default": "./dist/fences.mjs"
},
"./frontmatter": {
"types": "./dist/frontmatter.d.mts",
"import": "./dist/frontmatter.mjs",
"default": "./dist/frontmatter.mjs"
},
"./ir": {
"types": "./dist/ir.d.mts",
"import": "./dist/ir.mjs",
"default": "./dist/ir.mjs"
},
"./render": {
"types": "./dist/render.d.mts",
"import": "./dist/render.mjs",
"default": "./dist/render.mjs"
},
"./render-aware-chunking": {
"types": "./dist/render-aware-chunking.d.mts",
"import": "./dist/render-aware-chunking.mjs",
"default": "./dist/render-aware-chunking.mjs"
},
"./tables": {
"types": "./dist/tables.d.mts",
"import": "./dist/tables.mjs",
"default": "./dist/tables.mjs"
},
"./types": {
"types": "./dist/types.d.mts",
"import": "./dist/types.mjs",
"default": "./dist/types.mjs"
}
},
"dependencies": {
"markdown-it": "14.2.0",
"yaml": "2.9.0"
}
}

View File

@@ -0,0 +1,67 @@
function resolveChunkEarlyReturn(text: string, limit: number): string[] | undefined {
if (!text) {
return [];
}
if (limit <= 0) {
return [text];
}
if (text.length <= limit) {
return [text];
}
return undefined;
}
function scanParenAwareBreakpoints(text: string): { lastNewline: number; lastWhitespace: number } {
let lastNewline = -1;
let lastWhitespace = -1;
let depth = 0;
for (let i = 0; i < text.length; i++) {
const char = text[i];
if (char === "(") {
depth += 1;
continue;
}
if (char === ")" && depth > 0) {
depth -= 1;
continue;
}
if (depth !== 0) {
continue;
}
if (char === "\n") {
lastNewline = i;
} else if (/\s/.test(char)) {
lastWhitespace = i;
}
}
return { lastNewline, lastWhitespace };
}
export function chunkText(text: string, limit: number): string[] {
const early = resolveChunkEarlyReturn(text, limit);
if (early) {
return early;
}
const chunks: string[] = [];
let cursor = 0;
while (cursor < text.length) {
if (text.length - cursor <= limit) {
chunks.push(text.slice(cursor));
break;
}
const windowEnd = Math.min(text.length, cursor + limit);
const window = text.slice(cursor, windowEnd);
const { lastNewline, lastWhitespace } = scanParenAwareBreakpoints(window);
const breakOffset = lastNewline > 0 ? lastNewline : lastWhitespace;
const end = breakOffset > 0 ? cursor + breakOffset : windowEnd;
chunks.push(text.slice(cursor, end));
cursor = end;
while (cursor < text.length && /\s/.test(text[cursor] ?? "")) {
cursor += 1;
}
}
return chunks;
}

View File

@@ -0,0 +1,8 @@
export * from "./code-spans.js";
export * from "./fences.js";
export * from "./frontmatter.js";
export * from "./ir.js";
export * from "./render-aware-chunking.js";
export * from "./render.js";
export * from "./tables.js";
export * from "./types.js";

View File

@@ -0,0 +1,14 @@
import { describe, expect, it } from "vitest";
import { chunkMarkdownIR, type MarkdownIR } from "./ir.js";
describe("chunkMarkdownIR", () => {
it("keeps the final in-limit remainder together after a soft break", () => {
const ir: MarkdownIR = {
text: "abcdefgh ij kl",
styles: [],
links: [],
};
expect(chunkMarkdownIR(ir, 10).map((chunk) => chunk.text)).toEqual(["abcdefgh", "ij kl"]);
});
});

View File

@@ -1,6 +1,6 @@
import MarkdownIt from "markdown-it";
import { chunkText } from "../auto-reply/chunk.js";
import type { MarkdownTableMode } from "../config/types.base.js";
import { chunkText } from "./chunk-text.js";
import type { MarkdownTableMode } from "./types.js";
type ListState = {
type: "bullet" | "ordered";

View File

@@ -1,4 +1,3 @@
import { resolveIntegerOption } from "../shared/number-coercion.js";
import {
chunkMarkdownIR,
sliceMarkdownIR,
@@ -24,6 +23,13 @@ type RenderResolver<TRendered> = Pick<
"measureRendered" | "renderChunk"
>;
function resolveIntegerOption(value: number, fallback: number, opts: { min: number }): number {
if (!Number.isFinite(value)) {
return fallback;
}
return Math.max(opts.min, Math.trunc(value));
}
export function renderMarkdownIRChunksWithinLimit<TRendered>(
options: RenderMarkdownIRChunksWithinLimitOptions<TRendered>,
): RenderedMarkdownChunk<TRendered>[] {

View File

@@ -1,6 +1,6 @@
import type { MarkdownTableMode } from "../config/types.base.js";
import { markdownToIRWithMeta } from "./ir.js";
import { renderMarkdownWithMarkers } from "./render.js";
import type { MarkdownTableMode } from "./types.js";
const MARKDOWN_STYLE_MARKERS = {
bold: { open: "**", close: "**" },

View File

@@ -0,0 +1 @@
export type MarkdownTableMode = "off" | "bullets" | "code" | "block";

View File

@@ -13,6 +13,7 @@
"tsBuildInfoFile": "dist/.tsbuildinfo"
},
"include": [
"../../packages/markdown-core/src/**/*.ts",
"../../packages/media-generation-core/src/**/*.ts",
"../../src/plugin-sdk/**/*.ts",
"../../src/video-generation/dashscope-compatible.ts",

12
pnpm-lock.yaml generated
View File

@@ -149,9 +149,6 @@ importers:
linkedom:
specifier: 0.18.12
version: 0.18.12
markdown-it:
specifier: 14.2.0
version: 14.2.0
minimatch:
specifier: 10.2.5
version: 10.2.5
@@ -1824,6 +1821,15 @@ importers:
specifier: workspace:*
version: link:../llm-core
packages/markdown-core:
dependencies:
markdown-it:
specifier: 14.2.0
version: 14.2.0
yaml:
specifier: 2.9.0
version: 2.9.0
packages/media-generation-core: {}
packages/memory-host-sdk: {}

View File

@@ -47,11 +47,13 @@ export const BUILD_ALL_STEPS = [
"npm-shrinkwrap.json",
"packages/plugin-sdk/package.json",
"packages/llm-core/package.json",
"packages/markdown-core/package.json",
"packages/memory-host-sdk/package.json",
"tsconfig.json",
"tsconfig.plugin-sdk.dts.json",
"src/plugin-sdk",
"packages/llm-core/src",
"packages/markdown-core/src",
"packages/memory-host-sdk/src",
"packages/media-generation-core/src",
"src/types",

View File

@@ -59,6 +59,22 @@ export const EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS = {
"@openclaw/llm-core/types": ["../dist/plugin-sdk/packages/llm-core/src/types.d.ts"],
"@openclaw/llm-core/validation": ["../dist/plugin-sdk/packages/llm-core/src/validation.d.ts"],
"@openclaw/llm-core/*": ["../dist/plugin-sdk/packages/llm-core/src/*.d.ts"],
"@openclaw/markdown-core": ["../dist/plugin-sdk/packages/markdown-core/src/index.d.ts"],
"@openclaw/markdown-core/code-spans": [
"../dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts",
],
"@openclaw/markdown-core/fences": ["../dist/plugin-sdk/packages/markdown-core/src/fences.d.ts"],
"@openclaw/markdown-core/frontmatter": [
"../dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts",
],
"@openclaw/markdown-core/ir": ["../dist/plugin-sdk/packages/markdown-core/src/ir.d.ts"],
"@openclaw/markdown-core/render": ["../dist/plugin-sdk/packages/markdown-core/src/render.d.ts"],
"@openclaw/markdown-core/render-aware-chunking": [
"../dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts",
],
"@openclaw/markdown-core/tables": ["../dist/plugin-sdk/packages/markdown-core/src/tables.d.ts"],
"@openclaw/markdown-core/types": ["../dist/plugin-sdk/packages/markdown-core/src/types.d.ts"],
"@openclaw/markdown-core/*": ["../dist/plugin-sdk/packages/markdown-core/src/*.d.ts"],
"@openclaw/media-generation-core": [
"../dist/plugin-sdk/packages/media-generation-core/src/index.d.ts",
],

View File

@@ -16,6 +16,7 @@ const PLUGIN_SDK_TYPE_INPUTS = [
"src/plugin-sdk",
"src/auto-reply",
"packages/llm-core/src",
"packages/markdown-core/src",
"packages/memory-host-sdk/src",
"packages/media-generation-core/src",
"src/video-generation/dashscope-compatible.ts",
@@ -33,6 +34,15 @@ const ROOT_DTS_REQUIRED_OUTPUTS = [
"dist/plugin-sdk/packages/llm-core/src/utils/diagnostics.d.ts",
"dist/plugin-sdk/packages/llm-core/src/utils/event-stream.d.ts",
"dist/plugin-sdk/packages/llm-core/src/validation.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/fences.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/index.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/ir.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/render.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/tables.d.ts",
"dist/plugin-sdk/packages/markdown-core/src/types.d.ts",
"dist/plugin-sdk/packages/media-generation-core/src/capability-model-ref.d.ts",
"dist/plugin-sdk/packages/media-generation-core/src/catalog.d.ts",
"dist/plugin-sdk/packages/media-generation-core/src/index.d.ts",
@@ -46,6 +56,15 @@ const ROOT_DTS_REQUIRED_OUTPUTS = [
const PACKAGE_DTS_INPUTS = ["packages/plugin-sdk/tsconfig.json", ...PLUGIN_SDK_TYPE_INPUTS];
const PACKAGE_DTS_STAMP = "packages/plugin-sdk/dist/.boundary-dts.stamp";
const PACKAGE_DTS_REQUIRED_OUTPUTS = [
"packages/plugin-sdk/dist/packages/markdown-core/src/code-spans.d.ts",
"packages/plugin-sdk/dist/packages/markdown-core/src/fences.d.ts",
"packages/plugin-sdk/dist/packages/markdown-core/src/frontmatter.d.ts",
"packages/plugin-sdk/dist/packages/markdown-core/src/index.d.ts",
"packages/plugin-sdk/dist/packages/markdown-core/src/ir.d.ts",
"packages/plugin-sdk/dist/packages/markdown-core/src/render-aware-chunking.d.ts",
"packages/plugin-sdk/dist/packages/markdown-core/src/render.d.ts",
"packages/plugin-sdk/dist/packages/markdown-core/src/tables.d.ts",
"packages/plugin-sdk/dist/packages/markdown-core/src/types.d.ts",
"packages/plugin-sdk/dist/packages/media-generation-core/src/capability-model-ref.d.ts",
"packages/plugin-sdk/dist/packages/media-generation-core/src/catalog.d.ts",
"packages/plugin-sdk/dist/packages/media-generation-core/src/index.d.ts",

View File

@@ -10,6 +10,7 @@ const RUN_NODE_PACKAGE_SOURCE_ROOTS = [
// src/ so edits restart the same process that consumes them.
"packages/gateway-client/src",
"packages/gateway-protocol/src",
"packages/markdown-core/src",
"packages/media-generation-core/src",
"packages/net-policy/src",
];

View File

@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import * as fences from "../markdown/fences.js";
import * as fences from "../../packages/markdown-core/src/fences.js";
import { EmbeddedBlockChunker } from "./embedded-agent-block-chunker.js";
function createFlushOnParagraphChunker(params: { minChars: number; maxChars: number }) {

View File

@@ -1,5 +1,9 @@
import type { FenceSpan } from "../markdown/fences.js";
import { findFenceSpanAt, isSafeFenceBreak, parseFenceSpans } from "../markdown/fences.js";
import type { FenceSpan } from "../../packages/markdown-core/src/fences.js";
import {
findFenceSpanAt,
isSafeFenceBreak,
parseFenceSpans,
} from "../../packages/markdown-core/src/fences.js";
export type BlockReplyChunking = {
minChars: number;

View File

@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { createInlineCodeState } from "../markdown/code-spans.js";
import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js";
import { handleAgentEnd } from "./embedded-agent-subscribe.handlers.lifecycle.js";
import type { EmbeddedAgentSubscribeContext } from "./embedded-agent-subscribe.handlers.types.js";

View File

@@ -1,5 +1,5 @@
import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js";
import { emitAgentEvent } from "../infra/agent-events.js";
import { createInlineCodeState } from "../markdown/code-spans.js";
import { hasAcceptedSessionSpawn } from "./accepted-session-spawn.js";
import {
buildApiErrorObservationFields,

View File

@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js";
import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js";
import { createInlineCodeState } from "../markdown/code-spans.js";
import {
buildAssistantStreamData,
consumePendingAssistantReplyDirectivesIntoReply,

View File

@@ -1,4 +1,5 @@
import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload";
import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js";
import {
parseReplyDirectives,
type ReplyDirectiveParseResult,
@@ -7,7 +8,6 @@ import { splitTrailingDirective } from "../auto-reply/reply/streaming-directives
import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
import { emitAgentEvent } from "../infra/agent-events.js";
import type { AssistantMessage } from "../llm/types.js";
import { createInlineCodeState } from "../markdown/code-spans.js";
import { coerceChatContentText } from "../shared/chat-content.js";
import {
parseAssistantTextSignature,

View File

@@ -1,8 +1,8 @@
import type { InlineCodeState } from "../../packages/markdown-core/src/code-spans.js";
import type { FenceScanState } from "../../packages/markdown-core/src/fences.js";
import type { HeartbeatToolResponse } from "../auto-reply/heartbeat-tool-response.js";
import type { ReplyDirectiveParseResult } from "../auto-reply/reply/reply-directives.js";
import type { ReasoningLevel } from "../auto-reply/thinking.js";
import type { InlineCodeState } from "../markdown/code-spans.js";
import type { FenceScanState } from "../markdown/fences.js";
import type { HookRunner } from "../plugins/hooks.js";
import type { AcceptedSessionSpawn } from "./accepted-session-spawn.js";
import type { EmbeddedBlockChunker } from "./embedded-agent-block-chunker.js";

View File

@@ -1,3 +1,9 @@
import type { InlineCodeState } from "../../packages/markdown-core/src/code-spans.js";
import {
buildCodeSpanIndex,
createInlineCodeState,
} from "../../packages/markdown-core/src/code-spans.js";
import type { FenceScanState } from "../../packages/markdown-core/src/fences.js";
import { setReplyPayloadMetadata } from "../auto-reply/reply-payload.js";
import { parseReplyDirectives } from "../auto-reply/reply/reply-directives.js";
import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js";
@@ -5,9 +11,6 @@ import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
import { formatToolAggregate } from "../auto-reply/tool-meta.js";
import { emitAgentEvent } from "../infra/agent-events.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import type { InlineCodeState } from "../markdown/code-spans.js";
import { buildCodeSpanIndex, createInlineCodeState } from "../markdown/code-spans.js";
import type { FenceScanState } from "../markdown/fences.js";
import { normalizeOptionalString } from "../shared/string-coerce.js";
import { findFinalTagMatches } from "../shared/text/final-tags.js";
import { hasOrphanReasoningCloseBoundary } from "../shared/text/reasoning-tags.js";

View File

@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import * as fences from "../markdown/fences.js";
import * as fences from "../../packages/markdown-core/src/fences.js";
import { hasBalancedFences } from "../test-utils/chunk-test-helpers.js";
import {
chunkByNewline,

View File

@@ -2,10 +2,14 @@
// unintentionally breaking on newlines. Using [\s\S] keeps newlines inside
// the chunk so messages are only split when they truly exceed the limit.
import {
findFenceSpanAt,
isSafeFenceBreak,
parseFenceSpans,
} from "../../packages/markdown-core/src/fences.js";
import type { ChannelId } from "../channels/plugins/types.core.js";
import { resolveChannelStreamingChunkMode } from "../channels/streaming.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { findFenceSpanAt, isSafeFenceBreak, parseFenceSpans } from "../markdown/fences.js";
import { resolveAccountEntry } from "../routing/account-lookup.js";
import { normalizeAccountId } from "../routing/session-key.js";
import { chunkTextByBreakResolver } from "../shared/text-chunking.js";

View File

@@ -1,4 +1,4 @@
import { parseFenceSpans } from "../markdown/fences.js";
import { parseFenceSpans } from "../../packages/markdown-core/src/fences.js";
import { asFiniteNumber } from "../shared/number-coercion.js";
import { asOptionalRecord } from "../shared/record-coerce.js";

View File

@@ -1,4 +1,4 @@
import { parseFrontmatterBlock } from "../markdown/frontmatter.js";
import { parseFrontmatterBlock } from "../../packages/markdown-core/src/frontmatter.js";
import {
applyOpenClawManifestInstallCommonFields,
getFrontmatterString,

View File

@@ -132,6 +132,7 @@ describe("watch-node script", () => {
expect(watchPaths).toContain("extensions");
expect(watchPaths).toContain("packages/gateway-client/src");
expect(watchPaths).toContain("packages/gateway-protocol/src");
expect(watchPaths).toContain("packages/markdown-core/src");
expect(watchPaths).toContain("packages/media-generation-core/src");
expect(watchPaths).toContain("packages/net-policy/src");
expect(watchPaths).toContain("tsdown.config.ts");
@@ -141,6 +142,8 @@ describe("watch-node script", () => {
expect(watchOptions.ignored("packages/gateway-client/src/client.ts")).toBe(false);
expect(watchOptions.ignored("packages/gateway-client/src/client.test.ts")).toBe(true);
expect(watchOptions.ignored("packages/gateway-protocol/src/schema/cron.ts")).toBe(false);
expect(watchOptions.ignored("packages/markdown-core/src/ir.ts")).toBe(false);
expect(watchOptions.ignored("packages/markdown-core/src/ir.test.ts")).toBe(true);
expect(watchOptions.ignored("packages/media-generation-core/src/model-ref.ts")).toBe(false);
expect(watchOptions.ignored("packages/media-generation-core/src/model-ref.test.ts")).toBe(
true,

View File

@@ -10,7 +10,7 @@ import {
parseCanonicalIpAddress,
parseLooseIpAddress,
} from "@openclaw/net-policy/ip";
import { parseFenceSpans } from "../markdown/fences.js";
import { parseFenceSpans } from "../../packages/markdown-core/src/fences.js";
import { parseAudioTag } from "./audio-tags.js";
// Allow optional wrapping backticks and punctuation after the token; capture the core token.

View File

@@ -1,3 +1,3 @@
export { resolveMarkdownTableMode } from "../config/markdown-tables.js";
export { convertMarkdownTables } from "../markdown/tables.js";
export { convertMarkdownTables } from "../../packages/markdown-core/src/tables.js";
export type { MarkdownTableMode } from "../config/types.base.js";

View File

@@ -20,19 +20,19 @@ export {
type MarkdownStyle,
type MarkdownStyleSpan,
type MarkdownTableMeta,
} from "../markdown/ir.js";
} from "../../packages/markdown-core/src/ir.js";
export {
renderMarkdownIRChunksWithinLimit,
type RenderMarkdownIRChunksWithinLimitOptions,
} from "../markdown/render-aware-chunking.js";
} from "../../packages/markdown-core/src/render-aware-chunking.js";
export {
renderMarkdownWithMarkers,
type RenderLink,
type RenderOptions,
type RenderStyleMap,
type RenderStyleMarker,
} from "../markdown/render.js";
export { convertMarkdownTables } from "../markdown/tables.js";
} from "../../packages/markdown-core/src/render.js";
export { convertMarkdownTables } from "../../packages/markdown-core/src/tables.js";
export {
sanitizeAssistantVisibleText,
sanitizeAssistantVisibleTextWithOptions,

View File

@@ -8,10 +8,10 @@ export * from "../logging/diagnostic.js";
export * from "../logging/logger.js";
export * from "../logging/redact.js";
export * from "../logging/redact-identifier.js";
export * from "../markdown/ir.js";
export * from "../markdown/render-aware-chunking.js";
export * from "../markdown/render.js";
export * from "../markdown/tables.js";
export * from "../../packages/markdown-core/src/ir.js";
export * from "../../packages/markdown-core/src/render-aware-chunking.js";
export * from "../../packages/markdown-core/src/render.js";
export * from "../../packages/markdown-core/src/tables.js";
export * from "../shared/global-singleton.js";
export * from "../shared/record-coerce.js";
export * from "../shared/scoped-expiring-id-cache.js";

View File

@@ -1,8 +1,8 @@
import fs from "node:fs";
import path from "node:path";
import { parseFrontmatterBlock } from "../../packages/markdown-core/src/frontmatter.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { readRootJsonObjectSync } from "../infra/json-files.js";
import { parseFrontmatterBlock } from "../markdown/frontmatter.js";
import { isPathInsideWithRealpath } from "../security/scan-paths.js";
import {
normalizeOptionalLowercaseString,

View File

@@ -194,6 +194,7 @@ describe("opt-in extension package boundaries", () => {
expect(tsconfig.compilerOptions?.outDir).toBe("dist");
expect(tsconfig.compilerOptions?.rootDir).toBe("../..");
expect(tsconfig.include).toEqual([
"../../packages/markdown-core/src/**/*.ts",
"../../packages/media-generation-core/src/**/*.ts",
"../../src/plugin-sdk/**/*.ts",
"../../src/video-generation/dashscope-compatible.ts",

View File

@@ -1,3 +1,4 @@
import { convertMarkdownTables } from "../../../packages/markdown-core/src/tables.js";
import { resolveEffectiveMessagesConfig, resolveHumanDelayConfig } from "../../agents/identity.js";
import {
chunkByNewline,
@@ -68,7 +69,6 @@ import {
updateLastRoute,
} from "../../config/sessions.js";
import { getChannelActivity, recordChannelActivity } from "../../infra/channel-activity.js";
import { convertMarkdownTables } from "../../markdown/tables.js";
import {
fetchRemoteMedia,
readRemoteMediaBuffer,

View File

@@ -81,7 +81,7 @@ export type PluginRuntimeChannel = {
resolveTextChunkLimit: typeof import("../../auto-reply/chunk.js").resolveTextChunkLimit;
hasControlCommand: typeof import("../../auto-reply/command-detection.js").hasControlCommand;
resolveMarkdownTableMode: import("../../config/markdown-tables.types.js").ResolveMarkdownTableMode;
convertMarkdownTables: typeof import("../../markdown/tables.js").convertMarkdownTables;
convertMarkdownTables: typeof import("../../../packages/markdown-core/src/tables.js").convertMarkdownTables;
};
reply: {
dispatchReplyWithBufferedBlockDispatcher: DispatchReplyWithBufferedBlockDispatcher;

View File

@@ -1384,6 +1384,18 @@ describe("plugin sdk alias helpers", () => {
srcFile: "index.ts",
distFile: "index.mjs",
});
const markdownCore = writeWorkspacePackageEntry({
root: fixture.root,
packageDir: "markdown-core",
srcFile: "index.ts",
distFile: "index.mjs",
});
const markdownCoreTables = writeWorkspacePackageEntry({
root: fixture.root,
packageDir: "markdown-core",
srcFile: "tables.ts",
distFile: "tables.mjs",
});
const mediaGenerationModelRef = writeWorkspacePackageEntry({
root: fixture.root,
packageDir: "media-generation-core",
@@ -1400,6 +1412,8 @@ describe("plugin sdk alias helpers", () => {
fs.rmSync(gatewayClientTimeouts.distFile);
fs.rmSync(gatewayProtocol.distFile);
fs.rmSync(gatewayProtocolSchema.distFile);
fs.rmSync(markdownCore.distFile);
fs.rmSync(markdownCoreTables.distFile);
fs.rmSync(mediaGenerationCore.distFile);
fs.rmSync(mediaGenerationModelRef.distFile);
fs.rmSync(netPolicy.distFile);
@@ -1425,6 +1439,12 @@ describe("plugin sdk alias helpers", () => {
expect(fs.realpathSync(aliases["@openclaw/gateway-protocol/schema"] ?? "")).toBe(
fs.realpathSync(gatewayProtocolSchema.srcFile),
);
expect(fs.realpathSync(aliases["@openclaw/markdown-core"] ?? "")).toBe(
fs.realpathSync(markdownCore.srcFile),
);
expect(fs.realpathSync(aliases["@openclaw/markdown-core/tables"] ?? "")).toBe(
fs.realpathSync(markdownCoreTables.srcFile),
);
expect(fs.realpathSync(aliases["@openclaw/media-generation-core"] ?? "")).toBe(
fs.realpathSync(mediaGenerationCore.srcFile),
);
@@ -1459,6 +1479,12 @@ describe("plugin sdk alias helpers", () => {
srcFile: "catalog.ts",
distFile: "catalog.mjs",
});
const markdownCore = writeWorkspacePackageEntry({
root: fixture.root,
packageDir: "markdown-core",
srcFile: "render.ts",
distFile: "render.mjs",
});
const netPolicy = writeWorkspacePackageEntry({
root: fixture.root,
packageDir: "net-policy",
@@ -1480,6 +1506,9 @@ describe("plugin sdk alias helpers", () => {
expect(fs.realpathSync(aliases["@openclaw/gateway-protocol/connect-error-details"] ?? "")).toBe(
fs.realpathSync(gatewayProtocol.distFile),
);
expect(fs.realpathSync(aliases["@openclaw/markdown-core/render"] ?? "")).toBe(
fs.realpathSync(markdownCore.distFile),
);
expect(fs.realpathSync(aliases["@openclaw/media-generation-core/catalog"] ?? "")).toBe(
fs.realpathSync(mediaGenerationCore.distFile),
);

View File

@@ -560,6 +560,69 @@ const WORKSPACE_PACKAGE_ALIAS_ENTRIES = [
srcFile: "version.ts",
distFile: "version.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "",
srcFile: "index.ts",
distFile: "index.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "code-spans",
srcFile: "code-spans.ts",
distFile: "code-spans.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "fences",
srcFile: "fences.ts",
distFile: "fences.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "frontmatter",
srcFile: "frontmatter.ts",
distFile: "frontmatter.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "ir",
srcFile: "ir.ts",
distFile: "ir.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "render",
srcFile: "render.ts",
distFile: "render.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "render-aware-chunking",
srcFile: "render-aware-chunking.ts",
distFile: "render-aware-chunking.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "tables",
srcFile: "tables.ts",
distFile: "tables.mjs",
},
{
packageName: "@openclaw/markdown-core",
packageDir: "markdown-core",
subpath: "types",
srcFile: "types.ts",
distFile: "types.mjs",
},
{
packageName: "@openclaw/media-generation-core",
packageDir: "media-generation-core",

View File

@@ -1,5 +1,5 @@
import { parseFrontmatterBlock } from "../../../packages/markdown-core/src/frontmatter.js";
import { validateRegistryNpmSpec } from "../../infra/npm-registry-spec.js";
import { parseFrontmatterBlock } from "../../markdown/frontmatter.js";
import {
applyOpenClawManifestInstallCommonFields,
getFrontmatterString,

View File

@@ -175,7 +175,7 @@ describe("unit vitest config", () => {
expect(coverageInclude).toContain("src/commitments/runtime.ts");
expect(coverageInclude).toContain("src/media-generation/runtime-shared.ts");
expect(coverageInclude).toContain("src/web-search/runtime.ts");
expect(coverageInclude).not.toContain("src/markdown/render.ts");
expect(coverageInclude).not.toContain("packages/markdown-core/src/render.ts");
expect(coverageInclude).not.toContain("src/security/audit-workspace-skills.ts");
});

View File

@@ -31,7 +31,7 @@ const unitFastCandidateGlobs = [
"src/interactive/**/*.test.ts",
"src/link-understanding/**/*.test.ts",
"src/logging/**/*.test.ts",
"src/markdown/**/*.test.ts",
"packages/markdown-core/src/**/*.test.ts",
"src/media/**/*.test.ts",
"src/media-generation/**/*.test.ts",
"src/media-understanding/**/*.test.ts",

View File

@@ -65,6 +65,18 @@
"./packages/media-generation-core/src/normalization.ts"
],
"@openclaw/media-generation-core/*": ["./packages/media-generation-core/src/*"],
"@openclaw/markdown-core": ["./packages/markdown-core/src/index.ts"],
"@openclaw/markdown-core/code-spans": ["./packages/markdown-core/src/code-spans.ts"],
"@openclaw/markdown-core/fences": ["./packages/markdown-core/src/fences.ts"],
"@openclaw/markdown-core/frontmatter": ["./packages/markdown-core/src/frontmatter.ts"],
"@openclaw/markdown-core/ir": ["./packages/markdown-core/src/ir.ts"],
"@openclaw/markdown-core/render": ["./packages/markdown-core/src/render.ts"],
"@openclaw/markdown-core/render-aware-chunking": [
"./packages/markdown-core/src/render-aware-chunking.ts"
],
"@openclaw/markdown-core/tables": ["./packages/markdown-core/src/tables.ts"],
"@openclaw/markdown-core/types": ["./packages/markdown-core/src/types.ts"],
"@openclaw/markdown-core/*": ["./packages/markdown-core/src/*"],
"@openclaw/net-policy": ["./packages/net-policy/src/index.ts"],
"@openclaw/net-policy/ip": ["./packages/net-policy/src/ip.ts"],
"@openclaw/net-policy/ipv4": ["./packages/net-policy/src/ipv4.ts"],

View File

@@ -14,6 +14,7 @@
"include": [
"src/plugin-sdk/**/*.ts",
"packages/llm-core/src/**/*.ts",
"packages/markdown-core/src/**/*.ts",
"packages/media-generation-core/src/**/*.ts",
"packages/memory-host-sdk/src/**/*.ts",
"src/video-generation/dashscope-compatible.ts",

View File

@@ -394,6 +394,20 @@ function buildMediaGenerationCoreDistEntries(): Record<string, string> {
};
}
function buildMarkdownCoreDistEntries(): Record<string, string> {
return {
index: "packages/markdown-core/src/index.ts",
"code-spans": "packages/markdown-core/src/code-spans.ts",
fences: "packages/markdown-core/src/fences.ts",
frontmatter: "packages/markdown-core/src/frontmatter.ts",
ir: "packages/markdown-core/src/ir.ts",
render: "packages/markdown-core/src/render.ts",
"render-aware-chunking": "packages/markdown-core/src/render-aware-chunking.ts",
tables: "packages/markdown-core/src/tables.ts",
types: "packages/markdown-core/src/types.ts",
};
}
function buildSpeechCoreDistEntries(): Record<string, string> {
return {
api: "packages/speech-core/api.ts",
@@ -464,6 +478,12 @@ function shouldExternalizeLlmRuntimeDependency(id: string): boolean {
return id === "@openclaw/llm-core" || id.startsWith("@openclaw/llm-core/");
}
function shouldExternalizeMarkdownCoreDependency(id: string): boolean {
return (
id === "markdown-it" || id.startsWith("markdown-it/") || id === "yaml" || id.startsWith("yaml/")
);
}
const coreDistEntries = buildCoreDistEntries();
const dockerE2eHarnessEntries = buildDockerE2eHarnessEntries();
const rootBundledPluginBuildEntries = bundledPluginBuildEntries.filter(
@@ -539,6 +559,15 @@ export default defineConfig([
entry: buildMediaGenerationCoreDistEntries(),
outDir: "packages/media-generation-core/dist",
}),
nodeWorkspacePackageBuildConfig({
clean: true,
dts: RUN_NODE_SKIP_DTS_BUILD ? false : undefined,
entry: buildMarkdownCoreDistEntries(),
outDir: "packages/markdown-core/dist",
deps: {
neverBundle: shouldExternalizeMarkdownCoreDependency,
},
}),
nodeWorkspacePackageBuildConfig({
clean: true,
dts: RUN_NODE_SKIP_DTS_BUILD ? false : undefined,