From 99ffd714ce92ebe01fc5682d49887da88e761e89 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 30 May 2026 09:33:24 +0200 Subject: [PATCH] refactor: extract markdown core package (#88265) * refactor: extract markdown core package * refactor: remove old markdown sources * fix: use source paths for markdown core imports * fix: clean markdown package dependency ownership * fix: refresh root shrinkwrap for markdown dependency move --- config/knip.config.ts | 4 + .../tsconfig.package-boundary.paths.json | 30 ++++++++ extensions/xai/tsconfig.json | 30 ++++++++ npm-shrinkwrap.json | 74 ------------------- package.json | 1 - packages/markdown-core/package.json | 62 ++++++++++++++++ packages/markdown-core/src/chunk-text.ts | 67 +++++++++++++++++ .../markdown-core/src}/code-spans.ts | 0 .../markdown-core/src}/fences.ts | 0 .../markdown-core/src}/frontmatter.test.ts | 0 .../markdown-core/src}/frontmatter.ts | 0 packages/markdown-core/src/index.ts | 8 ++ .../src}/ir.blockquote-spacing.test.ts | 0 .../markdown-core/src/ir.chunking.test.ts | 14 ++++ .../markdown-core/src}/ir.hr-spacing.test.ts | 0 .../src}/ir.nested-lists.test.ts | 0 .../markdown-core/src}/ir.table-block.test.ts | 0 .../src}/ir.table-bullets.test.ts | 0 .../markdown-core/src}/ir.table-code.test.ts | 0 .../markdown-core/src}/ir.ts | 4 +- .../src}/render-aware-chunking.test.ts | 0 .../src}/render-aware-chunking.ts | 8 +- .../markdown-core/src}/render.ts | 0 .../markdown-core/src}/tables.test.ts | 0 .../markdown-core/src}/tables.ts | 2 +- packages/markdown-core/src/types.ts | 1 + packages/plugin-sdk/tsconfig.json | 1 + pnpm-lock.yaml | 12 ++- scripts/build-all.mjs | 2 + scripts/lib/extension-package-boundary.ts | 16 ++++ ...e-extension-package-boundary-artifacts.mjs | 19 +++++ scripts/run-node-watch-paths.mjs | 1 + .../embedded-agent-block-chunker.test.ts | 2 +- src/agents/embedded-agent-block-chunker.ts | 8 +- ...agent-subscribe.handlers.lifecycle.test.ts | 2 +- ...dded-agent-subscribe.handlers.lifecycle.ts | 2 +- ...-agent-subscribe.handlers.messages.test.ts | 2 +- ...edded-agent-subscribe.handlers.messages.ts | 2 +- ...embedded-agent-subscribe.handlers.types.ts | 4 +- src/agents/embedded-agent-subscribe.ts | 9 ++- src/auto-reply/chunk.test.ts | 2 +- src/auto-reply/chunk.ts | 6 +- src/chat/canvas-render.ts | 2 +- src/hooks/frontmatter.ts | 2 +- src/infra/watch-node.test.ts | 3 + src/media/parse.ts | 2 +- src/plugin-sdk/markdown-table-runtime.ts | 2 +- src/plugin-sdk/text-chunking.ts | 8 +- src/plugin-sdk/text-runtime.ts | 8 +- src/plugins/bundle-commands.ts | 2 +- ...tension-package-project-boundaries.test.ts | 1 + src/plugins/runtime/runtime-channel.ts | 2 +- src/plugins/runtime/types-channel.ts | 2 +- src/plugins/sdk-alias.test.ts | 29 ++++++++ src/plugins/sdk-alias.ts | 63 ++++++++++++++++ src/skills/loading/frontmatter.ts | 2 +- test/vitest-unit-config.test.ts | 2 +- test/vitest/vitest.unit-fast-paths.mjs | 2 +- tsconfig.json | 12 +++ tsconfig.plugin-sdk.dts.json | 1 + tsdown.config.ts | 29 ++++++++ 61 files changed, 455 insertions(+), 114 deletions(-) create mode 100644 packages/markdown-core/package.json create mode 100644 packages/markdown-core/src/chunk-text.ts rename {src/markdown => packages/markdown-core/src}/code-spans.ts (100%) rename {src/markdown => packages/markdown-core/src}/fences.ts (100%) rename {src/markdown => packages/markdown-core/src}/frontmatter.test.ts (100%) rename {src/markdown => packages/markdown-core/src}/frontmatter.ts (100%) create mode 100644 packages/markdown-core/src/index.ts rename {src/markdown => packages/markdown-core/src}/ir.blockquote-spacing.test.ts (100%) create mode 100644 packages/markdown-core/src/ir.chunking.test.ts rename {src/markdown => packages/markdown-core/src}/ir.hr-spacing.test.ts (100%) rename {src/markdown => packages/markdown-core/src}/ir.nested-lists.test.ts (100%) rename {src/markdown => packages/markdown-core/src}/ir.table-block.test.ts (100%) rename {src/markdown => packages/markdown-core/src}/ir.table-bullets.test.ts (100%) rename {src/markdown => packages/markdown-core/src}/ir.table-code.test.ts (100%) rename {src/markdown => packages/markdown-core/src}/ir.ts (99%) rename {src/markdown => packages/markdown-core/src}/render-aware-chunking.test.ts (100%) rename {src/markdown => packages/markdown-core/src}/render-aware-chunking.ts (97%) rename {src/markdown => packages/markdown-core/src}/render.ts (100%) rename {src/markdown => packages/markdown-core/src}/tables.test.ts (100%) rename {src/markdown => packages/markdown-core/src}/tables.ts (94%) create mode 100644 packages/markdown-core/src/types.ts diff --git a/config/knip.config.ts b/config/knip.config.ts index 6533c55e8ac..7f9931f89e1 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -180,6 +180,10 @@ const config = { entry: ["src/index.ts!", "src/ip.ts!"], project: ["src/**/*.ts!"], }, + "packages/markdown-core": { + entry: ["src/*.ts!"], + project: ["src/**/*.ts!"], + }, "packages/speech-core": { entry: ["api.ts!", "runtime-api.ts!", "speaker.ts!", "voice-models.ts!"], project: ["**/*.ts!"], diff --git a/extensions/tsconfig.package-boundary.paths.json b/extensions/tsconfig.package-boundary.paths.json index f4f195676bb..dacc50ade22 100644 --- a/extensions/tsconfig.package-boundary.paths.json +++ b/extensions/tsconfig.package-boundary.paths.json @@ -102,6 +102,36 @@ "../dist/plugin-sdk/packages/llm-core/src/validation.d.ts" ], "@openclaw/llm-core/*": ["../dist/plugin-sdk/packages/llm-core/src/*.d.ts"], + "@openclaw/markdown-core": [ + "../dist/plugin-sdk/packages/markdown-core/src/index.d.ts" + ], + "@openclaw/markdown-core/code-spans": [ + "../dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts" + ], + "@openclaw/markdown-core/fences": [ + "../dist/plugin-sdk/packages/markdown-core/src/fences.d.ts" + ], + "@openclaw/markdown-core/frontmatter": [ + "../dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts" + ], + "@openclaw/markdown-core/ir": [ + "../dist/plugin-sdk/packages/markdown-core/src/ir.d.ts" + ], + "@openclaw/markdown-core/render": [ + "../dist/plugin-sdk/packages/markdown-core/src/render.d.ts" + ], + "@openclaw/markdown-core/render-aware-chunking": [ + "../dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts" + ], + "@openclaw/markdown-core/tables": [ + "../dist/plugin-sdk/packages/markdown-core/src/tables.d.ts" + ], + "@openclaw/markdown-core/types": [ + "../dist/plugin-sdk/packages/markdown-core/src/types.d.ts" + ], + "@openclaw/markdown-core/*": [ + "../dist/plugin-sdk/packages/markdown-core/src/*.d.ts" + ], "@openclaw/media-generation-core": [ "../dist/plugin-sdk/packages/media-generation-core/src/index.d.ts" ], diff --git a/extensions/xai/tsconfig.json b/extensions/xai/tsconfig.json index b59b12cbf10..6f56799b27f 100644 --- a/extensions/xai/tsconfig.json +++ b/extensions/xai/tsconfig.json @@ -111,6 +111,36 @@ "@openclaw/llm-core/*": [ "../../dist/plugin-sdk/packages/llm-core/src/*.d.ts" ], + "@openclaw/markdown-core": [ + "../../dist/plugin-sdk/packages/markdown-core/src/index.d.ts" + ], + "@openclaw/markdown-core/code-spans": [ + "../../dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts" + ], + "@openclaw/markdown-core/fences": [ + "../../dist/plugin-sdk/packages/markdown-core/src/fences.d.ts" + ], + "@openclaw/markdown-core/frontmatter": [ + "../../dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts" + ], + "@openclaw/markdown-core/ir": [ + "../../dist/plugin-sdk/packages/markdown-core/src/ir.d.ts" + ], + "@openclaw/markdown-core/render": [ + "../../dist/plugin-sdk/packages/markdown-core/src/render.d.ts" + ], + "@openclaw/markdown-core/render-aware-chunking": [ + "../../dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts" + ], + "@openclaw/markdown-core/tables": [ + "../../dist/plugin-sdk/packages/markdown-core/src/tables.d.ts" + ], + "@openclaw/markdown-core/types": [ + "../../dist/plugin-sdk/packages/markdown-core/src/types.d.ts" + ], + "@openclaw/markdown-core/*": [ + "../../dist/plugin-sdk/packages/markdown-core/src/*.d.ts" + ], "@openclaw/media-generation-core": [ "../../dist/plugin-sdk/packages/media-generation-core/src/index.d.ts" ], diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 9f911cd1277..90eda437494 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -46,7 +46,6 @@ "jszip": "3.10.1", "kysely": "0.29.2", "linkedom": "0.18.12", - "markdown-it": "14.2.0", "minimatch": "10.2.5", "node-edge-tts": "1.2.10", "openai": "6.39.0", @@ -1053,12 +1052,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" - }, "node_modules/asn1.js": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", @@ -2503,25 +2496,6 @@ } } }, - "node_modules/linkify-it": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.1.tgz", - "integrity": "sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/markdown-it" - } - ], - "license": "MIT", - "dependencies": { - "uc.micro": "^2.0.0" - } - }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -2549,33 +2523,6 @@ "node": "20 || >=22" } }, - "node_modules/markdown-it": { - "version": "14.2.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.2.0.tgz", - "integrity": "sha512-1TGiQiJVRQ3NPmZH6sx5Cfnmg6GQm9jvC1ch4TK511NjSJvjzKLzn5pPfZRNZkRPZP0HqCioSndqH8v2nRaWVQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/markdown-it" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1", - "entities": "^4.4.0", - "linkify-it": "^5.0.1", - "mdurl": "^2.0.0", - "punycode.js": "^2.3.1", - "uc.micro": "^2.1.0" - }, - "bin": { - "markdown-it": "bin/markdown-it.mjs" - } - }, "node_modules/marked": { "version": "15.0.12", "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.12.tgz", @@ -2597,12 +2544,6 @@ "node": ">= 0.4" } }, - "node_modules/mdurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", - "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", - "license": "MIT" - }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -3048,15 +2989,6 @@ "node": ">= 0.10" } }, - "node_modules/punycode.js": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", - "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/qrcode": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/qrcode/-/qrcode-1.5.4.tgz", @@ -3848,12 +3780,6 @@ "node": ">=14.17" } }, - "node_modules/uc.micro": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", - "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", - "license": "MIT" - }, "node_modules/uhyphen": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz", diff --git a/package.json b/package.json index 4fc0e73a7c0..15c36eb1018 100644 --- a/package.json +++ b/package.json @@ -1882,7 +1882,6 @@ "jszip": "3.10.1", "kysely": "0.29.2", "linkedom": "0.18.12", - "markdown-it": "14.2.0", "minimatch": "10.2.5", "node-edge-tts": "1.2.10", "openai": "6.39.0", diff --git a/packages/markdown-core/package.json b/packages/markdown-core/package.json new file mode 100644 index 00000000000..73b2fb3e1e4 --- /dev/null +++ b/packages/markdown-core/package.json @@ -0,0 +1,62 @@ +{ + "name": "@openclaw/markdown-core", + "version": "0.0.0-private", + "private": true, + "files": [ + "dist" + ], + "type": "module", + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs", + "default": "./dist/index.mjs" + }, + "./code-spans": { + "types": "./dist/code-spans.d.mts", + "import": "./dist/code-spans.mjs", + "default": "./dist/code-spans.mjs" + }, + "./fences": { + "types": "./dist/fences.d.mts", + "import": "./dist/fences.mjs", + "default": "./dist/fences.mjs" + }, + "./frontmatter": { + "types": "./dist/frontmatter.d.mts", + "import": "./dist/frontmatter.mjs", + "default": "./dist/frontmatter.mjs" + }, + "./ir": { + "types": "./dist/ir.d.mts", + "import": "./dist/ir.mjs", + "default": "./dist/ir.mjs" + }, + "./render": { + "types": "./dist/render.d.mts", + "import": "./dist/render.mjs", + "default": "./dist/render.mjs" + }, + "./render-aware-chunking": { + "types": "./dist/render-aware-chunking.d.mts", + "import": "./dist/render-aware-chunking.mjs", + "default": "./dist/render-aware-chunking.mjs" + }, + "./tables": { + "types": "./dist/tables.d.mts", + "import": "./dist/tables.mjs", + "default": "./dist/tables.mjs" + }, + "./types": { + "types": "./dist/types.d.mts", + "import": "./dist/types.mjs", + "default": "./dist/types.mjs" + } + }, + "dependencies": { + "markdown-it": "14.2.0", + "yaml": "2.9.0" + } +} diff --git a/packages/markdown-core/src/chunk-text.ts b/packages/markdown-core/src/chunk-text.ts new file mode 100644 index 00000000000..105793ffc0b --- /dev/null +++ b/packages/markdown-core/src/chunk-text.ts @@ -0,0 +1,67 @@ +function resolveChunkEarlyReturn(text: string, limit: number): string[] | undefined { + if (!text) { + return []; + } + if (limit <= 0) { + return [text]; + } + if (text.length <= limit) { + return [text]; + } + return undefined; +} + +function scanParenAwareBreakpoints(text: string): { lastNewline: number; lastWhitespace: number } { + let lastNewline = -1; + let lastWhitespace = -1; + let depth = 0; + + for (let i = 0; i < text.length; i++) { + const char = text[i]; + if (char === "(") { + depth += 1; + continue; + } + if (char === ")" && depth > 0) { + depth -= 1; + continue; + } + if (depth !== 0) { + continue; + } + if (char === "\n") { + lastNewline = i; + } else if (/\s/.test(char)) { + lastWhitespace = i; + } + } + + return { lastNewline, lastWhitespace }; +} + +export function chunkText(text: string, limit: number): string[] { + const early = resolveChunkEarlyReturn(text, limit); + if (early) { + return early; + } + + const chunks: string[] = []; + let cursor = 0; + while (cursor < text.length) { + if (text.length - cursor <= limit) { + chunks.push(text.slice(cursor)); + break; + } + const windowEnd = Math.min(text.length, cursor + limit); + const window = text.slice(cursor, windowEnd); + const { lastNewline, lastWhitespace } = scanParenAwareBreakpoints(window); + const breakOffset = lastNewline > 0 ? lastNewline : lastWhitespace; + const end = breakOffset > 0 ? cursor + breakOffset : windowEnd; + chunks.push(text.slice(cursor, end)); + cursor = end; + while (cursor < text.length && /\s/.test(text[cursor] ?? "")) { + cursor += 1; + } + } + return chunks; +} diff --git a/src/markdown/code-spans.ts b/packages/markdown-core/src/code-spans.ts similarity index 100% rename from src/markdown/code-spans.ts rename to packages/markdown-core/src/code-spans.ts diff --git a/src/markdown/fences.ts b/packages/markdown-core/src/fences.ts similarity index 100% rename from src/markdown/fences.ts rename to packages/markdown-core/src/fences.ts diff --git a/src/markdown/frontmatter.test.ts b/packages/markdown-core/src/frontmatter.test.ts similarity index 100% rename from src/markdown/frontmatter.test.ts rename to packages/markdown-core/src/frontmatter.test.ts diff --git a/src/markdown/frontmatter.ts b/packages/markdown-core/src/frontmatter.ts similarity index 100% rename from src/markdown/frontmatter.ts rename to packages/markdown-core/src/frontmatter.ts diff --git a/packages/markdown-core/src/index.ts b/packages/markdown-core/src/index.ts new file mode 100644 index 00000000000..db0af7c6768 --- /dev/null +++ b/packages/markdown-core/src/index.ts @@ -0,0 +1,8 @@ +export * from "./code-spans.js"; +export * from "./fences.js"; +export * from "./frontmatter.js"; +export * from "./ir.js"; +export * from "./render-aware-chunking.js"; +export * from "./render.js"; +export * from "./tables.js"; +export * from "./types.js"; diff --git a/src/markdown/ir.blockquote-spacing.test.ts b/packages/markdown-core/src/ir.blockquote-spacing.test.ts similarity index 100% rename from src/markdown/ir.blockquote-spacing.test.ts rename to packages/markdown-core/src/ir.blockquote-spacing.test.ts diff --git a/packages/markdown-core/src/ir.chunking.test.ts b/packages/markdown-core/src/ir.chunking.test.ts new file mode 100644 index 00000000000..4267f4f72e6 --- /dev/null +++ b/packages/markdown-core/src/ir.chunking.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { chunkMarkdownIR, type MarkdownIR } from "./ir.js"; + +describe("chunkMarkdownIR", () => { + it("keeps the final in-limit remainder together after a soft break", () => { + const ir: MarkdownIR = { + text: "abcdefgh ij kl", + styles: [], + links: [], + }; + + expect(chunkMarkdownIR(ir, 10).map((chunk) => chunk.text)).toEqual(["abcdefgh", "ij kl"]); + }); +}); diff --git a/src/markdown/ir.hr-spacing.test.ts b/packages/markdown-core/src/ir.hr-spacing.test.ts similarity index 100% rename from src/markdown/ir.hr-spacing.test.ts rename to packages/markdown-core/src/ir.hr-spacing.test.ts diff --git a/src/markdown/ir.nested-lists.test.ts b/packages/markdown-core/src/ir.nested-lists.test.ts similarity index 100% rename from src/markdown/ir.nested-lists.test.ts rename to packages/markdown-core/src/ir.nested-lists.test.ts diff --git a/src/markdown/ir.table-block.test.ts b/packages/markdown-core/src/ir.table-block.test.ts similarity index 100% rename from src/markdown/ir.table-block.test.ts rename to packages/markdown-core/src/ir.table-block.test.ts diff --git a/src/markdown/ir.table-bullets.test.ts b/packages/markdown-core/src/ir.table-bullets.test.ts similarity index 100% rename from src/markdown/ir.table-bullets.test.ts rename to packages/markdown-core/src/ir.table-bullets.test.ts diff --git a/src/markdown/ir.table-code.test.ts b/packages/markdown-core/src/ir.table-code.test.ts similarity index 100% rename from src/markdown/ir.table-code.test.ts rename to packages/markdown-core/src/ir.table-code.test.ts diff --git a/src/markdown/ir.ts b/packages/markdown-core/src/ir.ts similarity index 99% rename from src/markdown/ir.ts rename to packages/markdown-core/src/ir.ts index b2266dcb6f9..df42090c6b0 100644 --- a/src/markdown/ir.ts +++ b/packages/markdown-core/src/ir.ts @@ -1,6 +1,6 @@ import MarkdownIt from "markdown-it"; -import { chunkText } from "../auto-reply/chunk.js"; -import type { MarkdownTableMode } from "../config/types.base.js"; +import { chunkText } from "./chunk-text.js"; +import type { MarkdownTableMode } from "./types.js"; type ListState = { type: "bullet" | "ordered"; diff --git a/src/markdown/render-aware-chunking.test.ts b/packages/markdown-core/src/render-aware-chunking.test.ts similarity index 100% rename from src/markdown/render-aware-chunking.test.ts rename to packages/markdown-core/src/render-aware-chunking.test.ts diff --git a/src/markdown/render-aware-chunking.ts b/packages/markdown-core/src/render-aware-chunking.ts similarity index 97% rename from src/markdown/render-aware-chunking.ts rename to packages/markdown-core/src/render-aware-chunking.ts index 861dc286d5b..d57e9382d8e 100644 --- a/src/markdown/render-aware-chunking.ts +++ b/packages/markdown-core/src/render-aware-chunking.ts @@ -1,4 +1,3 @@ -import { resolveIntegerOption } from "../shared/number-coercion.js"; import { chunkMarkdownIR, sliceMarkdownIR, @@ -24,6 +23,13 @@ type RenderResolver = Pick< "measureRendered" | "renderChunk" >; +function resolveIntegerOption(value: number, fallback: number, opts: { min: number }): number { + if (!Number.isFinite(value)) { + return fallback; + } + return Math.max(opts.min, Math.trunc(value)); +} + export function renderMarkdownIRChunksWithinLimit( options: RenderMarkdownIRChunksWithinLimitOptions, ): RenderedMarkdownChunk[] { diff --git a/src/markdown/render.ts b/packages/markdown-core/src/render.ts similarity index 100% rename from src/markdown/render.ts rename to packages/markdown-core/src/render.ts diff --git a/src/markdown/tables.test.ts b/packages/markdown-core/src/tables.test.ts similarity index 100% rename from src/markdown/tables.test.ts rename to packages/markdown-core/src/tables.test.ts diff --git a/src/markdown/tables.ts b/packages/markdown-core/src/tables.ts similarity index 94% rename from src/markdown/tables.ts rename to packages/markdown-core/src/tables.ts index 978d8219034..ffff2428315 100644 --- a/src/markdown/tables.ts +++ b/packages/markdown-core/src/tables.ts @@ -1,6 +1,6 @@ -import type { MarkdownTableMode } from "../config/types.base.js"; import { markdownToIRWithMeta } from "./ir.js"; import { renderMarkdownWithMarkers } from "./render.js"; +import type { MarkdownTableMode } from "./types.js"; const MARKDOWN_STYLE_MARKERS = { bold: { open: "**", close: "**" }, diff --git a/packages/markdown-core/src/types.ts b/packages/markdown-core/src/types.ts new file mode 100644 index 00000000000..bc789010d9b --- /dev/null +++ b/packages/markdown-core/src/types.ts @@ -0,0 +1 @@ +export type MarkdownTableMode = "off" | "bullets" | "code" | "block"; diff --git a/packages/plugin-sdk/tsconfig.json b/packages/plugin-sdk/tsconfig.json index 9fb52e38e68..24a61c1d04d 100644 --- a/packages/plugin-sdk/tsconfig.json +++ b/packages/plugin-sdk/tsconfig.json @@ -13,6 +13,7 @@ "tsBuildInfoFile": "dist/.tsbuildinfo" }, "include": [ + "../../packages/markdown-core/src/**/*.ts", "../../packages/media-generation-core/src/**/*.ts", "../../src/plugin-sdk/**/*.ts", "../../src/video-generation/dashscope-compatible.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 953be4a222e..020825c2d26 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -149,9 +149,6 @@ importers: linkedom: specifier: 0.18.12 version: 0.18.12 - markdown-it: - specifier: 14.2.0 - version: 14.2.0 minimatch: specifier: 10.2.5 version: 10.2.5 @@ -1824,6 +1821,15 @@ importers: specifier: workspace:* version: link:../llm-core + packages/markdown-core: + dependencies: + markdown-it: + specifier: 14.2.0 + version: 14.2.0 + yaml: + specifier: 2.9.0 + version: 2.9.0 + packages/media-generation-core: {} packages/memory-host-sdk: {} diff --git a/scripts/build-all.mjs b/scripts/build-all.mjs index fcd089f04da..c0fffc86f84 100644 --- a/scripts/build-all.mjs +++ b/scripts/build-all.mjs @@ -47,11 +47,13 @@ export const BUILD_ALL_STEPS = [ "npm-shrinkwrap.json", "packages/plugin-sdk/package.json", "packages/llm-core/package.json", + "packages/markdown-core/package.json", "packages/memory-host-sdk/package.json", "tsconfig.json", "tsconfig.plugin-sdk.dts.json", "src/plugin-sdk", "packages/llm-core/src", + "packages/markdown-core/src", "packages/memory-host-sdk/src", "packages/media-generation-core/src", "src/types", diff --git a/scripts/lib/extension-package-boundary.ts b/scripts/lib/extension-package-boundary.ts index cce9d7e0e35..b7021403b65 100644 --- a/scripts/lib/extension-package-boundary.ts +++ b/scripts/lib/extension-package-boundary.ts @@ -59,6 +59,22 @@ export const EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS = { "@openclaw/llm-core/types": ["../dist/plugin-sdk/packages/llm-core/src/types.d.ts"], "@openclaw/llm-core/validation": ["../dist/plugin-sdk/packages/llm-core/src/validation.d.ts"], "@openclaw/llm-core/*": ["../dist/plugin-sdk/packages/llm-core/src/*.d.ts"], + "@openclaw/markdown-core": ["../dist/plugin-sdk/packages/markdown-core/src/index.d.ts"], + "@openclaw/markdown-core/code-spans": [ + "../dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts", + ], + "@openclaw/markdown-core/fences": ["../dist/plugin-sdk/packages/markdown-core/src/fences.d.ts"], + "@openclaw/markdown-core/frontmatter": [ + "../dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts", + ], + "@openclaw/markdown-core/ir": ["../dist/plugin-sdk/packages/markdown-core/src/ir.d.ts"], + "@openclaw/markdown-core/render": ["../dist/plugin-sdk/packages/markdown-core/src/render.d.ts"], + "@openclaw/markdown-core/render-aware-chunking": [ + "../dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts", + ], + "@openclaw/markdown-core/tables": ["../dist/plugin-sdk/packages/markdown-core/src/tables.d.ts"], + "@openclaw/markdown-core/types": ["../dist/plugin-sdk/packages/markdown-core/src/types.d.ts"], + "@openclaw/markdown-core/*": ["../dist/plugin-sdk/packages/markdown-core/src/*.d.ts"], "@openclaw/media-generation-core": [ "../dist/plugin-sdk/packages/media-generation-core/src/index.d.ts", ], diff --git a/scripts/prepare-extension-package-boundary-artifacts.mjs b/scripts/prepare-extension-package-boundary-artifacts.mjs index 38498b4f692..15d7617816a 100644 --- a/scripts/prepare-extension-package-boundary-artifacts.mjs +++ b/scripts/prepare-extension-package-boundary-artifacts.mjs @@ -16,6 +16,7 @@ const PLUGIN_SDK_TYPE_INPUTS = [ "src/plugin-sdk", "src/auto-reply", "packages/llm-core/src", + "packages/markdown-core/src", "packages/memory-host-sdk/src", "packages/media-generation-core/src", "src/video-generation/dashscope-compatible.ts", @@ -33,6 +34,15 @@ const ROOT_DTS_REQUIRED_OUTPUTS = [ "dist/plugin-sdk/packages/llm-core/src/utils/diagnostics.d.ts", "dist/plugin-sdk/packages/llm-core/src/utils/event-stream.d.ts", "dist/plugin-sdk/packages/llm-core/src/validation.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/fences.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/index.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/ir.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/render.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/tables.d.ts", + "dist/plugin-sdk/packages/markdown-core/src/types.d.ts", "dist/plugin-sdk/packages/media-generation-core/src/capability-model-ref.d.ts", "dist/plugin-sdk/packages/media-generation-core/src/catalog.d.ts", "dist/plugin-sdk/packages/media-generation-core/src/index.d.ts", @@ -46,6 +56,15 @@ const ROOT_DTS_REQUIRED_OUTPUTS = [ const PACKAGE_DTS_INPUTS = ["packages/plugin-sdk/tsconfig.json", ...PLUGIN_SDK_TYPE_INPUTS]; const PACKAGE_DTS_STAMP = "packages/plugin-sdk/dist/.boundary-dts.stamp"; const PACKAGE_DTS_REQUIRED_OUTPUTS = [ + "packages/plugin-sdk/dist/packages/markdown-core/src/code-spans.d.ts", + "packages/plugin-sdk/dist/packages/markdown-core/src/fences.d.ts", + "packages/plugin-sdk/dist/packages/markdown-core/src/frontmatter.d.ts", + "packages/plugin-sdk/dist/packages/markdown-core/src/index.d.ts", + "packages/plugin-sdk/dist/packages/markdown-core/src/ir.d.ts", + "packages/plugin-sdk/dist/packages/markdown-core/src/render-aware-chunking.d.ts", + "packages/plugin-sdk/dist/packages/markdown-core/src/render.d.ts", + "packages/plugin-sdk/dist/packages/markdown-core/src/tables.d.ts", + "packages/plugin-sdk/dist/packages/markdown-core/src/types.d.ts", "packages/plugin-sdk/dist/packages/media-generation-core/src/capability-model-ref.d.ts", "packages/plugin-sdk/dist/packages/media-generation-core/src/catalog.d.ts", "packages/plugin-sdk/dist/packages/media-generation-core/src/index.d.ts", diff --git a/scripts/run-node-watch-paths.mjs b/scripts/run-node-watch-paths.mjs index 7a40c2deb69..ef0d59a40a3 100644 --- a/scripts/run-node-watch-paths.mjs +++ b/scripts/run-node-watch-paths.mjs @@ -10,6 +10,7 @@ const RUN_NODE_PACKAGE_SOURCE_ROOTS = [ // src/ so edits restart the same process that consumes them. "packages/gateway-client/src", "packages/gateway-protocol/src", + "packages/markdown-core/src", "packages/media-generation-core/src", "packages/net-policy/src", ]; diff --git a/src/agents/embedded-agent-block-chunker.test.ts b/src/agents/embedded-agent-block-chunker.test.ts index 934be79887c..841dd0834a6 100644 --- a/src/agents/embedded-agent-block-chunker.test.ts +++ b/src/agents/embedded-agent-block-chunker.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import * as fences from "../markdown/fences.js"; +import * as fences from "../../packages/markdown-core/src/fences.js"; import { EmbeddedBlockChunker } from "./embedded-agent-block-chunker.js"; function createFlushOnParagraphChunker(params: { minChars: number; maxChars: number }) { diff --git a/src/agents/embedded-agent-block-chunker.ts b/src/agents/embedded-agent-block-chunker.ts index 32a2a63df4f..36d00fa9694 100644 --- a/src/agents/embedded-agent-block-chunker.ts +++ b/src/agents/embedded-agent-block-chunker.ts @@ -1,5 +1,9 @@ -import type { FenceSpan } from "../markdown/fences.js"; -import { findFenceSpanAt, isSafeFenceBreak, parseFenceSpans } from "../markdown/fences.js"; +import type { FenceSpan } from "../../packages/markdown-core/src/fences.js"; +import { + findFenceSpanAt, + isSafeFenceBreak, + parseFenceSpans, +} from "../../packages/markdown-core/src/fences.js"; export type BlockReplyChunking = { minChars: number; diff --git a/src/agents/embedded-agent-subscribe.handlers.lifecycle.test.ts b/src/agents/embedded-agent-subscribe.handlers.lifecycle.test.ts index 7f1c743c970..bfa1542f83a 100644 --- a/src/agents/embedded-agent-subscribe.handlers.lifecycle.test.ts +++ b/src/agents/embedded-agent-subscribe.handlers.lifecycle.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { createInlineCodeState } from "../markdown/code-spans.js"; +import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; import { handleAgentEnd } from "./embedded-agent-subscribe.handlers.lifecycle.js"; import type { EmbeddedAgentSubscribeContext } from "./embedded-agent-subscribe.handlers.types.js"; diff --git a/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts b/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts index aef9e4d5370..33938074280 100644 --- a/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts +++ b/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts @@ -1,5 +1,5 @@ +import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; import { emitAgentEvent } from "../infra/agent-events.js"; -import { createInlineCodeState } from "../markdown/code-spans.js"; import { hasAcceptedSessionSpawn } from "./accepted-session-spawn.js"; import { buildApiErrorObservationFields, diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.test.ts index d4488025461..800636f397f 100644 --- a/src/agents/embedded-agent-subscribe.handlers.messages.test.ts +++ b/src/agents/embedded-agent-subscribe.handlers.messages.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; +import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js"; -import { createInlineCodeState } from "../markdown/code-spans.js"; import { buildAssistantStreamData, consumePendingAssistantReplyDirectivesIntoReply, diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.ts b/src/agents/embedded-agent-subscribe.handlers.messages.ts index ff0ccc937ef..1710f90d11d 100644 --- a/src/agents/embedded-agent-subscribe.handlers.messages.ts +++ b/src/agents/embedded-agent-subscribe.handlers.messages.ts @@ -1,4 +1,5 @@ import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload"; +import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; import { parseReplyDirectives, type ReplyDirectiveParseResult, @@ -7,7 +8,6 @@ import { splitTrailingDirective } from "../auto-reply/reply/streaming-directives import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; import { emitAgentEvent } from "../infra/agent-events.js"; import type { AssistantMessage } from "../llm/types.js"; -import { createInlineCodeState } from "../markdown/code-spans.js"; import { coerceChatContentText } from "../shared/chat-content.js"; import { parseAssistantTextSignature, diff --git a/src/agents/embedded-agent-subscribe.handlers.types.ts b/src/agents/embedded-agent-subscribe.handlers.types.ts index 314e06c1715..8c7c5a0dbcf 100644 --- a/src/agents/embedded-agent-subscribe.handlers.types.ts +++ b/src/agents/embedded-agent-subscribe.handlers.types.ts @@ -1,8 +1,8 @@ +import type { InlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; +import type { FenceScanState } from "../../packages/markdown-core/src/fences.js"; import type { HeartbeatToolResponse } from "../auto-reply/heartbeat-tool-response.js"; import type { ReplyDirectiveParseResult } from "../auto-reply/reply/reply-directives.js"; import type { ReasoningLevel } from "../auto-reply/thinking.js"; -import type { InlineCodeState } from "../markdown/code-spans.js"; -import type { FenceScanState } from "../markdown/fences.js"; import type { HookRunner } from "../plugins/hooks.js"; import type { AcceptedSessionSpawn } from "./accepted-session-spawn.js"; import type { EmbeddedBlockChunker } from "./embedded-agent-block-chunker.js"; diff --git a/src/agents/embedded-agent-subscribe.ts b/src/agents/embedded-agent-subscribe.ts index 69cf80d1939..bd69c6a10e7 100644 --- a/src/agents/embedded-agent-subscribe.ts +++ b/src/agents/embedded-agent-subscribe.ts @@ -1,3 +1,9 @@ +import type { InlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; +import { + buildCodeSpanIndex, + createInlineCodeState, +} from "../../packages/markdown-core/src/code-spans.js"; +import type { FenceScanState } from "../../packages/markdown-core/src/fences.js"; import { setReplyPayloadMetadata } from "../auto-reply/reply-payload.js"; import { parseReplyDirectives } from "../auto-reply/reply/reply-directives.js"; import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js"; @@ -5,9 +11,6 @@ import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; import { formatToolAggregate } from "../auto-reply/tool-meta.js"; import { emitAgentEvent } from "../infra/agent-events.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; -import type { InlineCodeState } from "../markdown/code-spans.js"; -import { buildCodeSpanIndex, createInlineCodeState } from "../markdown/code-spans.js"; -import type { FenceScanState } from "../markdown/fences.js"; import { normalizeOptionalString } from "../shared/string-coerce.js"; import { findFinalTagMatches } from "../shared/text/final-tags.js"; import { hasOrphanReasoningCloseBoundary } from "../shared/text/reasoning-tags.js"; diff --git a/src/auto-reply/chunk.test.ts b/src/auto-reply/chunk.test.ts index 4bcdd8491bd..ec29928fc71 100644 --- a/src/auto-reply/chunk.test.ts +++ b/src/auto-reply/chunk.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import * as fences from "../markdown/fences.js"; +import * as fences from "../../packages/markdown-core/src/fences.js"; import { hasBalancedFences } from "../test-utils/chunk-test-helpers.js"; import { chunkByNewline, diff --git a/src/auto-reply/chunk.ts b/src/auto-reply/chunk.ts index cabb0abc1de..99709f021db 100644 --- a/src/auto-reply/chunk.ts +++ b/src/auto-reply/chunk.ts @@ -2,10 +2,14 @@ // unintentionally breaking on newlines. Using [\s\S] keeps newlines inside // the chunk so messages are only split when they truly exceed the limit. +import { + findFenceSpanAt, + isSafeFenceBreak, + parseFenceSpans, +} from "../../packages/markdown-core/src/fences.js"; import type { ChannelId } from "../channels/plugins/types.core.js"; import { resolveChannelStreamingChunkMode } from "../channels/streaming.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { findFenceSpanAt, isSafeFenceBreak, parseFenceSpans } from "../markdown/fences.js"; import { resolveAccountEntry } from "../routing/account-lookup.js"; import { normalizeAccountId } from "../routing/session-key.js"; import { chunkTextByBreakResolver } from "../shared/text-chunking.js"; diff --git a/src/chat/canvas-render.ts b/src/chat/canvas-render.ts index 1f01b82979f..939475d3a3b 100644 --- a/src/chat/canvas-render.ts +++ b/src/chat/canvas-render.ts @@ -1,4 +1,4 @@ -import { parseFenceSpans } from "../markdown/fences.js"; +import { parseFenceSpans } from "../../packages/markdown-core/src/fences.js"; import { asFiniteNumber } from "../shared/number-coercion.js"; import { asOptionalRecord } from "../shared/record-coerce.js"; diff --git a/src/hooks/frontmatter.ts b/src/hooks/frontmatter.ts index 14f5b699c50..e2a75bf5a08 100644 --- a/src/hooks/frontmatter.ts +++ b/src/hooks/frontmatter.ts @@ -1,4 +1,4 @@ -import { parseFrontmatterBlock } from "../markdown/frontmatter.js"; +import { parseFrontmatterBlock } from "../../packages/markdown-core/src/frontmatter.js"; import { applyOpenClawManifestInstallCommonFields, getFrontmatterString, diff --git a/src/infra/watch-node.test.ts b/src/infra/watch-node.test.ts index 74dbf880263..5c3299d8572 100644 --- a/src/infra/watch-node.test.ts +++ b/src/infra/watch-node.test.ts @@ -132,6 +132,7 @@ describe("watch-node script", () => { expect(watchPaths).toContain("extensions"); expect(watchPaths).toContain("packages/gateway-client/src"); expect(watchPaths).toContain("packages/gateway-protocol/src"); + expect(watchPaths).toContain("packages/markdown-core/src"); expect(watchPaths).toContain("packages/media-generation-core/src"); expect(watchPaths).toContain("packages/net-policy/src"); expect(watchPaths).toContain("tsdown.config.ts"); @@ -141,6 +142,8 @@ describe("watch-node script", () => { expect(watchOptions.ignored("packages/gateway-client/src/client.ts")).toBe(false); expect(watchOptions.ignored("packages/gateway-client/src/client.test.ts")).toBe(true); expect(watchOptions.ignored("packages/gateway-protocol/src/schema/cron.ts")).toBe(false); + expect(watchOptions.ignored("packages/markdown-core/src/ir.ts")).toBe(false); + expect(watchOptions.ignored("packages/markdown-core/src/ir.test.ts")).toBe(true); expect(watchOptions.ignored("packages/media-generation-core/src/model-ref.ts")).toBe(false); expect(watchOptions.ignored("packages/media-generation-core/src/model-ref.test.ts")).toBe( true, diff --git a/src/media/parse.ts b/src/media/parse.ts index d803b4408cc..b80f7b15b8a 100644 --- a/src/media/parse.ts +++ b/src/media/parse.ts @@ -10,7 +10,7 @@ import { parseCanonicalIpAddress, parseLooseIpAddress, } from "@openclaw/net-policy/ip"; -import { parseFenceSpans } from "../markdown/fences.js"; +import { parseFenceSpans } from "../../packages/markdown-core/src/fences.js"; import { parseAudioTag } from "./audio-tags.js"; // Allow optional wrapping backticks and punctuation after the token; capture the core token. diff --git a/src/plugin-sdk/markdown-table-runtime.ts b/src/plugin-sdk/markdown-table-runtime.ts index 0efe2807915..24a621e29f1 100644 --- a/src/plugin-sdk/markdown-table-runtime.ts +++ b/src/plugin-sdk/markdown-table-runtime.ts @@ -1,3 +1,3 @@ export { resolveMarkdownTableMode } from "../config/markdown-tables.js"; -export { convertMarkdownTables } from "../markdown/tables.js"; +export { convertMarkdownTables } from "../../packages/markdown-core/src/tables.js"; export type { MarkdownTableMode } from "../config/types.base.js"; diff --git a/src/plugin-sdk/text-chunking.ts b/src/plugin-sdk/text-chunking.ts index 1cf94ae9f13..2bf1f456a42 100644 --- a/src/plugin-sdk/text-chunking.ts +++ b/src/plugin-sdk/text-chunking.ts @@ -20,19 +20,19 @@ export { type MarkdownStyle, type MarkdownStyleSpan, type MarkdownTableMeta, -} from "../markdown/ir.js"; +} from "../../packages/markdown-core/src/ir.js"; export { renderMarkdownIRChunksWithinLimit, type RenderMarkdownIRChunksWithinLimitOptions, -} from "../markdown/render-aware-chunking.js"; +} from "../../packages/markdown-core/src/render-aware-chunking.js"; export { renderMarkdownWithMarkers, type RenderLink, type RenderOptions, type RenderStyleMap, type RenderStyleMarker, -} from "../markdown/render.js"; -export { convertMarkdownTables } from "../markdown/tables.js"; +} from "../../packages/markdown-core/src/render.js"; +export { convertMarkdownTables } from "../../packages/markdown-core/src/tables.js"; export { sanitizeAssistantVisibleText, sanitizeAssistantVisibleTextWithOptions, diff --git a/src/plugin-sdk/text-runtime.ts b/src/plugin-sdk/text-runtime.ts index b53a3b46323..c74331dd984 100644 --- a/src/plugin-sdk/text-runtime.ts +++ b/src/plugin-sdk/text-runtime.ts @@ -8,10 +8,10 @@ export * from "../logging/diagnostic.js"; export * from "../logging/logger.js"; export * from "../logging/redact.js"; export * from "../logging/redact-identifier.js"; -export * from "../markdown/ir.js"; -export * from "../markdown/render-aware-chunking.js"; -export * from "../markdown/render.js"; -export * from "../markdown/tables.js"; +export * from "../../packages/markdown-core/src/ir.js"; +export * from "../../packages/markdown-core/src/render-aware-chunking.js"; +export * from "../../packages/markdown-core/src/render.js"; +export * from "../../packages/markdown-core/src/tables.js"; export * from "../shared/global-singleton.js"; export * from "../shared/record-coerce.js"; export * from "../shared/scoped-expiring-id-cache.js"; diff --git a/src/plugins/bundle-commands.ts b/src/plugins/bundle-commands.ts index 2b6120490d2..9dea69c911e 100644 --- a/src/plugins/bundle-commands.ts +++ b/src/plugins/bundle-commands.ts @@ -1,8 +1,8 @@ import fs from "node:fs"; import path from "node:path"; +import { parseFrontmatterBlock } from "../../packages/markdown-core/src/frontmatter.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { readRootJsonObjectSync } from "../infra/json-files.js"; -import { parseFrontmatterBlock } from "../markdown/frontmatter.js"; import { isPathInsideWithRealpath } from "../security/scan-paths.js"; import { normalizeOptionalLowercaseString, diff --git a/src/plugins/contracts/extension-package-project-boundaries.test.ts b/src/plugins/contracts/extension-package-project-boundaries.test.ts index afe0b53ac96..c88857c9c13 100644 --- a/src/plugins/contracts/extension-package-project-boundaries.test.ts +++ b/src/plugins/contracts/extension-package-project-boundaries.test.ts @@ -194,6 +194,7 @@ describe("opt-in extension package boundaries", () => { expect(tsconfig.compilerOptions?.outDir).toBe("dist"); expect(tsconfig.compilerOptions?.rootDir).toBe("../.."); expect(tsconfig.include).toEqual([ + "../../packages/markdown-core/src/**/*.ts", "../../packages/media-generation-core/src/**/*.ts", "../../src/plugin-sdk/**/*.ts", "../../src/video-generation/dashscope-compatible.ts", diff --git a/src/plugins/runtime/runtime-channel.ts b/src/plugins/runtime/runtime-channel.ts index 910d645a0bc..be82619782b 100644 --- a/src/plugins/runtime/runtime-channel.ts +++ b/src/plugins/runtime/runtime-channel.ts @@ -1,3 +1,4 @@ +import { convertMarkdownTables } from "../../../packages/markdown-core/src/tables.js"; import { resolveEffectiveMessagesConfig, resolveHumanDelayConfig } from "../../agents/identity.js"; import { chunkByNewline, @@ -68,7 +69,6 @@ import { updateLastRoute, } from "../../config/sessions.js"; import { getChannelActivity, recordChannelActivity } from "../../infra/channel-activity.js"; -import { convertMarkdownTables } from "../../markdown/tables.js"; import { fetchRemoteMedia, readRemoteMediaBuffer, diff --git a/src/plugins/runtime/types-channel.ts b/src/plugins/runtime/types-channel.ts index f6f6bf29180..37279482701 100644 --- a/src/plugins/runtime/types-channel.ts +++ b/src/plugins/runtime/types-channel.ts @@ -81,7 +81,7 @@ export type PluginRuntimeChannel = { resolveTextChunkLimit: typeof import("../../auto-reply/chunk.js").resolveTextChunkLimit; hasControlCommand: typeof import("../../auto-reply/command-detection.js").hasControlCommand; resolveMarkdownTableMode: import("../../config/markdown-tables.types.js").ResolveMarkdownTableMode; - convertMarkdownTables: typeof import("../../markdown/tables.js").convertMarkdownTables; + convertMarkdownTables: typeof import("../../../packages/markdown-core/src/tables.js").convertMarkdownTables; }; reply: { dispatchReplyWithBufferedBlockDispatcher: DispatchReplyWithBufferedBlockDispatcher; diff --git a/src/plugins/sdk-alias.test.ts b/src/plugins/sdk-alias.test.ts index bbaf850d7ec..adc439d19a5 100644 --- a/src/plugins/sdk-alias.test.ts +++ b/src/plugins/sdk-alias.test.ts @@ -1384,6 +1384,18 @@ describe("plugin sdk alias helpers", () => { srcFile: "index.ts", distFile: "index.mjs", }); + const markdownCore = writeWorkspacePackageEntry({ + root: fixture.root, + packageDir: "markdown-core", + srcFile: "index.ts", + distFile: "index.mjs", + }); + const markdownCoreTables = writeWorkspacePackageEntry({ + root: fixture.root, + packageDir: "markdown-core", + srcFile: "tables.ts", + distFile: "tables.mjs", + }); const mediaGenerationModelRef = writeWorkspacePackageEntry({ root: fixture.root, packageDir: "media-generation-core", @@ -1400,6 +1412,8 @@ describe("plugin sdk alias helpers", () => { fs.rmSync(gatewayClientTimeouts.distFile); fs.rmSync(gatewayProtocol.distFile); fs.rmSync(gatewayProtocolSchema.distFile); + fs.rmSync(markdownCore.distFile); + fs.rmSync(markdownCoreTables.distFile); fs.rmSync(mediaGenerationCore.distFile); fs.rmSync(mediaGenerationModelRef.distFile); fs.rmSync(netPolicy.distFile); @@ -1425,6 +1439,12 @@ describe("plugin sdk alias helpers", () => { expect(fs.realpathSync(aliases["@openclaw/gateway-protocol/schema"] ?? "")).toBe( fs.realpathSync(gatewayProtocolSchema.srcFile), ); + expect(fs.realpathSync(aliases["@openclaw/markdown-core"] ?? "")).toBe( + fs.realpathSync(markdownCore.srcFile), + ); + expect(fs.realpathSync(aliases["@openclaw/markdown-core/tables"] ?? "")).toBe( + fs.realpathSync(markdownCoreTables.srcFile), + ); expect(fs.realpathSync(aliases["@openclaw/media-generation-core"] ?? "")).toBe( fs.realpathSync(mediaGenerationCore.srcFile), ); @@ -1459,6 +1479,12 @@ describe("plugin sdk alias helpers", () => { srcFile: "catalog.ts", distFile: "catalog.mjs", }); + const markdownCore = writeWorkspacePackageEntry({ + root: fixture.root, + packageDir: "markdown-core", + srcFile: "render.ts", + distFile: "render.mjs", + }); const netPolicy = writeWorkspacePackageEntry({ root: fixture.root, packageDir: "net-policy", @@ -1480,6 +1506,9 @@ describe("plugin sdk alias helpers", () => { expect(fs.realpathSync(aliases["@openclaw/gateway-protocol/connect-error-details"] ?? "")).toBe( fs.realpathSync(gatewayProtocol.distFile), ); + expect(fs.realpathSync(aliases["@openclaw/markdown-core/render"] ?? "")).toBe( + fs.realpathSync(markdownCore.distFile), + ); expect(fs.realpathSync(aliases["@openclaw/media-generation-core/catalog"] ?? "")).toBe( fs.realpathSync(mediaGenerationCore.distFile), ); diff --git a/src/plugins/sdk-alias.ts b/src/plugins/sdk-alias.ts index 29bb514d2ef..4a769b179ad 100644 --- a/src/plugins/sdk-alias.ts +++ b/src/plugins/sdk-alias.ts @@ -560,6 +560,69 @@ const WORKSPACE_PACKAGE_ALIAS_ENTRIES = [ srcFile: "version.ts", distFile: "version.mjs", }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "", + srcFile: "index.ts", + distFile: "index.mjs", + }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "code-spans", + srcFile: "code-spans.ts", + distFile: "code-spans.mjs", + }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "fences", + srcFile: "fences.ts", + distFile: "fences.mjs", + }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "frontmatter", + srcFile: "frontmatter.ts", + distFile: "frontmatter.mjs", + }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "ir", + srcFile: "ir.ts", + distFile: "ir.mjs", + }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "render", + srcFile: "render.ts", + distFile: "render.mjs", + }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "render-aware-chunking", + srcFile: "render-aware-chunking.ts", + distFile: "render-aware-chunking.mjs", + }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "tables", + srcFile: "tables.ts", + distFile: "tables.mjs", + }, + { + packageName: "@openclaw/markdown-core", + packageDir: "markdown-core", + subpath: "types", + srcFile: "types.ts", + distFile: "types.mjs", + }, { packageName: "@openclaw/media-generation-core", packageDir: "media-generation-core", diff --git a/src/skills/loading/frontmatter.ts b/src/skills/loading/frontmatter.ts index c647cd517ff..f2b7e92eec9 100644 --- a/src/skills/loading/frontmatter.ts +++ b/src/skills/loading/frontmatter.ts @@ -1,5 +1,5 @@ +import { parseFrontmatterBlock } from "../../../packages/markdown-core/src/frontmatter.js"; import { validateRegistryNpmSpec } from "../../infra/npm-registry-spec.js"; -import { parseFrontmatterBlock } from "../../markdown/frontmatter.js"; import { applyOpenClawManifestInstallCommonFields, getFrontmatterString, diff --git a/test/vitest-unit-config.test.ts b/test/vitest-unit-config.test.ts index 61ccba1676f..0a7db4eff54 100644 --- a/test/vitest-unit-config.test.ts +++ b/test/vitest-unit-config.test.ts @@ -175,7 +175,7 @@ describe("unit vitest config", () => { expect(coverageInclude).toContain("src/commitments/runtime.ts"); expect(coverageInclude).toContain("src/media-generation/runtime-shared.ts"); expect(coverageInclude).toContain("src/web-search/runtime.ts"); - expect(coverageInclude).not.toContain("src/markdown/render.ts"); + expect(coverageInclude).not.toContain("packages/markdown-core/src/render.ts"); expect(coverageInclude).not.toContain("src/security/audit-workspace-skills.ts"); }); diff --git a/test/vitest/vitest.unit-fast-paths.mjs b/test/vitest/vitest.unit-fast-paths.mjs index ee4145b8d6a..b148729ff26 100644 --- a/test/vitest/vitest.unit-fast-paths.mjs +++ b/test/vitest/vitest.unit-fast-paths.mjs @@ -31,7 +31,7 @@ const unitFastCandidateGlobs = [ "src/interactive/**/*.test.ts", "src/link-understanding/**/*.test.ts", "src/logging/**/*.test.ts", - "src/markdown/**/*.test.ts", + "packages/markdown-core/src/**/*.test.ts", "src/media/**/*.test.ts", "src/media-generation/**/*.test.ts", "src/media-understanding/**/*.test.ts", diff --git a/tsconfig.json b/tsconfig.json index 824098403be..6a5d45986e8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -65,6 +65,18 @@ "./packages/media-generation-core/src/normalization.ts" ], "@openclaw/media-generation-core/*": ["./packages/media-generation-core/src/*"], + "@openclaw/markdown-core": ["./packages/markdown-core/src/index.ts"], + "@openclaw/markdown-core/code-spans": ["./packages/markdown-core/src/code-spans.ts"], + "@openclaw/markdown-core/fences": ["./packages/markdown-core/src/fences.ts"], + "@openclaw/markdown-core/frontmatter": ["./packages/markdown-core/src/frontmatter.ts"], + "@openclaw/markdown-core/ir": ["./packages/markdown-core/src/ir.ts"], + "@openclaw/markdown-core/render": ["./packages/markdown-core/src/render.ts"], + "@openclaw/markdown-core/render-aware-chunking": [ + "./packages/markdown-core/src/render-aware-chunking.ts" + ], + "@openclaw/markdown-core/tables": ["./packages/markdown-core/src/tables.ts"], + "@openclaw/markdown-core/types": ["./packages/markdown-core/src/types.ts"], + "@openclaw/markdown-core/*": ["./packages/markdown-core/src/*"], "@openclaw/net-policy": ["./packages/net-policy/src/index.ts"], "@openclaw/net-policy/ip": ["./packages/net-policy/src/ip.ts"], "@openclaw/net-policy/ipv4": ["./packages/net-policy/src/ipv4.ts"], diff --git a/tsconfig.plugin-sdk.dts.json b/tsconfig.plugin-sdk.dts.json index e2ca8f327c6..4116aaf437c 100644 --- a/tsconfig.plugin-sdk.dts.json +++ b/tsconfig.plugin-sdk.dts.json @@ -14,6 +14,7 @@ "include": [ "src/plugin-sdk/**/*.ts", "packages/llm-core/src/**/*.ts", + "packages/markdown-core/src/**/*.ts", "packages/media-generation-core/src/**/*.ts", "packages/memory-host-sdk/src/**/*.ts", "src/video-generation/dashscope-compatible.ts", diff --git a/tsdown.config.ts b/tsdown.config.ts index ee8f2bb11db..af1f47fccdb 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -394,6 +394,20 @@ function buildMediaGenerationCoreDistEntries(): Record { }; } +function buildMarkdownCoreDistEntries(): Record { + return { + index: "packages/markdown-core/src/index.ts", + "code-spans": "packages/markdown-core/src/code-spans.ts", + fences: "packages/markdown-core/src/fences.ts", + frontmatter: "packages/markdown-core/src/frontmatter.ts", + ir: "packages/markdown-core/src/ir.ts", + render: "packages/markdown-core/src/render.ts", + "render-aware-chunking": "packages/markdown-core/src/render-aware-chunking.ts", + tables: "packages/markdown-core/src/tables.ts", + types: "packages/markdown-core/src/types.ts", + }; +} + function buildSpeechCoreDistEntries(): Record { return { api: "packages/speech-core/api.ts", @@ -464,6 +478,12 @@ function shouldExternalizeLlmRuntimeDependency(id: string): boolean { return id === "@openclaw/llm-core" || id.startsWith("@openclaw/llm-core/"); } +function shouldExternalizeMarkdownCoreDependency(id: string): boolean { + return ( + id === "markdown-it" || id.startsWith("markdown-it/") || id === "yaml" || id.startsWith("yaml/") + ); +} + const coreDistEntries = buildCoreDistEntries(); const dockerE2eHarnessEntries = buildDockerE2eHarnessEntries(); const rootBundledPluginBuildEntries = bundledPluginBuildEntries.filter( @@ -539,6 +559,15 @@ export default defineConfig([ entry: buildMediaGenerationCoreDistEntries(), outDir: "packages/media-generation-core/dist", }), + nodeWorkspacePackageBuildConfig({ + clean: true, + dts: RUN_NODE_SKIP_DTS_BUILD ? false : undefined, + entry: buildMarkdownCoreDistEntries(), + outDir: "packages/markdown-core/dist", + deps: { + neverBundle: shouldExternalizeMarkdownCoreDependency, + }, + }), nodeWorkspacePackageBuildConfig({ clean: true, dts: RUN_NODE_SKIP_DTS_BUILD ? false : undefined,