mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-02 07:01:34 +00:00
docs: generate docs map at publish time (#117398)
* docs: generate docs map at publish time * fix: canonicalize Docker pack entry path * fix: preserve package lifecycle ownership
This commit is contained in:
committed by
GitHub
parent
e4d1b7e0d3
commit
385f1dee16
@@ -1,15 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Writes the `bin/docs-list` wrapper used by package scripts and docs tooling.
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const binDir = path.join(root, "bin");
|
||||
const binPath = path.join(binDir, "docs-list");
|
||||
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
|
||||
const wrapper = `#!/usr/bin/env node\nimport { spawnSync } from "node:child_process";\nimport path from "node:path";\nimport { fileURLToPath } from "node:url";\n\nconst here = path.dirname(fileURLToPath(import.meta.url));\nconst script = path.join(here, "..", "scripts", "docs-list.js");\n\nconst result = spawnSync(process.execPath, [script], { stdio: "inherit" });\nprocess.exit(result.status ?? 1);\n`;
|
||||
|
||||
fs.writeFileSync(binPath, wrapper, { mode: 0o755 });
|
||||
4
scripts/docs-list.d.ts
vendored
Normal file
4
scripts/docs-list.d.ts
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
export function renderDocsHeadingMap(
|
||||
docsDir?: string,
|
||||
options?: { relativePath?: (base: string, fullPath: string) => string },
|
||||
): string;
|
||||
@@ -1,27 +1,31 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Lists source docs pages and routes for docs-aware tooling.
|
||||
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
||||
// Lists source docs pages and renders on-demand heading metadata for docs-aware tooling.
|
||||
import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
|
||||
import { join, relative } from "node:path";
|
||||
|
||||
process.stdout.on("error", (error) => {
|
||||
if (error?.code === "EPIPE") {
|
||||
process.exit(0);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const DOCS_DIR = join(process.cwd(), "docs");
|
||||
if (!existsSync(DOCS_DIR)) {
|
||||
console.error("docs:list: missing docs directory. Run from repo root.");
|
||||
process.exit(1);
|
||||
}
|
||||
if (!statSync(DOCS_DIR).isDirectory()) {
|
||||
console.error("docs:list: docs path is not a directory.");
|
||||
process.exit(1);
|
||||
}
|
||||
const DOCS_LIST_EXCLUDED_DIRS = new Set(["archive", "research"]);
|
||||
const DOCS_MAP_EXCLUDED_DIRS = new Set([
|
||||
".generated",
|
||||
"archive",
|
||||
"assets",
|
||||
"images",
|
||||
"internal",
|
||||
"research",
|
||||
"snippets",
|
||||
]);
|
||||
const DOCS_MAP_EXCLUDED_FILES = new Set(["AGENTS.md", "CLAUDE.md", "docs_map.md"]);
|
||||
|
||||
const EXCLUDED_DIRS = new Set(["archive", "research"]);
|
||||
function assertDocsDir(docsDir) {
|
||||
if (!existsSync(docsDir)) {
|
||||
throw new Error("docs:list: missing docs directory. Run from repo root.");
|
||||
}
|
||||
if (!statSync(docsDir).isDirectory()) {
|
||||
throw new Error("docs:list: docs path is not a directory.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {unknown[]} values
|
||||
@@ -50,9 +54,14 @@ function compactStrings(values) {
|
||||
/**
|
||||
* @param {string} dir
|
||||
* @param {string} base
|
||||
* @param {{
|
||||
* excludedDirs?: Set<string>;
|
||||
* excludedFiles?: Set<string>;
|
||||
* relativePath?: (base: string, fullPath: string) => string;
|
||||
* }} options
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function walkMarkdownFiles(dir, base = dir) {
|
||||
function walkMarkdownFiles(dir, base = dir, options = {}) {
|
||||
const entries = readdirSync(dir, { withFileTypes: true });
|
||||
const files = [];
|
||||
for (const entry of entries) {
|
||||
@@ -61,12 +70,16 @@ function walkMarkdownFiles(dir, base = dir) {
|
||||
}
|
||||
const fullPath = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (EXCLUDED_DIRS.has(entry.name)) {
|
||||
if (options.excludedDirs?.has(entry.name)) {
|
||||
continue;
|
||||
}
|
||||
files.push(...walkMarkdownFiles(fullPath, base));
|
||||
} else if (entry.isFile() && /\.(md|mdx)$/i.test(entry.name)) {
|
||||
files.push(relative(base, fullPath));
|
||||
files.push(...walkMarkdownFiles(fullPath, base, options));
|
||||
} else if (
|
||||
entry.isFile() &&
|
||||
/\.(md|mdx)$/iu.test(entry.name) &&
|
||||
!options.excludedFiles?.has(entry.name)
|
||||
) {
|
||||
files.push((options.relativePath ?? relative)(base, fullPath));
|
||||
}
|
||||
}
|
||||
return files.toSorted((a, b) => a.localeCompare(b));
|
||||
@@ -156,24 +169,182 @@ function extractMetadata(fullPath) {
|
||||
return { summary: normalized, readWhen };
|
||||
}
|
||||
|
||||
console.log("Listing all markdown files in docs folder:");
|
||||
|
||||
const markdownFiles = walkMarkdownFiles(DOCS_DIR);
|
||||
|
||||
for (const relativePath of markdownFiles) {
|
||||
const fullPath = join(DOCS_DIR, relativePath);
|
||||
const { summary, readWhen, error } = extractMetadata(fullPath);
|
||||
if (summary) {
|
||||
console.log(`${relativePath} - ${summary}`);
|
||||
if (readWhen.length > 0) {
|
||||
console.log(` Read when: ${readWhen.join("; ")}`);
|
||||
}
|
||||
} else {
|
||||
const reason = error ? ` - [${error}]` : "";
|
||||
console.log(`${relativePath}${reason}`);
|
||||
function stripFrontmatter(raw) {
|
||||
if (!raw.startsWith("---\n") && !raw.startsWith("---\r\n")) {
|
||||
return raw;
|
||||
}
|
||||
const lines = raw.split(/\r?\n/u);
|
||||
for (let index = 1; index < lines.length; index += 1) {
|
||||
if (lines[index] === "---" || lines[index] === "...") {
|
||||
return lines.slice(index + 1).join("\n");
|
||||
}
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
console.log(
|
||||
'\nReminder: keep docs up to date as behavior changes. When your task matches any "Read when" hint above (React hooks, cache directives, database work, tests, etc.), read that doc before coding, and suggest new coverage when it is missing.',
|
||||
);
|
||||
function escapeMarkdownHtmlText(value) {
|
||||
return value.replace(/&/gu, "&").replace(/</gu, "<").replace(/>/gu, ">");
|
||||
}
|
||||
|
||||
function cleanHeadingText(value) {
|
||||
const codeSpans = [];
|
||||
const withCodePlaceholders = value.replace(/(`+)([^`\n]*?)\1/gu, (_match, _ticks, content) => {
|
||||
const index = codeSpans.push(content) - 1;
|
||||
return `\u{e000}${index}\u{e001}`;
|
||||
});
|
||||
const normalized = withCodePlaceholders
|
||||
.replace(/\s+#+\s*$/u, "")
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/gu, "$1")
|
||||
.replace(/[*_~`]/gu, "")
|
||||
.replace(/\s+/gu, " ")
|
||||
.trim()
|
||||
.replace(/\u{e000}(\d+)\u{e001}/gu, (_match, index) => {
|
||||
const content = codeSpans[Number(index)] ?? "";
|
||||
return /[*_~[\]()!]/u.test(content) ? `\`${content}\`` : content;
|
||||
});
|
||||
// Docs map is Markdown consumed by humans and agents. Escape HTML instead of
|
||||
// trying to strip tags so malformed source headings cannot reintroduce markup.
|
||||
return escapeMarkdownHtmlText(normalized);
|
||||
}
|
||||
|
||||
function extractHeadings(raw) {
|
||||
const headings = [];
|
||||
const lines = stripFrontmatter(raw).split(/\r?\n/u);
|
||||
let fenceMarker = null;
|
||||
|
||||
for (const rawLine of lines) {
|
||||
const trimmed = rawLine.trim();
|
||||
const fenceMatch = /^(?<marker>`{3,}|~{3,})/u.exec(trimmed);
|
||||
if (fenceMatch) {
|
||||
const marker = fenceMatch.groups.marker[0];
|
||||
fenceMarker = fenceMarker === marker ? null : (fenceMarker ?? marker);
|
||||
continue;
|
||||
}
|
||||
if (fenceMarker) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const match = /^(#{1,4})\s+(.+)$/u.exec(rawLine);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const text = cleanHeadingText(match[2]);
|
||||
if (text) {
|
||||
headings.push({ depth: match[1].length, text });
|
||||
}
|
||||
}
|
||||
|
||||
return headings;
|
||||
}
|
||||
|
||||
function routeForFile(relativePath) {
|
||||
const withoutExtension = relativePath.replace(/\.mdx?$/iu, "");
|
||||
if (withoutExtension === "index") {
|
||||
return "/";
|
||||
}
|
||||
if (withoutExtension.endsWith("/index")) {
|
||||
return `/${withoutExtension.slice(0, -"/index".length)}`;
|
||||
}
|
||||
return `/${withoutExtension}`;
|
||||
}
|
||||
|
||||
function normalizeDocsMapRelativePath(relativePath) {
|
||||
return relativePath.replace(/\\/gu, "/");
|
||||
}
|
||||
|
||||
/** Render the publish-only docs heading map without creating a source-tree mirror. */
|
||||
export function renderDocsHeadingMap(docsDir = DOCS_DIR, options = {}) {
|
||||
assertDocsDir(docsDir);
|
||||
const files = walkMarkdownFiles(docsDir, docsDir, {
|
||||
excludedDirs: DOCS_MAP_EXCLUDED_DIRS,
|
||||
excludedFiles: DOCS_MAP_EXCLUDED_FILES,
|
||||
relativePath: options.relativePath,
|
||||
})
|
||||
.map(normalizeDocsMapRelativePath)
|
||||
.toSorted((left, right) => (left < right ? -1 : left > right ? 1 : 0));
|
||||
const lines = [
|
||||
"---",
|
||||
'summary: "Generated heading map for OpenClaw docs pages"',
|
||||
'read_when: "Finding which docs page covers a topic before reading the page"',
|
||||
'title: "Docs map"',
|
||||
"---",
|
||||
"",
|
||||
"# OpenClaw docs map",
|
||||
"",
|
||||
"This file is generated from `docs/**/*.md` and `docs/**/*.mdx` headings to help agents navigate the documentation tree.",
|
||||
"Do not edit it by hand; run `pnpm docs:map:gen`.",
|
||||
"",
|
||||
];
|
||||
|
||||
for (const relativePath of files) {
|
||||
const headings = extractHeadings(readFileSync(join(docsDir, relativePath), "utf8"));
|
||||
lines.push(`## ${relativePath}`);
|
||||
lines.push("");
|
||||
lines.push(`- Route: ${routeForFile(relativePath)}`);
|
||||
if (headings.length === 0) {
|
||||
lines.push("- Headings: none");
|
||||
} else {
|
||||
lines.push("- Headings:");
|
||||
for (const heading of headings) {
|
||||
lines.push(` - H${heading.depth}: ${heading.text}`);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
function runDocsList(docsDir = DOCS_DIR) {
|
||||
assertDocsDir(docsDir);
|
||||
console.log("Listing all markdown files in docs folder:");
|
||||
const markdownFiles = walkMarkdownFiles(docsDir, docsDir, {
|
||||
excludedDirs: DOCS_LIST_EXCLUDED_DIRS,
|
||||
});
|
||||
|
||||
for (const relativePath of markdownFiles) {
|
||||
const fullPath = join(docsDir, relativePath);
|
||||
const { summary, readWhen, error } = extractMetadata(fullPath);
|
||||
if (summary) {
|
||||
console.log(`${relativePath} - ${summary}`);
|
||||
if (readWhen.length > 0) {
|
||||
console.log(` Read when: ${readWhen.join("; ")}`);
|
||||
}
|
||||
} else {
|
||||
const reason = error ? ` - [${error}]` : "";
|
||||
console.log(`${relativePath}${reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
'\nReminder: keep docs up to date as behavior changes. When your task matches any "Read when" hint above (React hooks, cache directives, database work, tests, etc.), read that doc before coding, and suggest new coverage when it is missing.',
|
||||
);
|
||||
}
|
||||
|
||||
function main(argv = process.argv.slice(2)) {
|
||||
process.stdout.on("error", (error) => {
|
||||
if (error?.code === "EPIPE") {
|
||||
process.exit(0);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
if (argv.length === 0) {
|
||||
runDocsList();
|
||||
return;
|
||||
}
|
||||
if (argv.length === 1 && argv[0] === "--headings") {
|
||||
process.stdout.write(renderDocsHeadingMap());
|
||||
return;
|
||||
}
|
||||
console.error("Usage: pnpm docs:list [--headings]");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
try {
|
||||
main();
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ export function applyLocaleNavLabelOverlay(
|
||||
): Record<string, unknown>;
|
||||
/** Composes the publish docs configuration with generated locale navigation. */
|
||||
export function composeDocsConfig(): Record<string, unknown>;
|
||||
/** Writes the public heading map into the publish tree without committing an expanded mirror. */
|
||||
export function writePublishedDocsMap(targetDocsDir: string): string;
|
||||
/**
|
||||
* Mirrors ClawHub docs into the target docs tree.
|
||||
*/
|
||||
|
||||
@@ -5,6 +5,7 @@ import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { renderDocsHeadingMap } from "./docs-list.js";
|
||||
import { repairMintlifyAccordionIndentation } from "./lib/mintlify-accordion.mjs";
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
@@ -784,6 +785,7 @@ function syncDocsTree(targetRoot, options = {}) {
|
||||
`${targetDocsDir}/`,
|
||||
]);
|
||||
pruneInternalDocs(targetDocsDir);
|
||||
writePublishedDocsMap(targetDocsDir);
|
||||
|
||||
for (const locale of GENERATED_LOCALES) {
|
||||
const sourceTmPath = path.join(SOURCE_DOCS_DIR, ".i18n", locale.tmFile);
|
||||
@@ -805,6 +807,13 @@ function syncDocsTree(targetRoot, options = {}) {
|
||||
return { clawhub: clawhubSource };
|
||||
}
|
||||
|
||||
/** Writes the public heading map into the publish tree without committing an expanded mirror. */
|
||||
export function writePublishedDocsMap(targetDocsDir) {
|
||||
const outputPath = path.join(targetDocsDir, "docs_map.md");
|
||||
fs.writeFileSync(outputPath, renderDocsHeadingMap(SOURCE_DOCS_DIR), "utf8");
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
function writeSyncMetadata(targetRoot, args, sources) {
|
||||
const metadata = {
|
||||
repository: args.sourceRepo || "",
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
export namespace testing {
|
||||
export { cleanHeadingText };
|
||||
}
|
||||
declare function cleanHeadingText(value: unknown): unknown;
|
||||
@@ -1,219 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Generates docs/docs_map.md from source docs headings for LLM navigation.
|
||||
import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = process.cwd();
|
||||
const DOCS_DIR = join(ROOT, "docs");
|
||||
const OUTPUT_PATH = join(DOCS_DIR, "docs_map.md");
|
||||
const MARKDOWN_EXTENSIONS = /\.mdx?$/iu;
|
||||
const EXCLUDED_DIRS = new Set([
|
||||
".generated",
|
||||
"archive",
|
||||
"assets",
|
||||
"images",
|
||||
"internal",
|
||||
"research",
|
||||
"snippets",
|
||||
]);
|
||||
const EXCLUDED_FILES = new Set(["AGENTS.md", "CLAUDE.md", "docs_map.md"]);
|
||||
|
||||
if (!existsSync(DOCS_DIR)) {
|
||||
console.error("docs:map: missing docs directory. Run from repo root.");
|
||||
process.exit(1);
|
||||
}
|
||||
if (!statSync(DOCS_DIR).isDirectory()) {
|
||||
console.error("docs:map: docs path is not a directory.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function normalizeSlashes(value) {
|
||||
return value.replace(/\\/gu, "/");
|
||||
}
|
||||
|
||||
function isMarkdownFile(name) {
|
||||
return MARKDOWN_EXTENSIONS.test(name);
|
||||
}
|
||||
|
||||
function shouldSkipFile(relativePath) {
|
||||
const parts = normalizeSlashes(relativePath).split("/");
|
||||
if (parts.some((part) => part.startsWith("."))) {
|
||||
return true;
|
||||
}
|
||||
if (parts.some((part) => EXCLUDED_DIRS.has(part))) {
|
||||
return true;
|
||||
}
|
||||
return EXCLUDED_FILES.has(parts.at(-1));
|
||||
}
|
||||
|
||||
function walkMarkdownFiles(dir, base = dir) {
|
||||
const files = [];
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name.startsWith(".")) {
|
||||
continue;
|
||||
}
|
||||
const fullPath = join(dir, entry.name);
|
||||
const relativePath = relative(base, fullPath);
|
||||
if (entry.isDirectory()) {
|
||||
if (EXCLUDED_DIRS.has(entry.name)) {
|
||||
continue;
|
||||
}
|
||||
files.push(...walkMarkdownFiles(fullPath, base));
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile() || !isMarkdownFile(entry.name) || shouldSkipFile(relativePath)) {
|
||||
continue;
|
||||
}
|
||||
files.push(normalizeSlashes(relativePath));
|
||||
}
|
||||
return files.toSorted((left, right) => (left < right ? -1 : left > right ? 1 : 0));
|
||||
}
|
||||
|
||||
function stripFrontmatter(raw) {
|
||||
if (!raw.startsWith("---\n") && !raw.startsWith("---\r\n")) {
|
||||
return raw;
|
||||
}
|
||||
const lines = raw.split(/\r?\n/u);
|
||||
for (let index = 1; index < lines.length; index += 1) {
|
||||
if (lines[index] === "---" || lines[index] === "...") {
|
||||
return lines.slice(index + 1).join("\n");
|
||||
}
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
function escapeMarkdownHtmlText(value) {
|
||||
return value.replace(/&/gu, "&").replace(/</gu, "<").replace(/>/gu, ">");
|
||||
}
|
||||
|
||||
function cleanHeadingText(value) {
|
||||
const codeSpans = [];
|
||||
const withCodePlaceholders = value.replace(/(`+)([^`\n]*?)\1/gu, (_match, _ticks, content) => {
|
||||
const index = codeSpans.push(content) - 1;
|
||||
return `\u{e000}${index}\u{e001}`;
|
||||
});
|
||||
const normalized = withCodePlaceholders
|
||||
.replace(/\s+#+\s*$/u, "")
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/gu, "$1")
|
||||
.replace(/[*_~`]/gu, "")
|
||||
.replace(/\s+/gu, " ")
|
||||
.trim()
|
||||
.replace(/\u{e000}(\d+)\u{e001}/gu, (_match, index) => {
|
||||
const content = codeSpans[Number(index)] ?? "";
|
||||
return /[*_~[\]()!]/u.test(content) ? `\`${content}\`` : content;
|
||||
});
|
||||
// Docs map is Markdown consumed by humans and agents. Escape HTML instead of
|
||||
// trying to strip tags so malformed source headings cannot reintroduce markup.
|
||||
return escapeMarkdownHtmlText(normalized);
|
||||
}
|
||||
|
||||
function extractHeadings(raw) {
|
||||
const headings = [];
|
||||
const lines = stripFrontmatter(raw).split(/\r?\n/u);
|
||||
let fenceMarker = null;
|
||||
|
||||
for (const rawLine of lines) {
|
||||
const trimmed = rawLine.trim();
|
||||
const fenceMatch = /^(?<marker>`{3,}|~{3,})/u.exec(trimmed);
|
||||
if (fenceMatch) {
|
||||
const marker = fenceMatch.groups.marker[0];
|
||||
fenceMarker = fenceMarker === marker ? null : (fenceMarker ?? marker);
|
||||
continue;
|
||||
}
|
||||
if (fenceMarker) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const match = /^(#{1,4})\s+(.+)$/u.exec(rawLine);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const text = cleanHeadingText(match[2]);
|
||||
if (text) {
|
||||
headings.push({ depth: match[1].length, text });
|
||||
}
|
||||
}
|
||||
|
||||
return headings;
|
||||
}
|
||||
|
||||
function routeForFile(relativePath) {
|
||||
const withoutExtension = relativePath.replace(/\.mdx?$/iu, "");
|
||||
if (withoutExtension === "index") {
|
||||
return "/";
|
||||
}
|
||||
if (withoutExtension.endsWith("/index")) {
|
||||
return `/${withoutExtension.slice(0, -"/index".length)}`;
|
||||
}
|
||||
return `/${withoutExtension}`;
|
||||
}
|
||||
|
||||
function renderDocsMap() {
|
||||
const files = walkMarkdownFiles(DOCS_DIR);
|
||||
const lines = [
|
||||
"---",
|
||||
'summary: "Generated heading map for OpenClaw docs pages"',
|
||||
'read_when: "Finding which docs page covers a topic before reading the page"',
|
||||
'title: "Docs map"',
|
||||
"---",
|
||||
"",
|
||||
"# OpenClaw docs map",
|
||||
"",
|
||||
"This file is generated from `docs/**/*.md` and `docs/**/*.mdx` headings to help agents navigate the documentation tree.",
|
||||
"Do not edit it by hand; run `pnpm docs:map:gen`.",
|
||||
"",
|
||||
];
|
||||
|
||||
for (const relativePath of files) {
|
||||
const fullPath = join(DOCS_DIR, relativePath);
|
||||
const headings = extractHeadings(readFileSync(fullPath, "utf8"));
|
||||
lines.push(`## ${relativePath}`);
|
||||
lines.push("");
|
||||
lines.push(`- Route: ${routeForFile(relativePath)}`);
|
||||
if (headings.length === 0) {
|
||||
lines.push("- Headings: none");
|
||||
} else {
|
||||
lines.push("- Headings:");
|
||||
for (const heading of headings) {
|
||||
lines.push(` - H${heading.depth}: ${heading.text}`);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const check = process.argv.includes("--check");
|
||||
const content = renderDocsMap();
|
||||
|
||||
if (check) {
|
||||
if (!existsSync(OUTPUT_PATH)) {
|
||||
console.error("docs:map: docs/docs_map.md is missing. Run `pnpm docs:map:gen`.");
|
||||
process.exit(1);
|
||||
}
|
||||
const current = readFileSync(OUTPUT_PATH, "utf8");
|
||||
if (current !== content) {
|
||||
console.error("docs:map: docs/docs_map.md is out of date. Run `pnpm docs:map:gen`.");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("docs:map: docs/docs_map.md is up to date.");
|
||||
return;
|
||||
}
|
||||
|
||||
writeFileSync(OUTPUT_PATH, content, "utf8");
|
||||
console.log(`docs:map: wrote ${normalizeSlashes(relative(ROOT, OUTPUT_PATH))}.`);
|
||||
}
|
||||
|
||||
const isMain = process.argv[1] ? fileURLToPath(import.meta.url) === process.argv[1] : false;
|
||||
|
||||
if (isMain) {
|
||||
main();
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
cleanHeadingText,
|
||||
};
|
||||
@@ -28,6 +28,12 @@ export function resolveRuntimePackEnvironment(
|
||||
now?: () => Date,
|
||||
readGitCommit?: () => string | null,
|
||||
): NodeJS.ProcessEnv & { OPENCLAW_BUILD_TIMESTAMP: string; GIT_COMMIT?: string };
|
||||
export function runPreparedRuntimePack<T>(
|
||||
prepare: () => void,
|
||||
pack: () => T,
|
||||
restore: () => void,
|
||||
): T;
|
||||
export function restoreRuntimePack(env: NodeJS.ProcessEnv, cwd?: string): void;
|
||||
export function rewriteWorkspaceDependencyVersions(
|
||||
packageJson: unknown,
|
||||
workspacePackages: unknown,
|
||||
|
||||
@@ -177,17 +177,34 @@ function prepareRuntimePack(profile, env) {
|
||||
});
|
||||
}
|
||||
|
||||
function restoreRuntimePack(env) {
|
||||
export function restoreRuntimePack(env, cwd = process.cwd()) {
|
||||
const script = `
|
||||
const mod = await import("./scripts/package-changelog.mjs");
|
||||
await mod.restorePackageChangelog();
|
||||
const { existsSync } = await import("node:fs");
|
||||
if (existsSync("./scripts/openclaw-postpack.mjs")) {
|
||||
const mod = await import("./scripts/openclaw-postpack.mjs");
|
||||
await mod.restorePrepackArtifacts();
|
||||
} else {
|
||||
// Historical source refs predate the composite lifecycle and only mutate CHANGELOG.md.
|
||||
const mod = await import("./scripts/package-changelog.mjs");
|
||||
await mod.restorePackageChangelog();
|
||||
}
|
||||
`;
|
||||
runChecked(process.execPath, ["--input-type=module", "--eval", script], {
|
||||
cwd,
|
||||
env,
|
||||
stdio: "inherit",
|
||||
});
|
||||
}
|
||||
|
||||
export function runPreparedRuntimePack(prepare, pack, restore) {
|
||||
prepare();
|
||||
try {
|
||||
return pack();
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
}
|
||||
|
||||
function packWorkspaceDependencies(npm, workspaceDirs, outputDir) {
|
||||
return workspaceDirs.map((packageDir) => {
|
||||
const packageJson = JSON.parse(readFileSync(join(packageDir, "package.json"), "utf8"));
|
||||
@@ -277,16 +294,17 @@ function main() {
|
||||
if (runtimePackPlan && runtimePackEnv && supportsPreparedRuntimePack(runtimePackEnv)) {
|
||||
// This adapter-only archive is installed into OCM and never published.
|
||||
// Standard npm pack still runs the full package build.
|
||||
try {
|
||||
prepareRuntimePack(runtimePackPlan.profile, runtimePackEnv);
|
||||
const result = runNpm(npm, runtimePackPlan.packArgs, {
|
||||
env: runtimePackEnv,
|
||||
stdio: "inherit",
|
||||
});
|
||||
return result.status ?? 1;
|
||||
} finally {
|
||||
restoreRuntimePack(runtimePackEnv);
|
||||
}
|
||||
return runPreparedRuntimePack(
|
||||
() => prepareRuntimePack(runtimePackPlan.profile, runtimePackEnv),
|
||||
() => {
|
||||
const result = runNpm(npm, runtimePackPlan.packArgs, {
|
||||
env: runtimePackEnv,
|
||||
stdio: "inherit",
|
||||
});
|
||||
return result.status ?? 1;
|
||||
},
|
||||
() => restoreRuntimePack(runtimePackEnv),
|
||||
);
|
||||
}
|
||||
const plan = resolveWorkspaceInstallPlan(args, workspaceDirs);
|
||||
if (!plan) {
|
||||
|
||||
2
scripts/openclaw-postpack.d.mts
Normal file
2
scripts/openclaw-postpack.d.mts
Normal file
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env node
|
||||
export function restorePrepackArtifacts(cwd?: string): Promise<void>;
|
||||
22
scripts/openclaw-postpack.mjs
Executable file
22
scripts/openclaw-postpack.mjs
Executable file
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
// Restores every source artifact temporarily rewritten for npm packaging.
|
||||
import { restorePackageChangelog } from "./package-changelog.mjs";
|
||||
import { restorePackageDocsMap } from "./package-docs-map.mjs";
|
||||
|
||||
export async function restorePrepackArtifacts(cwd = process.cwd()) {
|
||||
await restorePackageChangelog(cwd);
|
||||
// Release the lifecycle receipt only after every other source mutation settles.
|
||||
await restorePackageDocsMap(cwd);
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
try {
|
||||
await restorePrepackArtifacts();
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,9 @@ import { pathToFileURL } from "node:url";
|
||||
import { formatErrorMessage } from "../src/infra/errors.ts";
|
||||
import { readPositiveEnvInt } from "./lib/numeric-options.mjs";
|
||||
import { writePackageDistInventoryForPublish } from "./lib/package-dist-inventory.ts";
|
||||
import { restorePrepackArtifacts } from "./openclaw-postpack.mjs";
|
||||
import { preparePackageChangelog } from "./package-changelog.mjs";
|
||||
import { preparePackageDocsMap } from "./package-docs-map.mjs";
|
||||
import { createPnpmRunnerSpawnSpec } from "./pnpm-runner.mjs";
|
||||
const FULL_GIT_COMMIT_RE = /^[0-9a-f]{40}$/iu;
|
||||
const requiredPreparedPathGroups = [
|
||||
@@ -285,9 +287,29 @@ export async function preparePrepackArtifacts(env: NodeJS.ProcessEnv = process.e
|
||||
ensurePreparedArtifacts();
|
||||
await writeDistInventory();
|
||||
runBuildSmoke();
|
||||
await preparePackageChangelog(process.cwd(), {
|
||||
allowUnreleased: resolvePrepackAllowUnreleasedChangelog(env),
|
||||
});
|
||||
// The docs-map receipt serializes source-mutating pack lifecycles before the
|
||||
// changelog is touched, so concurrent packs cannot restore each other's files.
|
||||
await preparePackageDocsMap(process.cwd());
|
||||
try {
|
||||
await preparePackageChangelog(process.cwd(), {
|
||||
allowUnreleased: resolvePrepackAllowUnreleasedChangelog(env),
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await restorePrepackArtifacts(process.cwd());
|
||||
} catch (restoreError) {
|
||||
throw prepackPreparationRestoreError(error, restoreError);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function prepackPreparationRestoreError(error: unknown, restoreError: unknown): AggregateError {
|
||||
return new AggregateError(
|
||||
[error, restoreError],
|
||||
"Prepack preparation failed and source artifacts could not be restored.",
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
|
||||
2
scripts/package-docs-map.d.mts
Normal file
2
scripts/package-docs-map.d.mts
Normal file
@@ -0,0 +1,2 @@
|
||||
export function restorePackageDocsMap(cwd?: string): Promise<boolean>;
|
||||
export function preparePackageDocsMap(cwd?: string): Promise<boolean>;
|
||||
127
scripts/package-docs-map.mjs
Executable file
127
scripts/package-docs-map.mjs
Executable file
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Materializes the generated docs map only while npm assembles a package tarball.
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { renderDocsHeadingMap } from "./docs-list.js";
|
||||
|
||||
const DOCS_MAP_PATH = path.join("docs", "docs_map.md");
|
||||
const RECEIPT_PATH = path.join(".artifacts", "package-docs-map", "receipt.json");
|
||||
|
||||
function activePreparationError() {
|
||||
return Object.assign(
|
||||
new Error(
|
||||
`Another package preparation owns ${DOCS_MAP_PATH}; wait for it to finish or run \`node scripts/openclaw-postpack.mjs\` after an interrupted pack.`,
|
||||
),
|
||||
{ code: "PACKAGE_DOCS_MAP_ACTIVE" },
|
||||
);
|
||||
}
|
||||
|
||||
function preparationRestoreError(error, restoreError) {
|
||||
return new AggregateError(
|
||||
[error, restoreError],
|
||||
`Writing ${DOCS_MAP_PATH} failed and its source state could not be restored.`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
function sha256(content) {
|
||||
return createHash("sha256").update(content).digest("hex");
|
||||
}
|
||||
|
||||
/** Restore the source map stub after prepack replaced it with generated content. */
|
||||
export async function restorePackageDocsMap(cwd = process.cwd()) {
|
||||
const receiptPath = path.join(cwd, RECEIPT_PATH);
|
||||
if (!existsSync(receiptPath)) {
|
||||
return false;
|
||||
}
|
||||
const receipt = JSON.parse(await readFile(receiptPath, "utf8"));
|
||||
if (
|
||||
!receipt ||
|
||||
typeof receipt.generatedSha256 !== "string" ||
|
||||
!/^[0-9a-f]{64}$/u.test(receipt.generatedSha256) ||
|
||||
(receipt.original !== null && typeof receipt.original !== "string")
|
||||
) {
|
||||
throw new Error(`Invalid package docs-map receipt at ${RECEIPT_PATH}.`);
|
||||
}
|
||||
|
||||
const mapPath = path.join(cwd, DOCS_MAP_PATH);
|
||||
const current = existsSync(mapPath) ? await readFile(mapPath, "utf8") : null;
|
||||
if (current === receipt.original) {
|
||||
await rm(receiptPath, { force: true });
|
||||
return true;
|
||||
}
|
||||
if (current === null || sha256(current) !== receipt.generatedSha256) {
|
||||
throw new Error(
|
||||
`Refusing to restore ${DOCS_MAP_PATH} because it changed after prepack generated it.`,
|
||||
);
|
||||
}
|
||||
if (receipt.original === null) {
|
||||
await rm(mapPath);
|
||||
} else {
|
||||
await writeFile(mapPath, receipt.original, "utf8");
|
||||
}
|
||||
await rm(receiptPath, { force: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Generate the shipped docs map for npm while recording the source state for cleanup. */
|
||||
export async function preparePackageDocsMap(cwd = process.cwd()) {
|
||||
const receiptPath = path.join(cwd, RECEIPT_PATH);
|
||||
if (existsSync(receiptPath)) {
|
||||
throw activePreparationError();
|
||||
}
|
||||
const mapPath = path.join(cwd, DOCS_MAP_PATH);
|
||||
const content = renderDocsHeadingMap(path.join(cwd, "docs"));
|
||||
const original = existsSync(mapPath) ? await readFile(mapPath, "utf8") : null;
|
||||
if (original === content) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await mkdir(path.dirname(receiptPath), { recursive: true });
|
||||
try {
|
||||
await writeFile(
|
||||
receiptPath,
|
||||
`${JSON.stringify({ generatedSha256: sha256(content), original })}\n`,
|
||||
{ encoding: "utf8", flag: "wx" },
|
||||
);
|
||||
} catch (error) {
|
||||
if (error?.code === "EEXIST") {
|
||||
throw activePreparationError();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
await writeFile(mapPath, content, "utf8");
|
||||
} catch (error) {
|
||||
try {
|
||||
await restorePackageDocsMap(cwd);
|
||||
} catch (restoreError) {
|
||||
throw preparationRestoreError(error, restoreError);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function main(argv = process.argv.slice(2)) {
|
||||
if (argv.length !== 1 || (argv[0] !== "prepare" && argv[0] !== "restore")) {
|
||||
console.error("Usage: node scripts/package-docs-map.mjs <prepare|restore>");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const changed =
|
||||
argv[0] === "prepare" ? await preparePackageDocsMap() : await restorePackageDocsMap();
|
||||
console.error(
|
||||
changed
|
||||
? `package-docs-map: ${argv[0] === "prepare" ? "generated" : "removed"} transient docs map.`
|
||||
: `package-docs-map: no ${argv[0] === "prepare" ? "generation" : "cleanup"} needed.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
await main();
|
||||
}
|
||||
@@ -671,6 +671,40 @@ export async function prepareBundledAiRuntimePackage(
|
||||
}
|
||||
}
|
||||
|
||||
async function restorePackageSourceArtifacts(sourceDir, restoreDocsMap, restoreChangelog) {
|
||||
await restoreChangelog(sourceDir);
|
||||
// Release the lifecycle receipt only after every other source mutation settles.
|
||||
await restoreDocsMap(sourceDir);
|
||||
}
|
||||
|
||||
async function loadSourceDocsMapLifecycle(sourceDir) {
|
||||
const modulePath = path.join(sourceDir, "scripts", "package-docs-map.mjs");
|
||||
try {
|
||||
await fs.access(modulePath);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const lifecycle = await import(pathToFileURL(modulePath).href);
|
||||
if (
|
||||
typeof lifecycle.preparePackageDocsMap !== "function" ||
|
||||
typeof lifecycle.restorePackageDocsMap !== "function"
|
||||
) {
|
||||
throw new Error(`source package docs-map lifecycle is invalid: ${modulePath}`);
|
||||
}
|
||||
return lifecycle;
|
||||
}
|
||||
|
||||
function packagePreparationRestoreError(error, restoreError) {
|
||||
return new AggregateError(
|
||||
[error, restoreError],
|
||||
"Package preparation failed and source artifacts could not be restored.",
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
export async function packOpenClawPackageForDocker(sourceDir, outputDir, options = {}) {
|
||||
const runCaptureImpl = options.runCaptureImpl ?? runCapture;
|
||||
const prepareChangelog =
|
||||
@@ -680,13 +714,33 @@ export async function packOpenClawPackageForDocker(sourceDir, outputDir, options
|
||||
allowUnreleased: options.allowUnreleasedChangelog,
|
||||
}));
|
||||
const restoreChangelog = options.restoreChangelog ?? restorePackageChangelog;
|
||||
// Frozen refs own their package contents. Only refs carrying this lifecycle ship a generated map.
|
||||
const sourceDocsMapLifecycle =
|
||||
options.prepareDocsMap && options.restoreDocsMap
|
||||
? null
|
||||
: await loadSourceDocsMapLifecycle(sourceDir);
|
||||
const prepareDocsMap =
|
||||
options.prepareDocsMap ?? sourceDocsMapLifecycle?.preparePackageDocsMap ?? (async () => false);
|
||||
const restoreDocsMap =
|
||||
options.restoreDocsMap ?? sourceDocsMapLifecycle?.restorePackageDocsMap ?? (async () => false);
|
||||
const prepareBundledAiRuntime = options.prepareBundledAiRuntime ?? prepareBundledAiRuntimePackage;
|
||||
const packTool = options.pnpmPack ? "pnpm" : "npm";
|
||||
if (options.packJsonPath && options.pnpmPack) {
|
||||
throw new Error("packJsonPath cannot be combined with pnpmPack");
|
||||
}
|
||||
console.error("==> Packing OpenClaw package");
|
||||
await prepareChangelog(sourceDir);
|
||||
// This receipt is the package lifecycle lock; acquire it before touching CHANGELOG.md.
|
||||
await prepareDocsMap(sourceDir);
|
||||
try {
|
||||
await prepareChangelog(sourceDir);
|
||||
} catch (error) {
|
||||
try {
|
||||
await restorePackageSourceArtifacts(sourceDir, restoreDocsMap, restoreChangelog);
|
||||
} catch (restoreError) {
|
||||
throw packagePreparationRestoreError(error, restoreError);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
let packOutput;
|
||||
let cleanupBundledAiRuntime = async () => {};
|
||||
try {
|
||||
@@ -714,7 +768,7 @@ export async function packOpenClawPackageForDocker(sourceDir, outputDir, options
|
||||
try {
|
||||
await cleanupBundledAiRuntime();
|
||||
} finally {
|
||||
await restoreChangelog(sourceDir);
|
||||
await restorePackageSourceArtifacts(sourceDir, restoreDocsMap, restoreChangelog);
|
||||
}
|
||||
}
|
||||
// pnpm reports an absolute destination path. The directory was emptied before packing,
|
||||
@@ -797,7 +851,10 @@ async function main() {
|
||||
process.stdout.write(`${tarball}\n`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
if (
|
||||
process.argv[1] &&
|
||||
(await fs.realpath(process.argv[1])) === (await fs.realpath(fileURLToPath(import.meta.url)))
|
||||
) {
|
||||
await main().catch(
|
||||
/** @param {unknown} error */ (error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
|
||||
Reference in New Issue
Block a user