From 08492dfeee92aa451e24766b8e2723653722d59e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 5 Apr 2026 22:16:55 +0100 Subject: [PATCH] feat(memory-wiki): compile related backlinks blocks --- extensions/memory-wiki/src/compile.test.ts | 101 +++++++++++++ extensions/memory-wiki/src/compile.ts | 165 ++++++++++++++++++++- extensions/memory-wiki/src/markdown.ts | 20 ++- 3 files changed, 283 insertions(+), 3 deletions(-) diff --git a/extensions/memory-wiki/src/compile.test.ts b/extensions/memory-wiki/src/compile.test.ts index 3f99186c271..4b94384929a 100644 --- a/extensions/memory-wiki/src/compile.test.ts +++ b/extensions/memory-wiki/src/compile.test.ts @@ -67,4 +67,105 @@ describe("compileMemoryWikiVault", () => { "[[sources/alpha|Alpha]]", ); }); + + it("writes related blocks from source ids and shared sources", async () => { + const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-compile-")); + tempDirs.push(rootDir); + const config = resolveMemoryWikiConfig( + { vault: { path: rootDir } }, + { homedir: "/Users/tester" }, + ); + await initializeMemoryWikiVault(config); + + await fs.writeFile( + path.join(rootDir, "sources", "alpha.md"), + renderWikiMarkdown({ + frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha" }, + body: "# Alpha\n", + }), + "utf8", + ); + await fs.writeFile( + path.join(rootDir, "entities", "beta.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "entity", + id: "entity.beta", + title: "Beta", + sourceIds: ["source.alpha"], + }, + body: "# Beta\n", + }), + "utf8", + ); + await fs.writeFile( + path.join(rootDir, "concepts", "gamma.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "concept", + id: "concept.gamma", + title: "Gamma", + sourceIds: ["source.alpha"], + }, + body: "# Gamma\n", + }), + "utf8", + ); + + await compileMemoryWikiVault(config); + + await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain( + "## Related", + ); + await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain( + "[Alpha](sources/alpha.md)", + ); + await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain( + "[Gamma](concepts/gamma.md)", + ); + await expect(fs.readFile(path.join(rootDir, "sources", "alpha.md"), "utf8")).resolves.toContain( + "[Beta](entities/beta.md)", + ); + await expect(fs.readFile(path.join(rootDir, "sources", "alpha.md"), "utf8")).resolves.toContain( + "[Gamma](concepts/gamma.md)", + ); + }); + + it("ignores generated related links when computing backlinks on repeated compile", async () => { + const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-compile-")); + tempDirs.push(rootDir); + const config = resolveMemoryWikiConfig( + { vault: { path: rootDir } }, + { homedir: "/Users/tester" }, + ); + await initializeMemoryWikiVault(config); + + await fs.writeFile( + path.join(rootDir, "entities", "beta.md"), + renderWikiMarkdown({ + frontmatter: { pageType: "entity", id: "entity.beta", title: "Beta" }, + body: "# Beta\n", + }), + "utf8", + ); + await fs.writeFile( + path.join(rootDir, "concepts", "gamma.md"), + renderWikiMarkdown({ + frontmatter: { pageType: "concept", id: "concept.gamma", title: "Gamma" }, + body: "# Gamma\n\nSee [Beta](entities/beta.md).\n", + }), + "utf8", + ); + + await compileMemoryWikiVault(config); + const second = await compileMemoryWikiVault(config); + + expect(second.updatedFiles).toEqual([]); + await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain( + "[Gamma](concepts/gamma.md)", + ); + await expect( + fs.readFile(path.join(rootDir, "concepts", "gamma.md"), "utf8"), + ).resolves.not.toContain("### Referenced By"); + }); }); diff --git a/extensions/memory-wiki/src/compile.ts b/extensions/memory-wiki/src/compile.ts index 2b25aadf9f4..56824027f29 100644 --- a/extensions/memory-wiki/src/compile.ts +++ b/extensions/memory-wiki/src/compile.ts @@ -11,6 +11,8 @@ import { toWikiPageSummary, type WikiPageKind, type WikiPageSummary, + WIKI_RELATED_END_MARKER, + WIKI_RELATED_START_MARKER, } from "./markdown.js"; import { initializeMemoryWikiVault } from "./vault.js"; @@ -73,6 +75,162 @@ function buildPageCounts(pages: WikiPageSummary[]): Record }; } +function normalizeComparableTarget(value: string): string { + return value + .trim() + .replace(/\\/g, "/") + .replace(/\.md$/i, "") + .replace(/^\.\/+/, "") + .replace(/\/+$/, "") + .toLowerCase(); +} + +function uniquePages(pages: WikiPageSummary[]): WikiPageSummary[] { + const seen = new Set(); + const unique: WikiPageSummary[] = []; + for (const page of pages) { + const key = page.id ?? page.relativePath; + if (seen.has(key)) { + continue; + } + seen.add(key); + unique.push(page); + } + return unique; +} + +function buildPageLookupKeys(page: WikiPageSummary): Set { + const keys = new Set(); + keys.add(normalizeComparableTarget(page.relativePath)); + keys.add(normalizeComparableTarget(page.relativePath.replace(/\.md$/i, ""))); + keys.add(normalizeComparableTarget(page.title)); + if (page.id) { + keys.add(normalizeComparableTarget(page.id)); + } + return keys; +} + +function renderWikiPageLinks(params: { + config: ResolvedMemoryWikiConfig; + pages: WikiPageSummary[]; +}): string { + return params.pages + .map( + (page) => + `- ${formatWikiLink({ + renderMode: params.config.vault.renderMode, + relativePath: page.relativePath, + title: page.title, + })}`, + ) + .join("\n"); +} + +function buildRelatedBlockBody(params: { + config: ResolvedMemoryWikiConfig; + page: WikiPageSummary; + allPages: WikiPageSummary[]; +}): string { + const pagesById = new Map( + params.allPages.flatMap((candidate) => + candidate.id ? [[candidate.id, candidate] as const] : [], + ), + ); + const sourcePages = uniquePages( + params.page.sourceIds.flatMap((sourceId) => { + const page = pagesById.get(sourceId); + return page ? [page] : []; + }), + ); + const backlinkKeys = buildPageLookupKeys(params.page); + const backlinks = uniquePages( + params.allPages.filter((candidate) => { + if (candidate.relativePath === params.page.relativePath) { + return false; + } + if (candidate.sourceIds.includes(params.page.id ?? "")) { + return true; + } + return candidate.linkTargets.some((target) => + backlinkKeys.has(normalizeComparableTarget(target)), + ); + }), + ); + const relatedPages = uniquePages( + params.allPages.filter((candidate) => { + if (candidate.relativePath === params.page.relativePath) { + return false; + } + if (sourcePages.some((sourcePage) => sourcePage.relativePath === candidate.relativePath)) { + return false; + } + if (backlinks.some((backlink) => backlink.relativePath === candidate.relativePath)) { + return false; + } + if (params.page.sourceIds.length === 0 || candidate.sourceIds.length === 0) { + return false; + } + return params.page.sourceIds.some((sourceId) => candidate.sourceIds.includes(sourceId)); + }), + ); + + const sections: string[] = []; + if (sourcePages.length > 0) { + sections.push( + "### Sources", + renderWikiPageLinks({ config: params.config, pages: sourcePages }), + ); + } + if (backlinks.length > 0) { + sections.push( + "### Referenced By", + renderWikiPageLinks({ config: params.config, pages: backlinks }), + ); + } + if (relatedPages.length > 0) { + sections.push( + "### Related Pages", + renderWikiPageLinks({ config: params.config, pages: relatedPages }), + ); + } + if (sections.length === 0) { + return "- No related pages yet."; + } + return sections.join("\n\n"); +} + +async function refreshPageRelatedBlocks(params: { + config: ResolvedMemoryWikiConfig; + pages: WikiPageSummary[]; +}): Promise { + if (!params.config.render.createBacklinks) { + return []; + } + const updatedFiles: string[] = []; + for (const page of params.pages) { + const original = await fs.readFile(page.absolutePath, "utf8"); + const updated = withTrailingNewline( + replaceManagedMarkdownBlock({ + original, + heading: "## Related", + startMarker: WIKI_RELATED_START_MARKER, + endMarker: WIKI_RELATED_END_MARKER, + body: buildRelatedBlockBody({ + config: params.config, + page, + allPages: params.pages, + }), + }), + ); + if (updated === original) { + continue; + } + await fs.writeFile(page.absolutePath, updated, "utf8"); + updatedFiles.push(page.absolutePath); + } + return updatedFiles; +} + function renderSectionList(params: { config: ResolvedMemoryWikiConfig; pages: WikiPageSummary[]; @@ -162,9 +320,12 @@ export async function compileMemoryWikiVault( ): Promise { await initializeMemoryWikiVault(config); const rootDir = config.vault.path; - const pages = await readPageSummaries(rootDir); + let pages = await readPageSummaries(rootDir); + const updatedFiles = await refreshPageRelatedBlocks({ config, pages }); + if (updatedFiles.length > 0) { + pages = await readPageSummaries(rootDir); + } const counts = buildPageCounts(pages); - const updatedFiles: string[] = []; const rootIndexPath = path.join(rootDir, "index.md"); if ( diff --git a/extensions/memory-wiki/src/markdown.ts b/extensions/memory-wiki/src/markdown.ts index e9bf3a1f8b5..591a4061bc8 100644 --- a/extensions/memory-wiki/src/markdown.ts +++ b/extensions/memory-wiki/src/markdown.ts @@ -2,6 +2,8 @@ import path from "node:path"; import YAML from "yaml"; export const WIKI_PAGE_KINDS = ["entity", "concept", "source", "synthesis", "report"] as const; +export const WIKI_RELATED_START_MARKER = ""; +export const WIKI_RELATED_END_MARKER = ""; export type WikiPageKind = (typeof WIKI_PAGE_KINDS)[number]; @@ -38,6 +40,11 @@ function normalizeOptionalString(value: unknown): string | undefined { const FRONTMATTER_PATTERN = /^---\n([\s\S]*?)\n---\n?/; const OBSIDIAN_LINK_PATTERN = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g; +const MARKDOWN_LINK_PATTERN = /\[[^\]]+\]\(([^)]+)\)/g; +const RELATED_BLOCK_PATTERN = new RegExp( + `${WIKI_RELATED_START_MARKER}[\\s\\S]*?${WIKI_RELATED_END_MARKER}`, + "g", +); export function slugifyWikiSegment(raw: string): string { const slug = raw @@ -98,13 +105,24 @@ function normalizeStringList(value: unknown): string[] { } export function extractWikiLinks(markdown: string): string[] { + const searchable = markdown.replace(RELATED_BLOCK_PATTERN, ""); const links: string[] = []; - for (const match of markdown.matchAll(OBSIDIAN_LINK_PATTERN)) { + for (const match of searchable.matchAll(OBSIDIAN_LINK_PATTERN)) { const target = match[1]?.trim(); if (target) { links.push(target); } } + for (const match of searchable.matchAll(MARKDOWN_LINK_PATTERN)) { + const rawTarget = match[1]?.trim(); + if (!rawTarget || rawTarget.startsWith("#") || /^[a-z]+:/i.test(rawTarget)) { + continue; + } + const target = rawTarget.split("#")[0]?.split("?")[0]?.replace(/\\/g, "/").trim(); + if (target) { + links.push(target); + } + } return links; }