diff --git a/docs/cli/wiki.md b/docs/cli/wiki.md index 50901f0aa28..37dda2de22b 100644 --- a/docs/cli/wiki.md +++ b/docs/cli/wiki.md @@ -35,6 +35,7 @@ openclaw wiki status openclaw wiki doctor openclaw wiki init openclaw wiki ingest ./notes/alpha.md +openclaw wiki okf import ./knowledge-catalog/okf/bundles/ga4 openclaw wiki compile openclaw wiki lint openclaw wiki search "alpha" @@ -104,6 +105,31 @@ Notes: - imported source pages keep provenance in frontmatter - auto-compile can run after ingest when enabled +### `wiki okf import ` + +Import an unpacked Open Knowledge Format bundle into wiki concept pages. + +The importer reads every non-reserved `.md` concept document in the OKF +directory tree, requires a non-empty `type` field, and treats unknown OKF +`type` values as generic concepts. Reserved OKF `index.md` and `log.md` files +are not imported as concepts. + +Imported pages are flattened under `concepts/` so existing wiki compile, +search, get, digest, and dashboard flows see them immediately. The original OKF +concept ID, `type`, `resource`, `tags`, timestamp, source path, and full +frontmatter are preserved in the page frontmatter. Internal OKF markdown links +are rewritten to the generated wiki pages; broken or external links are left +unchanged. + +Examples: + +```bash +openclaw wiki okf import ./bundles/ga4 +openclaw wiki okf import ./bundles/ga4 --json +openclaw wiki search "BigQuery Table" --mode source-evidence --json +openclaw wiki get +``` + ### `wiki compile` Rebuild indexes, related blocks, dashboards, and compiled digests. @@ -233,6 +259,8 @@ These require the official `obsidian` CLI on `PATH` when - Use `wiki lint` before trusting contradictory or low-confidence content. - Use `wiki compile` after bulk imports or source changes when you want fresh dashboards and compiled digests immediately. +- Use `wiki okf import` when a data catalog, documentation export, or agent + enrichment pipeline already emits OKF markdown bundles. - Use `wiki bridge import` when bridge mode depends on newly exported memory artifacts. diff --git a/docs/plugins/memory-wiki.md b/docs/plugins/memory-wiki.md index 17b04a102b9..8aa0e464e6b 100644 --- a/docs/plugins/memory-wiki.md +++ b/docs/plugins/memory-wiki.md @@ -25,6 +25,7 @@ less like a pile of Markdown files. - Page-level provenance, confidence, contradictions, and open questions - Compiled digests for agent/runtime consumers - Wiki-native search/get/apply/lint tools +- Open Knowledge Format imports into compiled wiki concepts - Optional bridge mode that imports public artifacts from the active memory plugin - Optional Obsidian-friendly render mode and CLI integration @@ -135,6 +136,34 @@ The main page groups are: - `syntheses/` for compiled summaries and maintained rollups - `reports/` for generated dashboards +## Open Knowledge Format imports + +`memory-wiki` can import unpacked Open Knowledge Format bundles with: + +```bash +openclaw wiki okf import ./bundles/ga4 +``` + +This is the cleanest fit when a data catalog, documentation crawler, or +enrichment agent already produces OKF: keep OKF as the portable exchange +artifact, then let `memory-wiki` turn it into OpenClaw-native concept pages and +compiled digests. + +The importer follows the OKF v0.1 shape: + +- non-reserved `.md` files are concept documents +- each imported concept needs a non-empty `type` frontmatter field +- unknown OKF `type` values are accepted +- reserved `index.md` and `log.md` files are not imported as concepts +- broken or external markdown links are preserved + +Imported concept pages are flattened under `concepts/` so the existing compile, +search, get, dashboard, and prompt-digest paths see them without adding a second +wiki tree. Each page keeps the original OKF concept ID, source path, `type`, +`resource`, `tags`, timestamp, and full producer frontmatter. Internal OKF links +are rewritten to the generated wiki concept pages and also emitted as structured +`relationships` entries with `kind: okf-link`. + ## Structured claims and evidence Pages can carry structured `claims` frontmatter, not just freeform text. diff --git a/extensions/memory-wiki/src/cli.test.ts b/extensions/memory-wiki/src/cli.test.ts index 667bb5ca7ad..c98de2b71f3 100644 --- a/extensions/memory-wiki/src/cli.test.ts +++ b/extensions/memory-wiki/src/cli.test.ts @@ -10,6 +10,7 @@ import { runWikiChatGptImport, runWikiChatGptRollback, runWikiDoctor, + runWikiOkfImport, runWikiStatus, } from "./cli.js"; import type { MemoryWikiPluginConfig } from "./config.js"; @@ -27,6 +28,7 @@ vi.mock("openclaw/plugin-sdk/gateway-runtime", () => ({ const { createVault } = createMemoryWikiTestHarness(); let suiteRoot = ""; let caseIndex = 0; +let stdoutWriteMock: ReturnType; describe("memory-wiki cli", () => { beforeAll(async () => { @@ -41,8 +43,9 @@ describe("memory-wiki cli", () => { beforeEach(() => { callGatewayFromCliMock.mockReset(); + stdoutWriteMock = vi.fn(() => true); vi.spyOn(process.stdout, "write").mockImplementation( - (() => true) as typeof process.stdout.write, + stdoutWriteMock as unknown as typeof process.stdout.write, ); process.exitCode = undefined; }); @@ -174,6 +177,65 @@ describe("memory-wiki cli", () => { ); }); + it("registers OKF import and searches imported concepts", async () => { + const { rootDir, config } = await createCliVault(); + const bundlePath = path.join(rootDir, "okf-bundle"); + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + await fs.writeFile(path.join(bundlePath, "index.md"), "# Sales OKF\n", "utf8"); + await fs.writeFile( + path.join(bundlePath, "tables", "customers.md"), + `--- +type: BigQuery Table +title: Customers +--- + +Customer rows. +`, + "utf8", + ); + await fs.writeFile( + path.join(bundlePath, "tables", "orders.md"), + `--- +type: BigQuery Table +title: Orders +description: One row per completed order. +--- + +Orders join to [customers](/tables/customers.md). +`, + "utf8", + ); + + const program = new Command(); + program.name("test"); + registerWikiCli(program, config); + + await program.parseAsync(["wiki", "okf", "import", bundlePath, "--json"], { from: "user" }); + + const importOutput = String(stdoutWriteMock.mock.calls.at(-1)?.[0] ?? ""); + const importResult = JSON.parse(importOutput) as Awaited>; + expect(importResult.importedCount).toBe(2); + expect(importResult.pagePaths).toEqual( + expect.arrayContaining([ + expect.stringMatching(/^concepts\/okf-okf-bundle-[0-9a-f]{8}-tables-orders-/), + ]), + ); + + stdoutWriteMock.mockClear(); + await program.parseAsync(["wiki", "search", "completed order", "--json"], { from: "user" }); + + const searchOutput = String(stdoutWriteMock.mock.calls.at(-1)?.[0] ?? ""); + const searchResults = JSON.parse(searchOutput) as Array<{ path: string; title: string }>; + expect(searchResults).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + title: "Orders", + path: expect.stringMatching(/^concepts\/okf-okf-bundle-[0-9a-f]{8}-tables-orders-/), + }), + ]), + ); + }); + it("rejects apply confidence values outside the documented range", async () => { const { config } = await createCliVault(); const program = new Command(); diff --git a/extensions/memory-wiki/src/cli.ts b/extensions/memory-wiki/src/cli.ts index c4e8d819d2a..4cba66cdcbe 100644 --- a/extensions/memory-wiki/src/cli.ts +++ b/extensions/memory-wiki/src/cli.ts @@ -33,6 +33,7 @@ import { runObsidianOpen, runObsidianSearch, } from "./obsidian.js"; +import { formatOkfImportSummary, importMemoryWikiOkfBundle } from "./okf.js"; import { getMemoryWikiPage, searchMemoryWiki, @@ -88,6 +89,10 @@ type WikiIngestCommandOptions = { title?: string; }; +type WikiOkfImportCommandOptions = { + json?: boolean; +}; + type WikiSearchCommandOptions = { json?: boolean; maxResults?: number; @@ -590,6 +595,24 @@ export async function runWikiIngest(params: { }); } +export async function runWikiOkfImport(params: { + config: ResolvedMemoryWikiConfig; + bundlePath: string; + json?: boolean; + stdout?: Pick; +}) { + return runWikiCommandWithSummary({ + json: params.json, + stdout: params.stdout, + run: () => + importMemoryWikiOkfBundle({ + config: params.config, + bundlePath: params.bundlePath, + }), + render: formatOkfImportSummary, + }); +} + export async function runWikiSearch(params: { config: ResolvedMemoryWikiConfig; appConfig?: OpenClawConfig; @@ -965,6 +988,16 @@ export function registerWikiCli( await runWikiIngest({ config, inputPath, title: opts.title, json: opts.json }); }); + const okf = wiki.command("okf").description("Import Open Knowledge Format bundles"); + okf + .command("import") + .description("Import an unpacked OKF bundle into wiki concept pages") + .argument("", "OKF bundle directory") + .option("--json", "Print JSON") + .action(async (bundlePath: string, opts: WikiOkfImportCommandOptions) => { + await runWikiOkfImport({ config, bundlePath, json: opts.json }); + }); + addWikiSearchConfigOptions( wiki .command("search") diff --git a/extensions/memory-wiki/src/log.ts b/extensions/memory-wiki/src/log.ts index 837ce247ac2..f500bb4fc6c 100644 --- a/extensions/memory-wiki/src/log.ts +++ b/extensions/memory-wiki/src/log.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime"; type MemoryWikiLogEntry = { - type: "init" | "ingest" | "compile" | "lint"; + type: "init" | "ingest" | "okf-import" | "compile" | "lint"; timestamp: string; details?: Record; }; diff --git a/extensions/memory-wiki/src/okf.test.ts b/extensions/memory-wiki/src/okf.test.ts new file mode 100644 index 00000000000..39eff5717f9 --- /dev/null +++ b/extensions/memory-wiki/src/okf.test.ts @@ -0,0 +1,609 @@ +// Memory Wiki tests cover Open Knowledge Format import behavior. +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { parseWikiMarkdown } from "./markdown.js"; +import { importMemoryWikiOkfBundle } from "./okf.js"; +import { searchMemoryWiki } from "./query.js"; +import { createMemoryWikiTestHarness } from "./test-helpers.js"; + +const { createTempDir, createVault } = createMemoryWikiTestHarness(); + +function getOnlyPagePath(paths: string[]): string { + expect(paths).toHaveLength(1); + const [pagePath] = paths; + if (!pagePath) { + throw new Error("Expected OKF import to produce one page path."); + } + return pagePath; +} + +async function writeOkfBundle(rootDir: string) { + const bundlePath = path.join(rootDir, "sales-okf"); + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + await fs.mkdir(path.join(bundlePath, "metrics"), { recursive: true }); + await fs.writeFile( + path.join(bundlePath, "index.md"), + `--- +id: sales-okf +okf_version: "0.1" +--- + +# Sales Bundle +`, + "utf8", + ); + await fs.writeFile(path.join(bundlePath, "log.md"), "# Directory Update Log\n", "utf8"); + await fs.writeFile( + path.join(bundlePath, "tables", "customers.md"), + `--- +type: BigQuery Table +title: Customers +description: Customer table. +resource: https://console.cloud.google.com/bigquery?p=acme&d=sales&t=customers +tags: [sales, customers] +timestamp: 2026-05-28T00:00:00Z +producer_field: + owner: data +--- + +# Schema + +Customer rows. +`, + "utf8", + ); + await fs.writeFile( + path.join(bundlePath, "tables", "orders.md"), + `--- +type: BigQuery Table +title: Orders +description: One row per completed order. +tags: + - sales + - orders +--- + +# Schema + +Joined with [Customers](/tables/customers.md) and the [weekly metric](../metrics/weekly-active-users.md). +Titled link to [weekly metric](../metrics/weekly-active-users.md "metric docs"). + +Inline code keeps \`[customers](/tables/customers.md)\` unchanged. + +\`\`\`markdown +[customers](/tables/customers.md) +\`\`\` + +External citation stays as [BigQuery](https://cloud.google.com/bigquery). +`, + "utf8", + ); + await fs.writeFile( + path.join(bundlePath, "metrics", "weekly-active-users.md"), + `--- +type: Metric +title: Weekly Active Users +--- + +Computed from [orders](../tables/orders.md). +`, + "utf8", + ); + await fs.writeFile( + path.join(bundlePath, "tables", "draft.md"), + `--- +title: Draft +--- + +Missing type. +`, + "utf8", + ); + return bundlePath; +} + +describe("importMemoryWikiOkfBundle", () => { + it("imports OKF concept documents as searchable wiki concept pages", async () => { + const rootDir = await createTempDir("memory-wiki-okf-"); + const bundlePath = await writeOkfBundle(rootDir); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + + const result = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + + expect(result.okfVersion).toBe("0.1"); + expect(result.importedCount).toBe(3); + expect(result.skippedCount).toBe(1); + expect(result.warnings[0]).toMatchObject({ + code: "missing-type", + path: "tables/draft.md", + }); + expect(result.pagePaths).toHaveLength(3); + const repeat = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 5, 0), + }); + expect(repeat.importedCount).toBe(3); + expect(repeat.updatedCount).toBe(0); + + const ordersPath = result.pagePaths.find((pagePath) => pagePath.includes("orders")); + expect(ordersPath).toBeTruthy(); + const ordersRaw = await fs.readFile(path.join(config.vault.path, ordersPath!), "utf8"); + const orders = parseWikiMarkdown(ordersRaw); + expect(orders.frontmatter).toMatchObject({ + pageType: "concept", + title: "Orders", + sourceType: "okf", + provenanceMode: "okf-import", + okfConceptId: "tables/orders", + okfType: "BigQuery Table", + }); + expect(orders.frontmatter.sourceIds).toEqual([ + expect.stringMatching(/^source\.okf\.sales-okf$/), + ]); + expect(orders.body).toMatch(/\]\(okf-sales-okf-tables-customers-/); + expect(orders.body).toMatch(/\]\(okf-sales-okf-metrics-weekly-active-users-/); + expect(orders.body).toContain('"metric docs"'); + expect(orders.body).toContain("`[customers](/tables/customers.md)`"); + expect(orders.body).toContain("```markdown\n[customers](/tables/customers.md)\n```"); + expect(orders.body).toContain("https://cloud.google.com/bigquery"); + + const okf = orders.frontmatter.okf as Record; + expect(okf).toMatchObject({ + version: "0.1", + bundleName: "sales-okf", + conceptId: "tables/orders", + sourceRelativePath: "tables/orders.md", + }); + expect(orders.frontmatter.relationships).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + targetPath: expect.stringMatching(/^concepts\/okf-sales-okf-tables-customers-/), + kind: "okf-link", + }), + expect.objectContaining({ + targetPath: expect.stringMatching( + /^concepts\/okf-sales-okf-metrics-weekly-active-users-/, + ), + kind: "okf-link", + }), + ]), + ); + + const customersPath = result.pagePaths.find((pagePath) => pagePath.includes("customers")); + const customersRaw = await fs.readFile(path.join(config.vault.path, customersPath!), "utf8"); + const customers = parseWikiMarkdown(customersRaw); + const customersOkf = customers.frontmatter.okf as Record; + expect(customersOkf.frontmatter).toMatchObject({ + producer_field: { owner: "data" }, + }); + + const searchResults = await searchMemoryWiki({ + config, + query: "completed order", + searchCorpus: "wiki", + }); + expect(searchResults.map((searchResult) => searchResult.path)).toContain(ordersPath); + }); + + it("caps generated concept filenames for long OKF concept paths", async () => { + const rootDir = await createTempDir("memory-wiki-okf-long-"); + const bundlePath = path.join(rootDir, "long-okf"); + const deepSegments = Array.from({ length: 40 }, (_, index) => `segment-${index}`); + const deepDir = path.join(bundlePath, ...deepSegments); + await fs.mkdir(deepDir, { recursive: true }); + await fs.writeFile( + path.join(deepDir, "orders.md"), + `--- +type: BigQuery Table +title: Long Orders +--- + +Long concept body. +`, + "utf8", + ); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + + const result = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + + expect(result.importedCount).toBe(1); + const [pagePath] = result.pagePaths; + expect(pagePath).toBeDefined(); + if (!pagePath) { + throw new Error("Expected OKF import to produce a page path."); + } + const fileName = path.basename(pagePath); + expect(Buffer.byteLength(fileName)).toBeLessThanOrEqual(255); + await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain( + "Long concept body.", + ); + }); + + it("namespaces concept pages by bundle so repeated OKF paths do not overwrite", async () => { + const rootDir = await createTempDir("memory-wiki-okf-bundles-"); + const firstBundle = path.join(rootDir, "first-bundle"); + const secondBundle = path.join(rootDir, "second-bundle"); + for (const [bundlePath, title] of [ + [firstBundle, "First Customers"], + [secondBundle, "Second Customers"], + ] as const) { + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + await fs.writeFile( + path.join(bundlePath, "tables", "customers.md"), + `--- +type: BigQuery Table +title: ${title} +--- + +${title} body. +`, + "utf8", + ); + } + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + + const first = await importMemoryWikiOkfBundle({ + config, + bundlePath: firstBundle, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + const second = await importMemoryWikiOkfBundle({ + config, + bundlePath: secondBundle, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + + const firstPath = getOnlyPagePath(first.pagePaths); + const secondPath = getOnlyPagePath(second.pagePaths); + expect(firstPath).not.toBe(secondPath); + await expect(fs.readFile(path.join(config.vault.path, firstPath), "utf8")).resolves.toContain( + "First Customers body.", + ); + await expect(fs.readFile(path.join(config.vault.path, secondPath), "utf8")).resolves.toContain( + "Second Customers body.", + ); + }); + + it("removes stale concept pages when an OKF bundle drops a concept", async () => { + const rootDir = await createTempDir("memory-wiki-okf-remove-"); + const bundlePath = path.join(rootDir, "removing-okf"); + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + const customersPath = path.join(bundlePath, "tables", "customers.md"); + const ordersPath = path.join(bundlePath, "tables", "orders.md"); + await fs.writeFile( + customersPath, + `--- +type: BigQuery Table +title: Customers +--- + +Customer body. +`, + "utf8", + ); + await fs.writeFile( + ordersPath, + `--- +type: BigQuery Table +title: Orders +--- + +Order body. +`, + "utf8", + ); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + const first = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + const stalePagePath = first.pagePaths.find((pagePath) => pagePath.includes("orders")); + expect(stalePagePath).toBeDefined(); + if (!stalePagePath) { + throw new Error("Expected initial OKF import to include orders."); + } + + await fs.rm(ordersPath); + const second = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + + expect(second.importedCount).toBe(1); + expect(second.removedCount).toBe(1); + expect(second.removedPagePaths).toEqual([stalePagePath]); + await expect(fs.stat(path.join(config.vault.path, stalePagePath))).rejects.toThrow(); + const results = await searchMemoryWiki({ + config, + query: "Order body", + searchCorpus: "wiki", + }); + expect(results).toHaveLength(0); + }); + + it("does not prune existing pages when current OKF scan has invalid concepts", async () => { + const rootDir = await createTempDir("memory-wiki-okf-invalid-"); + const bundlePath = path.join(rootDir, "invalid-okf"); + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + const customersPath = path.join(bundlePath, "tables", "customers.md"); + await fs.writeFile( + customersPath, + `--- +type: BigQuery Table +title: Customers +--- + +Customer body. +`, + "utf8", + ); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + const first = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + const pagePath = getOnlyPagePath(first.pagePaths); + await fs.writeFile( + customersPath, + `--- +title: Customers +--- + +Temporarily invalid body. +`, + "utf8", + ); + + const second = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + + expect(second.importedCount).toBe(0); + expect(second.skippedCount).toBe(1); + expect(second.removedCount).toBe(0); + await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain( + "Customer body.", + ); + }); + + it("detects body-only changes on timestamp-shaped markdown lines", async () => { + const rootDir = await createTempDir("memory-wiki-okf-body-timestamp-"); + const bundlePath = path.join(rootDir, "body-timestamp-okf"); + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + const conceptPath = path.join(bundlePath, "tables", "events.md"); + await fs.writeFile( + conceptPath, + `--- +type: BigQuery Table +title: Events +--- + +updatedAt: 2026-06-12 +`, + "utf8", + ); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + const first = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + const pagePath = getOnlyPagePath(first.pagePaths); + await fs.writeFile( + conceptPath, + `--- +type: BigQuery Table +title: Events +--- + +updatedAt: 2026-06-13 +`, + "utf8", + ); + + const second = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 13, 10, 0, 0), + }); + + expect(second.updatedCount).toBe(1); + await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain( + "updatedAt: 2026-06-13", + ); + }); + + it("rewrites percent-encoded OKF markdown links and preserves suffixes", async () => { + const rootDir = await createTempDir("memory-wiki-okf-encoded-link-"); + const bundlePath = path.join(rootDir, "encoded-okf"); + await fs.mkdir(bundlePath, { recursive: true }); + await fs.writeFile( + path.join(bundlePath, "BigQuery Table.md"), + `--- +type: BigQuery Table +title: BigQuery Table +--- + +Table body. +`, + "utf8", + ); + await fs.writeFile( + path.join(bundlePath, "links.md"), + `--- +type: Concept +title: Links +--- + +See [table](BigQuery%20Table.md?view=compact#columns). +`, + "utf8", + ); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + + const result = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + + const linksPath = result.pagePaths.find((pagePath) => pagePath.includes("links")); + expect(linksPath).toBeDefined(); + if (!linksPath) { + throw new Error("Expected links page to be imported."); + } + await expect(fs.readFile(path.join(config.vault.path, linksPath), "utf8")).resolves.toMatch( + /\[table\]\(okf-encoded-okf-[0-9a-f]{8}-bigquery-table-[^)]+\.md\?view=compact#columns\)/, + ); + }); + + it("imports OKF concept frontmatter with CRLF line endings", async () => { + const rootDir = await createTempDir("memory-wiki-okf-crlf-"); + const bundlePath = path.join(rootDir, "crlf-okf"); + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + await fs.writeFile( + path.join(bundlePath, "tables", "events.md"), + [ + "---", + "type: BigQuery Table", + "title: Events", + "---", + "", + "Windows-flavored frontmatter.", + "", + ].join("\r\n"), + "utf8", + ); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + + const result = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + + expect(result.importedCount).toBe(1); + expect(result.skippedCount).toBe(0); + await expect( + fs.readFile(path.join(config.vault.path, getOnlyPagePath(result.pagePaths)), "utf8"), + ).resolves.toContain("Windows-flavored frontmatter."); + }); + + it("refuses to write imported OKF concept pages through symlinks", async () => { + const rootDir = await createTempDir("memory-wiki-okf-symlink-"); + const bundlePath = path.join(rootDir, "safe-okf"); + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + const conceptPath = path.join(bundlePath, "tables", "customers.md"); + await fs.writeFile( + conceptPath, + `--- +type: BigQuery Table +title: Customers +--- + +Original body. +`, + "utf8", + ); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + const first = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + const pagePath = getOnlyPagePath(first.pagePaths); + const pageAbsolutePath = path.join(config.vault.path, pagePath); + const externalTarget = path.join(rootDir, "outside.md"); + await fs.writeFile(externalTarget, "external target\n", "utf8"); + await fs.rm(pageAbsolutePath); + await fs.symlink(externalTarget, pageAbsolutePath); + await fs.writeFile( + conceptPath, + `--- +type: BigQuery Table +title: Customers +--- + +Updated body. +`, + "utf8", + ); + + await expect( + importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 11, 0, 0), + }), + ).rejects.toThrow("through symlink"); + await expect(fs.readFile(externalTarget, "utf8")).resolves.toBe("external target\n"); + }); + + it("refuses to import OKF concept files through hardlinks", async () => { + const rootDir = await createTempDir("memory-wiki-okf-hardlink-"); + const bundlePath = path.join(rootDir, "hardlink-okf"); + await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }); + const externalSource = path.join(rootDir, "outside.md"); + await fs.writeFile( + externalSource, + `--- +type: BigQuery Table +title: Private +--- + +private body +`, + "utf8", + ); + await fs.link(externalSource, path.join(bundlePath, "tables", "private.md")); + const { config } = await createVault({ + rootDir: path.join(rootDir, "vault"), + }); + + const result = await importMemoryWikiOkfBundle({ + config, + bundlePath, + nowMs: Date.UTC(2026, 5, 12, 10, 0, 0), + }); + + expect(result.importedCount).toBe(0); + expect(result.skippedCount).toBe(1); + expect(result.warnings[0]).toMatchObject({ + code: "unreadable-entry", + path: "tables/private.md", + }); + }); +}); diff --git a/extensions/memory-wiki/src/okf.ts b/extensions/memory-wiki/src/okf.ts new file mode 100644 index 00000000000..070d6c4a4ec --- /dev/null +++ b/extensions/memory-wiki/src/okf.ts @@ -0,0 +1,746 @@ +// Memory Wiki plugin module implements Open Knowledge Format import behavior. +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { FsSafeError, root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; +import { + normalizeOptionalString, + normalizeSingleOrTrimmedStringList, + uniqueStrings, +} from "openclaw/plugin-sdk/string-coerce-runtime"; +import { compileMemoryWikiVault } from "./compile.js"; +import type { ResolvedMemoryWikiConfig } from "./config.js"; +import { appendMemoryWikiLog } from "./log.js"; +import { + createWikiPageFilename, + parseWikiMarkdown, + renderWikiMarkdown, + slugifyWikiSegment, + WIKI_RELATED_END_MARKER, + WIKI_RELATED_START_MARKER, +} from "./markdown.js"; +import { resolveMemoryWikiTimestamp } from "./time.js"; +import { initializeMemoryWikiVault } from "./vault.js"; + +const OKF_RESERVED_FILENAMES = new Set(["index.md", "log.md"]); +const OKF_MARKDOWN_LINK_PATTERN = /(!?)\[([^\]]*)\]\(([^)]+)\)/g; +const OKF_FENCE_PATTERN = /^ {0,3}(`{3,}|~{3,})/; +const OKF_RELATED_SECTION_PATTERN = new RegExp( + `\\n+## Related\\n${WIKI_RELATED_START_MARKER}[\\s\\S]*?${WIKI_RELATED_END_MARKER}\\n?`, + "g", +); +const OKF_VOLATILE_TIMESTAMP_LINE_PATTERN = /^(?:importedAt|updatedAt): .*\n/gm; +const OKF_HASH_CHARS = 8; + +type FileStatLike = { + isFile?: unknown; + nlink?: unknown; +}; + +type OkfConceptDocument = { + conceptId: string; + relativePath: string; + absolutePath: string; + frontmatter: Record; + body: string; + type: string; + title: string; + description?: string; + resource?: string; + tags: string[]; + timestamp?: string; +}; + +type OkfImportedPage = { + conceptId: string; + sourcePath: string; + pageId: string; + pagePath: string; + title: string; + created: boolean; +}; + +export type ImportMemoryWikiOkfWarning = { + code: "invalid-concept" | "missing-type" | "unreadable-entry"; + path: string; + message: string; +}; + +export type ImportMemoryWikiOkfResult = { + bundlePath: string; + bundleName: string; + okfVersion?: string; + importedCount: number; + updatedCount: number; + removedCount: number; + skippedCount: number; + pagePaths: string[]; + removedPagePaths: string[]; + warnings: ImportMemoryWikiOkfWarning[]; + indexUpdatedFiles: string[]; +}; + +function toPosixPath(value: string): string { + return value.split(path.sep).join("/"); +} + +function trimMarkdownExtension(value: string): string { + return value.replace(/\.md$/i, ""); +} + +function isRegularFileStat(value: unknown): value is FileStatLike & { nlink: number } { + if (!value || typeof value !== "object") { + return false; + } + const stat = value as FileStatLike; + const isFile = + typeof stat.isFile === "function" + ? (stat.isFile as () => boolean).call(stat) + : stat.isFile === true; + return isFile && typeof stat.nlink === "number"; +} + +type OkfBundleMetadata = { + key: string; + version?: string; +}; + +function createOkfBundleKey(params: { + rootFrontmatter: Record; + bundleName: string; + bundlePath: string; +}): string { + const producerId = + normalizeOptionalString(params.rootFrontmatter.id) ?? + normalizeOptionalString(params.rootFrontmatter.okf_id); + if (producerId) { + return slugifyWikiSegment(producerId); + } + const label = + normalizeOptionalString(params.rootFrontmatter.name) ?? + normalizeOptionalString(params.rootFrontmatter.title) ?? + params.bundleName; + const hash = createHash("sha1").update(params.bundlePath).digest("hex").slice(0, OKF_HASH_CHARS); + return `${slugifyWikiSegment(label)}-${hash}`; +} + +function createOkfPageStem(bundleKey: string, conceptId: string): string { + const slug = slugifyWikiSegment(conceptId.replace(/\//g, "-")); + const hash = createHash("sha1").update(conceptId).digest("hex").slice(0, OKF_HASH_CHARS); + return `okf-${bundleKey}-${slug}-${hash}`; +} + +function createOkfPageIdentity( + bundleKey: string, + conceptId: string, +): { pageId: string; pagePath: string } { + const fileName = createWikiPageFilename(createOkfPageStem(bundleKey, conceptId)); + const stem = trimMarkdownExtension(fileName); + return { + pageId: `concept.${stem}`, + pagePath: `concepts/${fileName}`, + }; +} + +async function collectOkfMarkdownFiles( + rootDir: string, + warnings: ImportMemoryWikiOkfWarning[], +): Promise { + async function walk(relativeDir: string): Promise { + const absoluteDir = path.join(rootDir, relativeDir); + const entries = await fs.readdir(absoluteDir, { withFileTypes: true }).catch((err: unknown) => { + warnings.push({ + code: "unreadable-entry", + path: toPosixPath(relativeDir) || ".", + message: err instanceof Error ? err.message : "Unable to read OKF directory.", + }); + return []; + }); + const files: string[] = []; + for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) { + if (entry.name === ".git" || entry.name === "node_modules") { + continue; + } + const relativePath = path.join(relativeDir, entry.name); + if (entry.isDirectory()) { + files.push(...(await walk(relativePath))); + continue; + } + if (entry.isFile() && entry.name.endsWith(".md")) { + files.push(relativePath); + } + } + return files; + } + + return (await walk("")).map(toPosixPath).toSorted((left, right) => left.localeCompare(right)); +} + +function parseOkfMarkdown( + content: string, + relativePath: string, +): { + frontmatter: Record; + body: string; + warning?: ImportMemoryWikiOkfWarning; +} { + const normalizedContent = content.replace(/\r\n/g, "\n"); + try { + return parseWikiMarkdown(normalizedContent); + } catch (err) { + return { + frontmatter: {}, + body: normalizedContent, + warning: { + code: "invalid-concept", + path: relativePath, + message: err instanceof Error ? err.message : "Unable to parse OKF frontmatter.", + }, + }; + } +} + +async function readOkfTextFile(params: { + bundlePath: string; + relativePath: string; + warnings: ImportMemoryWikiOkfWarning[]; +}): Promise { + const root = await fsRoot(params.bundlePath); + const stat = await root.stat(params.relativePath).catch((err: unknown) => { + params.warnings.push({ + code: "unreadable-entry", + path: params.relativePath, + message: err instanceof Error ? err.message : "Unable to read OKF concept.", + }); + return null; + }); + if (!stat) { + return null; + } + if (!isRegularFileStat(stat)) { + params.warnings.push({ + code: "unreadable-entry", + path: params.relativePath, + message: "Refusing to import OKF concept through non-regular or hardlinked file.", + }); + return null; + } + return await root.readText(params.relativePath).catch((err: unknown) => { + params.warnings.push({ + code: "unreadable-entry", + path: params.relativePath, + message: err instanceof Error ? err.message : "Unable to read OKF concept.", + }); + return null; + }); +} + +function deriveOkfTitle(relativePath: string, frontmatter: Record): string { + return ( + normalizeOptionalString(frontmatter.title) ?? + path.posix.basename(relativePath, ".md").replace(/[-_]+/g, " ").trim() ?? + trimMarkdownExtension(relativePath) + ); +} + +function normalizeOkfConcept(params: { + bundlePath: string; + relativePath: string; + content: string; +}): { concept?: OkfConceptDocument; warning?: ImportMemoryWikiOkfWarning } { + const parsed = parseOkfMarkdown(params.content, params.relativePath); + if (parsed.warning) { + return { warning: parsed.warning }; + } + + const type = normalizeOptionalString(parsed.frontmatter.type); + if (!type) { + return { + warning: { + code: "missing-type", + path: params.relativePath, + message: "OKF concept is missing required non-empty type frontmatter.", + }, + }; + } + + const conceptId = trimMarkdownExtension(params.relativePath); + const timestamp = normalizeOptionalString(parsed.frontmatter.timestamp); + return { + concept: { + conceptId, + relativePath: params.relativePath, + absolutePath: path.join(params.bundlePath, params.relativePath), + frontmatter: parsed.frontmatter, + body: parsed.body, + type, + title: deriveOkfTitle(params.relativePath, parsed.frontmatter), + ...(normalizeOptionalString(parsed.frontmatter.description) + ? { description: normalizeOptionalString(parsed.frontmatter.description) } + : {}), + ...(normalizeOptionalString(parsed.frontmatter.resource) + ? { resource: normalizeOptionalString(parsed.frontmatter.resource) } + : {}), + tags: normalizeSingleOrTrimmedStringList(parsed.frontmatter.tags), + ...(timestamp ? { timestamp } : {}), + }, + }; +} + +function splitMarkdownLinkDestination(target: string): { + destination: string; + titleSuffix: string; +} { + const trimmed = target.trim(); + if (trimmed.startsWith("<")) { + const end = trimmed.indexOf(">"); + if (end > 0) { + return { + destination: trimmed.slice(1, end), + titleSuffix: trimmed.slice(end + 1), + }; + } + } + const match = trimmed.match(/^(\S+)(\s+[\s\S]+)?$/); + return { + destination: match?.[1] ?? trimmed, + titleSuffix: match?.[2] ?? "", + }; +} + +function resolveOkfMarkdownTarget(sourceRelativePath: string, target: string): string | null { + const { destination } = splitMarkdownLinkDestination(target); + const trimmed = destination.trim(); + if (!trimmed || trimmed.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(trimmed)) { + return null; + } + + const rawTargetWithoutSuffix = trimmed.split("#")[0]?.split("?")[0]?.replace(/\\/g, "/").trim(); + const targetWithoutSuffix = safeDecodeOkfLinkPath(rawTargetWithoutSuffix); + if (!targetWithoutSuffix || !targetWithoutSuffix.endsWith(".md")) { + return null; + } + + const normalized = targetWithoutSuffix.startsWith("/") + ? path.posix.normalize(targetWithoutSuffix.slice(1)) + : path.posix.normalize( + path.posix.join(path.posix.dirname(sourceRelativePath), targetWithoutSuffix), + ); + const conceptId = trimMarkdownExtension(normalized); + return conceptId.startsWith("../") ? null : conceptId; +} + +function safeDecodeOkfLinkPath(value: string | undefined): string { + if (!value) { + return ""; + } + try { + return decodeURIComponent(value); + } catch { + return value; + } +} + +function getMarkdownDestinationSuffix(destination: string): string { + const queryIndex = destination.indexOf("?"); + const fragmentIndex = destination.indexOf("#"); + const suffixIndex = queryIndex === -1 + ? fragmentIndex + : fragmentIndex === -1 + ? queryIndex + : Math.min(queryIndex, fragmentIndex); + return suffixIndex === -1 ? "" : destination.slice(suffixIndex); +} + +function rewriteOkfMarkdownLinks(params: { + body: string; + sourcePagePath: string; + sourceRelativePath: string; + pageByConceptId: Map; +}): { body: string; linkedConceptIds: string[] } { + const linkedConceptIds: string[] = []; + const rewriteLinks = (markdown: string) => + markdown.replace( + OKF_MARKDOWN_LINK_PATTERN, + (match, imagePrefix: string, label: string, rawTarget: string) => { + const conceptId = resolveOkfMarkdownTarget(params.sourceRelativePath, rawTarget); + if (!conceptId) { + return match; + } + const target = params.pageByConceptId.get(conceptId); + if (!target) { + return match; + } + linkedConceptIds.push(conceptId); + const { destination, titleSuffix } = splitMarkdownLinkDestination(rawTarget); + const relativeTarget = path.posix.relative( + path.posix.dirname(params.sourcePagePath), + target.pagePath, + ); + const suffix = getMarkdownDestinationSuffix(destination); + return `${imagePrefix}[${label}](${relativeTarget}${suffix}${titleSuffix})`; + }, + ); + const body = rewriteMarkdownOutsideCode(params.body, rewriteLinks); + return { body, linkedConceptIds: uniqueStrings(linkedConceptIds) }; +} + +function rewriteMarkdownLineOutsideInlineCode( + line: string, + rewriteLinks: (markdown: string) => string, +): string { + let result = ""; + let cursor = 0; + while (cursor < line.length) { + const codeStart = line.indexOf("`", cursor); + if (codeStart === -1) { + result += rewriteLinks(line.slice(cursor)); + break; + } + result += rewriteLinks(line.slice(cursor, codeStart)); + const delimiter = line.slice(codeStart).match(/^`+/)?.[0] ?? "`"; + const codeEnd = line.indexOf(delimiter, codeStart + delimiter.length); + if (codeEnd === -1) { + result += line.slice(codeStart); + break; + } + result += line.slice(codeStart, codeEnd + delimiter.length); + cursor = codeEnd + delimiter.length; + } + return result; +} + +function rewriteMarkdownOutsideCode( + markdown: string, + rewriteLinks: (markdown: string) => string, +): string { + const lines = markdown.split(/(\n)/); + let inFence = false; + let fenceDelimiter = ""; + return lines + .map((line) => { + if (line === "\n") { + return line; + } + const fenceMatch = line.match(OKF_FENCE_PATTERN); + if (fenceMatch) { + const delimiter = fenceMatch[1] ?? ""; + const closesFence = + inFence && + delimiter.startsWith(fenceDelimiter[0] ?? "") && + delimiter.length >= fenceDelimiter.length; + const opensFence = !inFence; + if (opensFence) { + inFence = true; + fenceDelimiter = delimiter; + } else if (closesFence) { + inFence = false; + fenceDelimiter = ""; + } + return line; + } + return inFence ? line : rewriteMarkdownLineOutsideInlineCode(line, rewriteLinks); + }) + .join(""); +} + +function normalizeOkfRenderedPageForComparison(content: string): string { + const withoutRelated = content.replace(OKF_RELATED_SECTION_PATTERN, "\n"); + const frontmatterMatch = withoutRelated.match(/^---\n([\s\S]*?)\n---\n?/); + if (!frontmatterMatch) { + return withoutRelated.trimEnd(); + } + const normalizedFrontmatter = + frontmatterMatch[1]?.replace(OKF_VOLATILE_TIMESTAMP_LINE_PATTERN, "") ?? ""; + const frontmatterBody = normalizedFrontmatter.endsWith("\n") + ? normalizedFrontmatter + : `${normalizedFrontmatter}\n`; + return `---\n${frontmatterBody}---\n${withoutRelated.slice(frontmatterMatch[0].length)}`.trimEnd(); +} + +async function writeOkfConceptPage(params: { + vaultRoot: string; + pagePath: string; + content: string; +}): Promise<{ changed: boolean; created: boolean }> { + const vault = await fsRoot(params.vaultRoot); + const pageStat = await vault.stat(params.pagePath).catch((error: unknown) => { + if ( + error instanceof FsSafeError && + (error.code === "not-found" || error.code === "path-alias") + ) { + return null; + } + throw error; + }); + const existing = pageStat ? await vault.readText(params.pagePath).catch(() => "") : ""; + if ( + existing === params.content || + normalizeOkfRenderedPageForComparison(existing) === + normalizeOkfRenderedPageForComparison(params.content) + ) { + return { changed: false, created: !pageStat }; + } + try { + if (isRegularFileStat(pageStat) && pageStat.nlink > 1) { + await vault.remove(params.pagePath); + } + await vault.write(params.pagePath, params.content); + } catch (error) { + if (error instanceof FsSafeError) { + if (error.code !== "symlink" && error.code !== "path-alias") { + throw new Error( + `Refusing to write OKF concept page (${error.code}): ${params.pagePath}: ${error.message}`, + { cause: error }, + ); + } + throw new Error(`Refusing to write OKF concept page through symlink: ${params.pagePath}`, { + cause: error, + }); + } + throw error; + } + return { changed: true, created: !pageStat }; +} + +async function removeStaleOkfConceptPages(params: { + vaultRoot: string; + bundleKey: string; + currentPagePaths: Set; +}): Promise { + const vault = await fsRoot(params.vaultRoot); + const conceptsDir = path.join(params.vaultRoot, "concepts"); + const entries = await fs.readdir(conceptsDir, { withFileTypes: true }).catch(() => []); + const removedPagePaths: string[] = []; + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") { + continue; + } + const pagePath = `concepts/${entry.name}`; + if (params.currentPagePaths.has(pagePath)) { + continue; + } + const raw = await vault.readText(pagePath).catch(() => ""); + const parsed = parseWikiMarkdown(raw); + const okf = parsed.frontmatter.okf; + if ( + okf && + typeof okf === "object" && + !Array.isArray(okf) && + (okf as Record).bundleKey === params.bundleKey + ) { + await vault.remove(pagePath); + removedPagePaths.push(pagePath); + } + } + return removedPagePaths; +} + +function readRootOkfMetadata(params: { + rootIndex: string | undefined; + bundleName: string; + bundlePath: string; +}): OkfBundleMetadata { + if (!params.rootIndex) { + return { + key: createOkfBundleKey({ + rootFrontmatter: {}, + bundleName: params.bundleName, + bundlePath: params.bundlePath, + }), + }; + } + const parsed = parseOkfMarkdown(params.rootIndex, "index.md"); + return { + key: createOkfBundleKey({ + rootFrontmatter: parsed.frontmatter, + bundleName: params.bundleName, + bundlePath: params.bundlePath, + }), + ...(normalizeOptionalString(parsed.frontmatter.okf_version) + ? { version: normalizeOptionalString(parsed.frontmatter.okf_version) } + : {}), + }; +} + +function formatOkfImportSummary(result: ImportMemoryWikiOkfResult): string { + return `Imported ${result.importedCount} OKF concept${result.importedCount === 1 ? "" : "s"} from ${result.bundlePath} into memory wiki. Updated ${result.updatedCount}; removed ${result.removedCount}; skipped ${result.skippedCount}; refreshed ${result.indexUpdatedFiles.length} index file${result.indexUpdatedFiles.length === 1 ? "" : "s"}.`; +} + +export { formatOkfImportSummary }; + +export async function importMemoryWikiOkfBundle(params: { + config: ResolvedMemoryWikiConfig; + bundlePath: string; + nowMs?: number; +}): Promise { + await initializeMemoryWikiVault(params.config, { nowMs: params.nowMs }); + const bundlePath = path.resolve(params.bundlePath); + const stat = await fs.stat(bundlePath); + if (!stat.isDirectory()) { + throw new Error("wiki okf import expects an unpacked OKF bundle directory."); + } + + const warnings: ImportMemoryWikiOkfWarning[] = []; + const markdownFiles = await collectOkfMarkdownFiles(bundlePath, warnings); + const concepts: OkfConceptDocument[] = []; + let rootIndexContent: string | undefined; + + for (const relativePath of markdownFiles) { + if (relativePath === "index.md") { + rootIndexContent = + (await readOkfTextFile({ bundlePath, relativePath, warnings })) ?? undefined; + } + if (OKF_RESERVED_FILENAMES.has(path.posix.basename(relativePath))) { + continue; + } + const content = await readOkfTextFile({ bundlePath, relativePath, warnings }); + if (content === null) { + continue; + } + const normalized = normalizeOkfConcept({ bundlePath, relativePath, content }); + if (normalized.warning) { + warnings.push(normalized.warning); + continue; + } + if (normalized.concept) { + concepts.push(normalized.concept); + } + } + + const timestamp = resolveMemoryWikiTimestamp(params.nowMs); + const bundleName = path.basename(bundlePath); + const bundleMetadata = readRootOkfMetadata({ + rootIndex: rootIndexContent, + bundleName, + bundlePath, + }); + const bundleKey = bundleMetadata.key; + const pageByConceptId = new Map(); + for (const concept of concepts) { + pageByConceptId.set(concept.conceptId, { + ...createOkfPageIdentity(bundleKey, concept.conceptId), + title: concept.title, + }); + } + + const importedPages: OkfImportedPage[] = []; + let updatedCount = 0; + + await fs.mkdir(path.join(params.config.vault.path, "concepts"), { recursive: true }); + for (const concept of concepts.toSorted((left, right) => + left.conceptId.localeCompare(right.conceptId), + )) { + const page = pageByConceptId.get(concept.conceptId); + if (!page) { + continue; + } + const rewritten = rewriteOkfMarkdownLinks({ + body: concept.body, + sourcePagePath: page.pagePath, + sourceRelativePath: concept.relativePath, + pageByConceptId, + }); + const relationships = rewritten.linkedConceptIds.flatMap((conceptId) => { + const target = pageByConceptId.get(conceptId); + return target + ? [ + { + targetId: target.pageId, + targetPath: target.pagePath, + targetTitle: target.title, + kind: "okf-link", + evidenceKind: "okf-markdown-link", + }, + ] + : []; + }); + + const frontmatter = { + pageType: "concept", + id: page.pageId, + title: concept.title, + sourceType: "okf", + provenanceMode: "okf-import", + sourcePath: concept.absolutePath, + okfConceptId: concept.conceptId, + okfType: concept.type, + sourceIds: [`source.okf.${bundleKey}`], + importedAt: timestamp, + updatedAt: concept.timestamp ?? timestamp, + status: "active", + ...(concept.description ? { description: concept.description } : {}), + ...(concept.resource ? { resource: concept.resource } : {}), + ...(concept.tags.length > 0 ? { tags: concept.tags } : {}), + ...(concept.timestamp ? { okfTimestamp: concept.timestamp } : {}), + ...(relationships.length > 0 ? { relationships } : {}), + okf: { + ...(bundleMetadata.version ? { version: bundleMetadata.version } : {}), + bundleName, + bundleKey, + conceptId: concept.conceptId, + sourceRelativePath: concept.relativePath, + frontmatter: concept.frontmatter, + }, + }; + + const writeResult = await writeOkfConceptPage({ + vaultRoot: params.config.vault.path, + pagePath: page.pagePath, + content: renderWikiMarkdown({ + frontmatter, + body: rewritten.body, + }), + }); + if (!writeResult.created && writeResult.changed) { + updatedCount++; + } + importedPages.push({ + conceptId: concept.conceptId, + sourcePath: concept.absolutePath, + pageId: page.pageId, + pagePath: page.pagePath, + title: concept.title, + created: writeResult.created, + }); + } + const currentPagePaths = new Set(importedPages.map((page) => page.pagePath)); + const removedPagePaths = + warnings.length === 0 + ? await removeStaleOkfConceptPages({ + vaultRoot: params.config.vault.path, + bundleKey, + currentPagePaths, + }) + : []; + + await appendMemoryWikiLog(params.config.vault.path, { + type: "okf-import", + timestamp, + details: { + bundlePath, + bundleName, + importedCount: importedPages.length, + updatedCount, + removedCount: removedPagePaths.length, + skippedCount: warnings.length, + pagePaths: importedPages.map((page) => page.pagePath), + removedPagePaths, + }, + }); + + const compile = await compileMemoryWikiVault(params.config); + return { + bundlePath, + bundleName, + ...(bundleMetadata.version ? { okfVersion: bundleMetadata.version } : {}), + importedCount: importedPages.length, + updatedCount, + removedCount: removedPagePaths.length, + skippedCount: warnings.length, + pagePaths: importedPages.map((page) => page.pagePath), + removedPagePaths, + warnings, + indexUpdatedFiles: compile.updatedFiles, + }; +}