mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-22 16:08:13 +00:00
feat(memory-wiki): import OKF bundles
This commit is contained in:
@@ -35,6 +35,7 @@ openclaw wiki status
|
||||
openclaw wiki doctor
|
||||
openclaw wiki init
|
||||
openclaw wiki ingest ./notes/alpha.md
|
||||
openclaw wiki okf import ./knowledge-catalog/okf/bundles/ga4
|
||||
openclaw wiki compile
|
||||
openclaw wiki lint
|
||||
openclaw wiki search "alpha"
|
||||
@@ -104,6 +105,31 @@ Notes:
|
||||
- imported source pages keep provenance in frontmatter
|
||||
- auto-compile can run after ingest when enabled
|
||||
|
||||
### `wiki okf import <path>`
|
||||
|
||||
Import an unpacked Open Knowledge Format bundle into wiki concept pages.
|
||||
|
||||
The importer reads every non-reserved `.md` concept document in the OKF
|
||||
directory tree, requires a non-empty `type` field, and treats unknown OKF
|
||||
`type` values as generic concepts. Reserved OKF `index.md` and `log.md` files
|
||||
are not imported as concepts.
|
||||
|
||||
Imported pages are flattened under `concepts/` so existing wiki compile,
|
||||
search, get, digest, and dashboard flows see them immediately. The original OKF
|
||||
concept ID, `type`, `resource`, `tags`, timestamp, source path, and full
|
||||
frontmatter are preserved in the page frontmatter. Internal OKF markdown links
|
||||
are rewritten to the generated wiki pages; broken or external links are left
|
||||
unchanged.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
openclaw wiki okf import ./bundles/ga4
|
||||
openclaw wiki okf import ./bundles/ga4 --json
|
||||
openclaw wiki search "BigQuery Table" --mode source-evidence --json
|
||||
openclaw wiki get <path-from-json-result>
|
||||
```
|
||||
|
||||
### `wiki compile`
|
||||
|
||||
Rebuild indexes, related blocks, dashboards, and compiled digests.
|
||||
@@ -233,6 +259,8 @@ These require the official `obsidian` CLI on `PATH` when
|
||||
- Use `wiki lint` before trusting contradictory or low-confidence content.
|
||||
- Use `wiki compile` after bulk imports or source changes when you want fresh
|
||||
dashboards and compiled digests immediately.
|
||||
- Use `wiki okf import` when a data catalog, documentation export, or agent
|
||||
enrichment pipeline already emits OKF markdown bundles.
|
||||
- Use `wiki bridge import` when bridge mode depends on newly exported memory
|
||||
artifacts.
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ less like a pile of Markdown files.
|
||||
- Page-level provenance, confidence, contradictions, and open questions
|
||||
- Compiled digests for agent/runtime consumers
|
||||
- Wiki-native search/get/apply/lint tools
|
||||
- Open Knowledge Format imports into compiled wiki concepts
|
||||
- Optional bridge mode that imports public artifacts from the active memory plugin
|
||||
- Optional Obsidian-friendly render mode and CLI integration
|
||||
|
||||
@@ -135,6 +136,34 @@ The main page groups are:
|
||||
- `syntheses/` for compiled summaries and maintained rollups
|
||||
- `reports/` for generated dashboards
|
||||
|
||||
## Open Knowledge Format imports
|
||||
|
||||
`memory-wiki` can import unpacked Open Knowledge Format bundles with:
|
||||
|
||||
```bash
|
||||
openclaw wiki okf import ./bundles/ga4
|
||||
```
|
||||
|
||||
This is the cleanest fit when a data catalog, documentation crawler, or
|
||||
enrichment agent already produces OKF: keep OKF as the portable exchange
|
||||
artifact, then let `memory-wiki` turn it into OpenClaw-native concept pages and
|
||||
compiled digests.
|
||||
|
||||
The importer follows the OKF v0.1 shape:
|
||||
|
||||
- non-reserved `.md` files are concept documents
|
||||
- each imported concept needs a non-empty `type` frontmatter field
|
||||
- unknown OKF `type` values are accepted
|
||||
- reserved `index.md` and `log.md` files are not imported as concepts
|
||||
- broken or external markdown links are preserved
|
||||
|
||||
Imported concept pages are flattened under `concepts/` so the existing compile,
|
||||
search, get, dashboard, and prompt-digest paths see them without adding a second
|
||||
wiki tree. Each page keeps the original OKF concept ID, source path, `type`,
|
||||
`resource`, `tags`, timestamp, and full producer frontmatter. Internal OKF links
|
||||
are rewritten to the generated wiki concept pages and also emitted as structured
|
||||
`relationships` entries with `kind: okf-link`.
|
||||
|
||||
## Structured claims and evidence
|
||||
|
||||
Pages can carry structured `claims` frontmatter, not just freeform text.
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
runWikiChatGptImport,
|
||||
runWikiChatGptRollback,
|
||||
runWikiDoctor,
|
||||
runWikiOkfImport,
|
||||
runWikiStatus,
|
||||
} from "./cli.js";
|
||||
import type { MemoryWikiPluginConfig } from "./config.js";
|
||||
@@ -27,6 +28,7 @@ vi.mock("openclaw/plugin-sdk/gateway-runtime", () => ({
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
let suiteRoot = "";
|
||||
let caseIndex = 0;
|
||||
let stdoutWriteMock: ReturnType<typeof vi.fn>;
|
||||
|
||||
describe("memory-wiki cli", () => {
|
||||
beforeAll(async () => {
|
||||
@@ -41,8 +43,9 @@ describe("memory-wiki cli", () => {
|
||||
|
||||
beforeEach(() => {
|
||||
callGatewayFromCliMock.mockReset();
|
||||
stdoutWriteMock = vi.fn(() => true);
|
||||
vi.spyOn(process.stdout, "write").mockImplementation(
|
||||
(() => true) as typeof process.stdout.write,
|
||||
stdoutWriteMock as unknown as typeof process.stdout.write,
|
||||
);
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
@@ -174,6 +177,65 @@ describe("memory-wiki cli", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("registers OKF import and searches imported concepts", async () => {
|
||||
const { rootDir, config } = await createCliVault();
|
||||
const bundlePath = path.join(rootDir, "okf-bundle");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(path.join(bundlePath, "index.md"), "# Sales OKF\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer rows.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
description: One row per completed order.
|
||||
---
|
||||
|
||||
Orders join to [customers](/tables/customers.md).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const program = new Command();
|
||||
program.name("test");
|
||||
registerWikiCli(program, config);
|
||||
|
||||
await program.parseAsync(["wiki", "okf", "import", bundlePath, "--json"], { from: "user" });
|
||||
|
||||
const importOutput = String(stdoutWriteMock.mock.calls.at(-1)?.[0] ?? "");
|
||||
const importResult = JSON.parse(importOutput) as Awaited<ReturnType<typeof runWikiOkfImport>>;
|
||||
expect(importResult.importedCount).toBe(2);
|
||||
expect(importResult.pagePaths).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringMatching(/^concepts\/okf-okf-bundle-[0-9a-f]{8}-tables-orders-/),
|
||||
]),
|
||||
);
|
||||
|
||||
stdoutWriteMock.mockClear();
|
||||
await program.parseAsync(["wiki", "search", "completed order", "--json"], { from: "user" });
|
||||
|
||||
const searchOutput = String(stdoutWriteMock.mock.calls.at(-1)?.[0] ?? "");
|
||||
const searchResults = JSON.parse(searchOutput) as Array<{ path: string; title: string }>;
|
||||
expect(searchResults).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
title: "Orders",
|
||||
path: expect.stringMatching(/^concepts\/okf-okf-bundle-[0-9a-f]{8}-tables-orders-/),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects apply confidence values outside the documented range", async () => {
|
||||
const { config } = await createCliVault();
|
||||
const program = new Command();
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
runObsidianOpen,
|
||||
runObsidianSearch,
|
||||
} from "./obsidian.js";
|
||||
import { formatOkfImportSummary, importMemoryWikiOkfBundle } from "./okf.js";
|
||||
import {
|
||||
getMemoryWikiPage,
|
||||
searchMemoryWiki,
|
||||
@@ -88,6 +89,10 @@ type WikiIngestCommandOptions = {
|
||||
title?: string;
|
||||
};
|
||||
|
||||
type WikiOkfImportCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiSearchCommandOptions = {
|
||||
json?: boolean;
|
||||
maxResults?: number;
|
||||
@@ -590,6 +595,24 @@ export async function runWikiIngest(params: {
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiOkfImport(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
bundlePath: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () =>
|
||||
importMemoryWikiOkfBundle({
|
||||
config: params.config,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
render: formatOkfImportSummary,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiSearch(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
@@ -965,6 +988,16 @@ export function registerWikiCli(
|
||||
await runWikiIngest({ config, inputPath, title: opts.title, json: opts.json });
|
||||
});
|
||||
|
||||
const okf = wiki.command("okf").description("Import Open Knowledge Format bundles");
|
||||
okf
|
||||
.command("import")
|
||||
.description("Import an unpacked OKF bundle into wiki concept pages")
|
||||
.argument("<path>", "OKF bundle directory")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (bundlePath: string, opts: WikiOkfImportCommandOptions) => {
|
||||
await runWikiOkfImport({ config, bundlePath, json: opts.json });
|
||||
});
|
||||
|
||||
addWikiSearchConfigOptions(
|
||||
wiki
|
||||
.command("search")
|
||||
|
||||
@@ -4,7 +4,7 @@ import path from "node:path";
|
||||
import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime";
|
||||
|
||||
type MemoryWikiLogEntry = {
|
||||
type: "init" | "ingest" | "compile" | "lint";
|
||||
type: "init" | "ingest" | "okf-import" | "compile" | "lint";
|
||||
timestamp: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
609
extensions/memory-wiki/src/okf.test.ts
Normal file
609
extensions/memory-wiki/src/okf.test.ts
Normal file
@@ -0,0 +1,609 @@
|
||||
// Memory Wiki tests cover Open Knowledge Format import behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseWikiMarkdown } from "./markdown.js";
|
||||
import { importMemoryWikiOkfBundle } from "./okf.js";
|
||||
import { searchMemoryWiki } from "./query.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempDir, createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
function getOnlyPagePath(paths: string[]): string {
|
||||
expect(paths).toHaveLength(1);
|
||||
const [pagePath] = paths;
|
||||
if (!pagePath) {
|
||||
throw new Error("Expected OKF import to produce one page path.");
|
||||
}
|
||||
return pagePath;
|
||||
}
|
||||
|
||||
async function writeOkfBundle(rootDir: string) {
|
||||
const bundlePath = path.join(rootDir, "sales-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.mkdir(path.join(bundlePath, "metrics"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "index.md"),
|
||||
`---
|
||||
id: sales-okf
|
||||
okf_version: "0.1"
|
||||
---
|
||||
|
||||
# Sales Bundle
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(path.join(bundlePath, "log.md"), "# Directory Update Log\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
description: Customer table.
|
||||
resource: https://console.cloud.google.com/bigquery?p=acme&d=sales&t=customers
|
||||
tags: [sales, customers]
|
||||
timestamp: 2026-05-28T00:00:00Z
|
||||
producer_field:
|
||||
owner: data
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Customer rows.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
description: One row per completed order.
|
||||
tags:
|
||||
- sales
|
||||
- orders
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Joined with [Customers](/tables/customers.md) and the [weekly metric](../metrics/weekly-active-users.md).
|
||||
Titled link to [weekly metric](../metrics/weekly-active-users.md "metric docs").
|
||||
|
||||
Inline code keeps \`[customers](/tables/customers.md)\` unchanged.
|
||||
|
||||
\`\`\`markdown
|
||||
[customers](/tables/customers.md)
|
||||
\`\`\`
|
||||
|
||||
External citation stays as [BigQuery](https://cloud.google.com/bigquery).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "metrics", "weekly-active-users.md"),
|
||||
`---
|
||||
type: Metric
|
||||
title: Weekly Active Users
|
||||
---
|
||||
|
||||
Computed from [orders](../tables/orders.md).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "draft.md"),
|
||||
`---
|
||||
title: Draft
|
||||
---
|
||||
|
||||
Missing type.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
return bundlePath;
|
||||
}
|
||||
|
||||
describe("importMemoryWikiOkfBundle", () => {
|
||||
it("imports OKF concept documents as searchable wiki concept pages", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-");
|
||||
const bundlePath = await writeOkfBundle(rootDir);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.okfVersion).toBe("0.1");
|
||||
expect(result.importedCount).toBe(3);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
expect(result.warnings[0]).toMatchObject({
|
||||
code: "missing-type",
|
||||
path: "tables/draft.md",
|
||||
});
|
||||
expect(result.pagePaths).toHaveLength(3);
|
||||
const repeat = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 5, 0),
|
||||
});
|
||||
expect(repeat.importedCount).toBe(3);
|
||||
expect(repeat.updatedCount).toBe(0);
|
||||
|
||||
const ordersPath = result.pagePaths.find((pagePath) => pagePath.includes("orders"));
|
||||
expect(ordersPath).toBeTruthy();
|
||||
const ordersRaw = await fs.readFile(path.join(config.vault.path, ordersPath!), "utf8");
|
||||
const orders = parseWikiMarkdown(ordersRaw);
|
||||
expect(orders.frontmatter).toMatchObject({
|
||||
pageType: "concept",
|
||||
title: "Orders",
|
||||
sourceType: "okf",
|
||||
provenanceMode: "okf-import",
|
||||
okfConceptId: "tables/orders",
|
||||
okfType: "BigQuery Table",
|
||||
});
|
||||
expect(orders.frontmatter.sourceIds).toEqual([
|
||||
expect.stringMatching(/^source\.okf\.sales-okf$/),
|
||||
]);
|
||||
expect(orders.body).toMatch(/\]\(okf-sales-okf-tables-customers-/);
|
||||
expect(orders.body).toMatch(/\]\(okf-sales-okf-metrics-weekly-active-users-/);
|
||||
expect(orders.body).toContain('"metric docs"');
|
||||
expect(orders.body).toContain("`[customers](/tables/customers.md)`");
|
||||
expect(orders.body).toContain("```markdown\n[customers](/tables/customers.md)\n```");
|
||||
expect(orders.body).toContain("https://cloud.google.com/bigquery");
|
||||
|
||||
const okf = orders.frontmatter.okf as Record<string, unknown>;
|
||||
expect(okf).toMatchObject({
|
||||
version: "0.1",
|
||||
bundleName: "sales-okf",
|
||||
conceptId: "tables/orders",
|
||||
sourceRelativePath: "tables/orders.md",
|
||||
});
|
||||
expect(orders.frontmatter.relationships).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
targetPath: expect.stringMatching(/^concepts\/okf-sales-okf-tables-customers-/),
|
||||
kind: "okf-link",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
targetPath: expect.stringMatching(
|
||||
/^concepts\/okf-sales-okf-metrics-weekly-active-users-/,
|
||||
),
|
||||
kind: "okf-link",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const customersPath = result.pagePaths.find((pagePath) => pagePath.includes("customers"));
|
||||
const customersRaw = await fs.readFile(path.join(config.vault.path, customersPath!), "utf8");
|
||||
const customers = parseWikiMarkdown(customersRaw);
|
||||
const customersOkf = customers.frontmatter.okf as Record<string, unknown>;
|
||||
expect(customersOkf.frontmatter).toMatchObject({
|
||||
producer_field: { owner: "data" },
|
||||
});
|
||||
|
||||
const searchResults = await searchMemoryWiki({
|
||||
config,
|
||||
query: "completed order",
|
||||
searchCorpus: "wiki",
|
||||
});
|
||||
expect(searchResults.map((searchResult) => searchResult.path)).toContain(ordersPath);
|
||||
});
|
||||
|
||||
it("caps generated concept filenames for long OKF concept paths", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-long-");
|
||||
const bundlePath = path.join(rootDir, "long-okf");
|
||||
const deepSegments = Array.from({ length: 40 }, (_, index) => `segment-${index}`);
|
||||
const deepDir = path.join(bundlePath, ...deepSegments);
|
||||
await fs.mkdir(deepDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(deepDir, "orders.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Long Orders
|
||||
---
|
||||
|
||||
Long concept body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(1);
|
||||
const [pagePath] = result.pagePaths;
|
||||
expect(pagePath).toBeDefined();
|
||||
if (!pagePath) {
|
||||
throw new Error("Expected OKF import to produce a page path.");
|
||||
}
|
||||
const fileName = path.basename(pagePath);
|
||||
expect(Buffer.byteLength(fileName)).toBeLessThanOrEqual(255);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"Long concept body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("namespaces concept pages by bundle so repeated OKF paths do not overwrite", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-bundles-");
|
||||
const firstBundle = path.join(rootDir, "first-bundle");
|
||||
const secondBundle = path.join(rootDir, "second-bundle");
|
||||
for (const [bundlePath, title] of [
|
||||
[firstBundle, "First Customers"],
|
||||
[secondBundle, "Second Customers"],
|
||||
] as const) {
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "customers.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: ${title}
|
||||
---
|
||||
|
||||
${title} body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath: firstBundle,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath: secondBundle,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
const firstPath = getOnlyPagePath(first.pagePaths);
|
||||
const secondPath = getOnlyPagePath(second.pagePaths);
|
||||
expect(firstPath).not.toBe(secondPath);
|
||||
await expect(fs.readFile(path.join(config.vault.path, firstPath), "utf8")).resolves.toContain(
|
||||
"First Customers body.",
|
||||
);
|
||||
await expect(fs.readFile(path.join(config.vault.path, secondPath), "utf8")).resolves.toContain(
|
||||
"Second Customers body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("removes stale concept pages when an OKF bundle drops a concept", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-remove-");
|
||||
const bundlePath = path.join(rootDir, "removing-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const customersPath = path.join(bundlePath, "tables", "customers.md");
|
||||
const ordersPath = path.join(bundlePath, "tables", "orders.md");
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
ordersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Orders
|
||||
---
|
||||
|
||||
Order body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const stalePagePath = first.pagePaths.find((pagePath) => pagePath.includes("orders"));
|
||||
expect(stalePagePath).toBeDefined();
|
||||
if (!stalePagePath) {
|
||||
throw new Error("Expected initial OKF import to include orders.");
|
||||
}
|
||||
|
||||
await fs.rm(ordersPath);
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.importedCount).toBe(1);
|
||||
expect(second.removedCount).toBe(1);
|
||||
expect(second.removedPagePaths).toEqual([stalePagePath]);
|
||||
await expect(fs.stat(path.join(config.vault.path, stalePagePath))).rejects.toThrow();
|
||||
const results = await searchMemoryWiki({
|
||||
config,
|
||||
query: "Order body",
|
||||
searchCorpus: "wiki",
|
||||
});
|
||||
expect(results).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not prune existing pages when current OKF scan has invalid concepts", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-invalid-");
|
||||
const bundlePath = path.join(rootDir, "invalid-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const customersPath = path.join(bundlePath, "tables", "customers.md");
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Customer body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
await fs.writeFile(
|
||||
customersPath,
|
||||
`---
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Temporarily invalid body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.importedCount).toBe(0);
|
||||
expect(second.skippedCount).toBe(1);
|
||||
expect(second.removedCount).toBe(0);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"Customer body.",
|
||||
);
|
||||
});
|
||||
|
||||
it("detects body-only changes on timestamp-shaped markdown lines", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-body-timestamp-");
|
||||
const bundlePath = path.join(rootDir, "body-timestamp-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const conceptPath = path.join(bundlePath, "tables", "events.md");
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Events
|
||||
---
|
||||
|
||||
updatedAt: 2026-06-12
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Events
|
||||
---
|
||||
|
||||
updatedAt: 2026-06-13
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const second = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 13, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(second.updatedCount).toBe(1);
|
||||
await expect(fs.readFile(path.join(config.vault.path, pagePath), "utf8")).resolves.toContain(
|
||||
"updatedAt: 2026-06-13",
|
||||
);
|
||||
});
|
||||
|
||||
it("rewrites percent-encoded OKF markdown links and preserves suffixes", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-encoded-link-");
|
||||
const bundlePath = path.join(rootDir, "encoded-okf");
|
||||
await fs.mkdir(bundlePath, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "BigQuery Table.md"),
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: BigQuery Table
|
||||
---
|
||||
|
||||
Table body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "links.md"),
|
||||
`---
|
||||
type: Concept
|
||||
title: Links
|
||||
---
|
||||
|
||||
See [table](BigQuery%20Table.md?view=compact#columns).
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
const linksPath = result.pagePaths.find((pagePath) => pagePath.includes("links"));
|
||||
expect(linksPath).toBeDefined();
|
||||
if (!linksPath) {
|
||||
throw new Error("Expected links page to be imported.");
|
||||
}
|
||||
await expect(fs.readFile(path.join(config.vault.path, linksPath), "utf8")).resolves.toMatch(
|
||||
/\[table\]\(okf-encoded-okf-[0-9a-f]{8}-bigquery-table-[^)]+\.md\?view=compact#columns\)/,
|
||||
);
|
||||
});
|
||||
|
||||
it("imports OKF concept frontmatter with CRLF line endings", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-crlf-");
|
||||
const bundlePath = path.join(rootDir, "crlf-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(bundlePath, "tables", "events.md"),
|
||||
[
|
||||
"---",
|
||||
"type: BigQuery Table",
|
||||
"title: Events",
|
||||
"---",
|
||||
"",
|
||||
"Windows-flavored frontmatter.",
|
||||
"",
|
||||
].join("\r\n"),
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(1);
|
||||
expect(result.skippedCount).toBe(0);
|
||||
await expect(
|
||||
fs.readFile(path.join(config.vault.path, getOnlyPagePath(result.pagePaths)), "utf8"),
|
||||
).resolves.toContain("Windows-flavored frontmatter.");
|
||||
});
|
||||
|
||||
it("refuses to write imported OKF concept pages through symlinks", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-symlink-");
|
||||
const bundlePath = path.join(rootDir, "safe-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const conceptPath = path.join(bundlePath, "tables", "customers.md");
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Original body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
const first = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
const pagePath = getOnlyPagePath(first.pagePaths);
|
||||
const pageAbsolutePath = path.join(config.vault.path, pagePath);
|
||||
const externalTarget = path.join(rootDir, "outside.md");
|
||||
await fs.writeFile(externalTarget, "external target\n", "utf8");
|
||||
await fs.rm(pageAbsolutePath);
|
||||
await fs.symlink(externalTarget, pageAbsolutePath);
|
||||
await fs.writeFile(
|
||||
conceptPath,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Customers
|
||||
---
|
||||
|
||||
Updated body.
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await expect(
|
||||
importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 11, 0, 0),
|
||||
}),
|
||||
).rejects.toThrow("through symlink");
|
||||
await expect(fs.readFile(externalTarget, "utf8")).resolves.toBe("external target\n");
|
||||
});
|
||||
|
||||
it("refuses to import OKF concept files through hardlinks", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-okf-hardlink-");
|
||||
const bundlePath = path.join(rootDir, "hardlink-okf");
|
||||
await fs.mkdir(path.join(bundlePath, "tables"), { recursive: true });
|
||||
const externalSource = path.join(rootDir, "outside.md");
|
||||
await fs.writeFile(
|
||||
externalSource,
|
||||
`---
|
||||
type: BigQuery Table
|
||||
title: Private
|
||||
---
|
||||
|
||||
private body
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.link(externalSource, path.join(bundlePath, "tables", "private.md"));
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await importMemoryWikiOkfBundle({
|
||||
config,
|
||||
bundlePath,
|
||||
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.importedCount).toBe(0);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
expect(result.warnings[0]).toMatchObject({
|
||||
code: "unreadable-entry",
|
||||
path: "tables/private.md",
|
||||
});
|
||||
});
|
||||
});
|
||||
746
extensions/memory-wiki/src/okf.ts
Normal file
746
extensions/memory-wiki/src/okf.ts
Normal file
@@ -0,0 +1,746 @@
|
||||
// Memory Wiki plugin module implements Open Knowledge Format import behavior.
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { FsSafeError, root as fsRoot } from "openclaw/plugin-sdk/security-runtime";
|
||||
import {
|
||||
normalizeOptionalString,
|
||||
normalizeSingleOrTrimmedStringList,
|
||||
uniqueStrings,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
import {
|
||||
createWikiPageFilename,
|
||||
parseWikiMarkdown,
|
||||
renderWikiMarkdown,
|
||||
slugifyWikiSegment,
|
||||
WIKI_RELATED_END_MARKER,
|
||||
WIKI_RELATED_START_MARKER,
|
||||
} from "./markdown.js";
|
||||
import { resolveMemoryWikiTimestamp } from "./time.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
const OKF_RESERVED_FILENAMES = new Set(["index.md", "log.md"]);
|
||||
const OKF_MARKDOWN_LINK_PATTERN = /(!?)\[([^\]]*)\]\(([^)]+)\)/g;
|
||||
const OKF_FENCE_PATTERN = /^ {0,3}(`{3,}|~{3,})/;
|
||||
const OKF_RELATED_SECTION_PATTERN = new RegExp(
|
||||
`\\n+## Related\\n${WIKI_RELATED_START_MARKER}[\\s\\S]*?${WIKI_RELATED_END_MARKER}\\n?`,
|
||||
"g",
|
||||
);
|
||||
const OKF_VOLATILE_TIMESTAMP_LINE_PATTERN = /^(?:importedAt|updatedAt): .*\n/gm;
|
||||
const OKF_HASH_CHARS = 8;
|
||||
|
||||
type FileStatLike = {
|
||||
isFile?: unknown;
|
||||
nlink?: unknown;
|
||||
};
|
||||
|
||||
type OkfConceptDocument = {
|
||||
conceptId: string;
|
||||
relativePath: string;
|
||||
absolutePath: string;
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
type: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
resource?: string;
|
||||
tags: string[];
|
||||
timestamp?: string;
|
||||
};
|
||||
|
||||
type OkfImportedPage = {
|
||||
conceptId: string;
|
||||
sourcePath: string;
|
||||
pageId: string;
|
||||
pagePath: string;
|
||||
title: string;
|
||||
created: boolean;
|
||||
};
|
||||
|
||||
export type ImportMemoryWikiOkfWarning = {
|
||||
code: "invalid-concept" | "missing-type" | "unreadable-entry";
|
||||
path: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type ImportMemoryWikiOkfResult = {
|
||||
bundlePath: string;
|
||||
bundleName: string;
|
||||
okfVersion?: string;
|
||||
importedCount: number;
|
||||
updatedCount: number;
|
||||
removedCount: number;
|
||||
skippedCount: number;
|
||||
pagePaths: string[];
|
||||
removedPagePaths: string[];
|
||||
warnings: ImportMemoryWikiOkfWarning[];
|
||||
indexUpdatedFiles: string[];
|
||||
};
|
||||
|
||||
function toPosixPath(value: string): string {
|
||||
return value.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function trimMarkdownExtension(value: string): string {
|
||||
return value.replace(/\.md$/i, "");
|
||||
}
|
||||
|
||||
function isRegularFileStat(value: unknown): value is FileStatLike & { nlink: number } {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const stat = value as FileStatLike;
|
||||
const isFile =
|
||||
typeof stat.isFile === "function"
|
||||
? (stat.isFile as () => boolean).call(stat)
|
||||
: stat.isFile === true;
|
||||
return isFile && typeof stat.nlink === "number";
|
||||
}
|
||||
|
||||
type OkfBundleMetadata = {
|
||||
key: string;
|
||||
version?: string;
|
||||
};
|
||||
|
||||
function createOkfBundleKey(params: {
|
||||
rootFrontmatter: Record<string, unknown>;
|
||||
bundleName: string;
|
||||
bundlePath: string;
|
||||
}): string {
|
||||
const producerId =
|
||||
normalizeOptionalString(params.rootFrontmatter.id) ??
|
||||
normalizeOptionalString(params.rootFrontmatter.okf_id);
|
||||
if (producerId) {
|
||||
return slugifyWikiSegment(producerId);
|
||||
}
|
||||
const label =
|
||||
normalizeOptionalString(params.rootFrontmatter.name) ??
|
||||
normalizeOptionalString(params.rootFrontmatter.title) ??
|
||||
params.bundleName;
|
||||
const hash = createHash("sha1").update(params.bundlePath).digest("hex").slice(0, OKF_HASH_CHARS);
|
||||
return `${slugifyWikiSegment(label)}-${hash}`;
|
||||
}
|
||||
|
||||
function createOkfPageStem(bundleKey: string, conceptId: string): string {
|
||||
const slug = slugifyWikiSegment(conceptId.replace(/\//g, "-"));
|
||||
const hash = createHash("sha1").update(conceptId).digest("hex").slice(0, OKF_HASH_CHARS);
|
||||
return `okf-${bundleKey}-${slug}-${hash}`;
|
||||
}
|
||||
|
||||
function createOkfPageIdentity(
|
||||
bundleKey: string,
|
||||
conceptId: string,
|
||||
): { pageId: string; pagePath: string } {
|
||||
const fileName = createWikiPageFilename(createOkfPageStem(bundleKey, conceptId));
|
||||
const stem = trimMarkdownExtension(fileName);
|
||||
return {
|
||||
pageId: `concept.${stem}`,
|
||||
pagePath: `concepts/${fileName}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectOkfMarkdownFiles(
|
||||
rootDir: string,
|
||||
warnings: ImportMemoryWikiOkfWarning[],
|
||||
): Promise<string[]> {
|
||||
async function walk(relativeDir: string): Promise<string[]> {
|
||||
const absoluteDir = path.join(rootDir, relativeDir);
|
||||
const entries = await fs.readdir(absoluteDir, { withFileTypes: true }).catch((err: unknown) => {
|
||||
warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: toPosixPath(relativeDir) || ".",
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF directory.",
|
||||
});
|
||||
return [];
|
||||
});
|
||||
const files: string[] = [];
|
||||
for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) {
|
||||
if (entry.name === ".git" || entry.name === "node_modules") {
|
||||
continue;
|
||||
}
|
||||
const relativePath = path.join(relativeDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await walk(relativePath)));
|
||||
continue;
|
||||
}
|
||||
if (entry.isFile() && entry.name.endsWith(".md")) {
|
||||
files.push(relativePath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
return (await walk("")).map(toPosixPath).toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function parseOkfMarkdown(
|
||||
content: string,
|
||||
relativePath: string,
|
||||
): {
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
warning?: ImportMemoryWikiOkfWarning;
|
||||
} {
|
||||
const normalizedContent = content.replace(/\r\n/g, "\n");
|
||||
try {
|
||||
return parseWikiMarkdown(normalizedContent);
|
||||
} catch (err) {
|
||||
return {
|
||||
frontmatter: {},
|
||||
body: normalizedContent,
|
||||
warning: {
|
||||
code: "invalid-concept",
|
||||
path: relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to parse OKF frontmatter.",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function readOkfTextFile(params: {
|
||||
bundlePath: string;
|
||||
relativePath: string;
|
||||
warnings: ImportMemoryWikiOkfWarning[];
|
||||
}): Promise<string | null> {
|
||||
const root = await fsRoot(params.bundlePath);
|
||||
const stat = await root.stat(params.relativePath).catch((err: unknown) => {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF concept.",
|
||||
});
|
||||
return null;
|
||||
});
|
||||
if (!stat) {
|
||||
return null;
|
||||
}
|
||||
if (!isRegularFileStat(stat)) {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: "Refusing to import OKF concept through non-regular or hardlinked file.",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
return await root.readText(params.relativePath).catch((err: unknown) => {
|
||||
params.warnings.push({
|
||||
code: "unreadable-entry",
|
||||
path: params.relativePath,
|
||||
message: err instanceof Error ? err.message : "Unable to read OKF concept.",
|
||||
});
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
function deriveOkfTitle(relativePath: string, frontmatter: Record<string, unknown>): string {
|
||||
return (
|
||||
normalizeOptionalString(frontmatter.title) ??
|
||||
path.posix.basename(relativePath, ".md").replace(/[-_]+/g, " ").trim() ??
|
||||
trimMarkdownExtension(relativePath)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeOkfConcept(params: {
|
||||
bundlePath: string;
|
||||
relativePath: string;
|
||||
content: string;
|
||||
}): { concept?: OkfConceptDocument; warning?: ImportMemoryWikiOkfWarning } {
|
||||
const parsed = parseOkfMarkdown(params.content, params.relativePath);
|
||||
if (parsed.warning) {
|
||||
return { warning: parsed.warning };
|
||||
}
|
||||
|
||||
const type = normalizeOptionalString(parsed.frontmatter.type);
|
||||
if (!type) {
|
||||
return {
|
||||
warning: {
|
||||
code: "missing-type",
|
||||
path: params.relativePath,
|
||||
message: "OKF concept is missing required non-empty type frontmatter.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const conceptId = trimMarkdownExtension(params.relativePath);
|
||||
const timestamp = normalizeOptionalString(parsed.frontmatter.timestamp);
|
||||
return {
|
||||
concept: {
|
||||
conceptId,
|
||||
relativePath: params.relativePath,
|
||||
absolutePath: path.join(params.bundlePath, params.relativePath),
|
||||
frontmatter: parsed.frontmatter,
|
||||
body: parsed.body,
|
||||
type,
|
||||
title: deriveOkfTitle(params.relativePath, parsed.frontmatter),
|
||||
...(normalizeOptionalString(parsed.frontmatter.description)
|
||||
? { description: normalizeOptionalString(parsed.frontmatter.description) }
|
||||
: {}),
|
||||
...(normalizeOptionalString(parsed.frontmatter.resource)
|
||||
? { resource: normalizeOptionalString(parsed.frontmatter.resource) }
|
||||
: {}),
|
||||
tags: normalizeSingleOrTrimmedStringList(parsed.frontmatter.tags),
|
||||
...(timestamp ? { timestamp } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function splitMarkdownLinkDestination(target: string): {
|
||||
destination: string;
|
||||
titleSuffix: string;
|
||||
} {
|
||||
const trimmed = target.trim();
|
||||
if (trimmed.startsWith("<")) {
|
||||
const end = trimmed.indexOf(">");
|
||||
if (end > 0) {
|
||||
return {
|
||||
destination: trimmed.slice(1, end),
|
||||
titleSuffix: trimmed.slice(end + 1),
|
||||
};
|
||||
}
|
||||
}
|
||||
const match = trimmed.match(/^(\S+)(\s+[\s\S]+)?$/);
|
||||
return {
|
||||
destination: match?.[1] ?? trimmed,
|
||||
titleSuffix: match?.[2] ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
function resolveOkfMarkdownTarget(sourceRelativePath: string, target: string): string | null {
|
||||
const { destination } = splitMarkdownLinkDestination(target);
|
||||
const trimmed = destination.trim();
|
||||
if (!trimmed || trimmed.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(trimmed)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rawTargetWithoutSuffix = trimmed.split("#")[0]?.split("?")[0]?.replace(/\\/g, "/").trim();
|
||||
const targetWithoutSuffix = safeDecodeOkfLinkPath(rawTargetWithoutSuffix);
|
||||
if (!targetWithoutSuffix || !targetWithoutSuffix.endsWith(".md")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalized = targetWithoutSuffix.startsWith("/")
|
||||
? path.posix.normalize(targetWithoutSuffix.slice(1))
|
||||
: path.posix.normalize(
|
||||
path.posix.join(path.posix.dirname(sourceRelativePath), targetWithoutSuffix),
|
||||
);
|
||||
const conceptId = trimMarkdownExtension(normalized);
|
||||
return conceptId.startsWith("../") ? null : conceptId;
|
||||
}
|
||||
|
||||
function safeDecodeOkfLinkPath(value: string | undefined): string {
|
||||
if (!value) {
|
||||
return "";
|
||||
}
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function getMarkdownDestinationSuffix(destination: string): string {
|
||||
const queryIndex = destination.indexOf("?");
|
||||
const fragmentIndex = destination.indexOf("#");
|
||||
const suffixIndex = queryIndex === -1
|
||||
? fragmentIndex
|
||||
: fragmentIndex === -1
|
||||
? queryIndex
|
||||
: Math.min(queryIndex, fragmentIndex);
|
||||
return suffixIndex === -1 ? "" : destination.slice(suffixIndex);
|
||||
}
|
||||
|
||||
function rewriteOkfMarkdownLinks(params: {
|
||||
body: string;
|
||||
sourcePagePath: string;
|
||||
sourceRelativePath: string;
|
||||
pageByConceptId: Map<string, { pageId: string; pagePath: string; title: string }>;
|
||||
}): { body: string; linkedConceptIds: string[] } {
|
||||
const linkedConceptIds: string[] = [];
|
||||
const rewriteLinks = (markdown: string) =>
|
||||
markdown.replace(
|
||||
OKF_MARKDOWN_LINK_PATTERN,
|
||||
(match, imagePrefix: string, label: string, rawTarget: string) => {
|
||||
const conceptId = resolveOkfMarkdownTarget(params.sourceRelativePath, rawTarget);
|
||||
if (!conceptId) {
|
||||
return match;
|
||||
}
|
||||
const target = params.pageByConceptId.get(conceptId);
|
||||
if (!target) {
|
||||
return match;
|
||||
}
|
||||
linkedConceptIds.push(conceptId);
|
||||
const { destination, titleSuffix } = splitMarkdownLinkDestination(rawTarget);
|
||||
const relativeTarget = path.posix.relative(
|
||||
path.posix.dirname(params.sourcePagePath),
|
||||
target.pagePath,
|
||||
);
|
||||
const suffix = getMarkdownDestinationSuffix(destination);
|
||||
return `${imagePrefix}[${label}](${relativeTarget}${suffix}${titleSuffix})`;
|
||||
},
|
||||
);
|
||||
const body = rewriteMarkdownOutsideCode(params.body, rewriteLinks);
|
||||
return { body, linkedConceptIds: uniqueStrings(linkedConceptIds) };
|
||||
}
|
||||
|
||||
function rewriteMarkdownLineOutsideInlineCode(
|
||||
line: string,
|
||||
rewriteLinks: (markdown: string) => string,
|
||||
): string {
|
||||
let result = "";
|
||||
let cursor = 0;
|
||||
while (cursor < line.length) {
|
||||
const codeStart = line.indexOf("`", cursor);
|
||||
if (codeStart === -1) {
|
||||
result += rewriteLinks(line.slice(cursor));
|
||||
break;
|
||||
}
|
||||
result += rewriteLinks(line.slice(cursor, codeStart));
|
||||
const delimiter = line.slice(codeStart).match(/^`+/)?.[0] ?? "`";
|
||||
const codeEnd = line.indexOf(delimiter, codeStart + delimiter.length);
|
||||
if (codeEnd === -1) {
|
||||
result += line.slice(codeStart);
|
||||
break;
|
||||
}
|
||||
result += line.slice(codeStart, codeEnd + delimiter.length);
|
||||
cursor = codeEnd + delimiter.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function rewriteMarkdownOutsideCode(
|
||||
markdown: string,
|
||||
rewriteLinks: (markdown: string) => string,
|
||||
): string {
|
||||
const lines = markdown.split(/(\n)/);
|
||||
let inFence = false;
|
||||
let fenceDelimiter = "";
|
||||
return lines
|
||||
.map((line) => {
|
||||
if (line === "\n") {
|
||||
return line;
|
||||
}
|
||||
const fenceMatch = line.match(OKF_FENCE_PATTERN);
|
||||
if (fenceMatch) {
|
||||
const delimiter = fenceMatch[1] ?? "";
|
||||
const closesFence =
|
||||
inFence &&
|
||||
delimiter.startsWith(fenceDelimiter[0] ?? "") &&
|
||||
delimiter.length >= fenceDelimiter.length;
|
||||
const opensFence = !inFence;
|
||||
if (opensFence) {
|
||||
inFence = true;
|
||||
fenceDelimiter = delimiter;
|
||||
} else if (closesFence) {
|
||||
inFence = false;
|
||||
fenceDelimiter = "";
|
||||
}
|
||||
return line;
|
||||
}
|
||||
return inFence ? line : rewriteMarkdownLineOutsideInlineCode(line, rewriteLinks);
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
|
||||
function normalizeOkfRenderedPageForComparison(content: string): string {
|
||||
const withoutRelated = content.replace(OKF_RELATED_SECTION_PATTERN, "\n");
|
||||
const frontmatterMatch = withoutRelated.match(/^---\n([\s\S]*?)\n---\n?/);
|
||||
if (!frontmatterMatch) {
|
||||
return withoutRelated.trimEnd();
|
||||
}
|
||||
const normalizedFrontmatter =
|
||||
frontmatterMatch[1]?.replace(OKF_VOLATILE_TIMESTAMP_LINE_PATTERN, "") ?? "";
|
||||
const frontmatterBody = normalizedFrontmatter.endsWith("\n")
|
||||
? normalizedFrontmatter
|
||||
: `${normalizedFrontmatter}\n`;
|
||||
return `---\n${frontmatterBody}---\n${withoutRelated.slice(frontmatterMatch[0].length)}`.trimEnd();
|
||||
}
|
||||
|
||||
async function writeOkfConceptPage(params: {
|
||||
vaultRoot: string;
|
||||
pagePath: string;
|
||||
content: string;
|
||||
}): Promise<{ changed: boolean; created: boolean }> {
|
||||
const vault = await fsRoot(params.vaultRoot);
|
||||
const pageStat = await vault.stat(params.pagePath).catch((error: unknown) => {
|
||||
if (
|
||||
error instanceof FsSafeError &&
|
||||
(error.code === "not-found" || error.code === "path-alias")
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
const existing = pageStat ? await vault.readText(params.pagePath).catch(() => "") : "";
|
||||
if (
|
||||
existing === params.content ||
|
||||
normalizeOkfRenderedPageForComparison(existing) ===
|
||||
normalizeOkfRenderedPageForComparison(params.content)
|
||||
) {
|
||||
return { changed: false, created: !pageStat };
|
||||
}
|
||||
try {
|
||||
if (isRegularFileStat(pageStat) && pageStat.nlink > 1) {
|
||||
await vault.remove(params.pagePath);
|
||||
}
|
||||
await vault.write(params.pagePath, params.content);
|
||||
} catch (error) {
|
||||
if (error instanceof FsSafeError) {
|
||||
if (error.code !== "symlink" && error.code !== "path-alias") {
|
||||
throw new Error(
|
||||
`Refusing to write OKF concept page (${error.code}): ${params.pagePath}: ${error.message}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
throw new Error(`Refusing to write OKF concept page through symlink: ${params.pagePath}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { changed: true, created: !pageStat };
|
||||
}
|
||||
|
||||
async function removeStaleOkfConceptPages(params: {
|
||||
vaultRoot: string;
|
||||
bundleKey: string;
|
||||
currentPagePaths: Set<string>;
|
||||
}): Promise<string[]> {
|
||||
const vault = await fsRoot(params.vaultRoot);
|
||||
const conceptsDir = path.join(params.vaultRoot, "concepts");
|
||||
const entries = await fs.readdir(conceptsDir, { withFileTypes: true }).catch(() => []);
|
||||
const removedPagePaths: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
|
||||
continue;
|
||||
}
|
||||
const pagePath = `concepts/${entry.name}`;
|
||||
if (params.currentPagePaths.has(pagePath)) {
|
||||
continue;
|
||||
}
|
||||
const raw = await vault.readText(pagePath).catch(() => "");
|
||||
const parsed = parseWikiMarkdown(raw);
|
||||
const okf = parsed.frontmatter.okf;
|
||||
if (
|
||||
okf &&
|
||||
typeof okf === "object" &&
|
||||
!Array.isArray(okf) &&
|
||||
(okf as Record<string, unknown>).bundleKey === params.bundleKey
|
||||
) {
|
||||
await vault.remove(pagePath);
|
||||
removedPagePaths.push(pagePath);
|
||||
}
|
||||
}
|
||||
return removedPagePaths;
|
||||
}
|
||||
|
||||
function readRootOkfMetadata(params: {
|
||||
rootIndex: string | undefined;
|
||||
bundleName: string;
|
||||
bundlePath: string;
|
||||
}): OkfBundleMetadata {
|
||||
if (!params.rootIndex) {
|
||||
return {
|
||||
key: createOkfBundleKey({
|
||||
rootFrontmatter: {},
|
||||
bundleName: params.bundleName,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
};
|
||||
}
|
||||
const parsed = parseOkfMarkdown(params.rootIndex, "index.md");
|
||||
return {
|
||||
key: createOkfBundleKey({
|
||||
rootFrontmatter: parsed.frontmatter,
|
||||
bundleName: params.bundleName,
|
||||
bundlePath: params.bundlePath,
|
||||
}),
|
||||
...(normalizeOptionalString(parsed.frontmatter.okf_version)
|
||||
? { version: normalizeOptionalString(parsed.frontmatter.okf_version) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function formatOkfImportSummary(result: ImportMemoryWikiOkfResult): string {
|
||||
return `Imported ${result.importedCount} OKF concept${result.importedCount === 1 ? "" : "s"} from ${result.bundlePath} into memory wiki. Updated ${result.updatedCount}; removed ${result.removedCount}; skipped ${result.skippedCount}; refreshed ${result.indexUpdatedFiles.length} index file${result.indexUpdatedFiles.length === 1 ? "" : "s"}.`;
|
||||
}
|
||||
|
||||
export { formatOkfImportSummary };
|
||||
|
||||
export async function importMemoryWikiOkfBundle(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
bundlePath: string;
|
||||
nowMs?: number;
|
||||
}): Promise<ImportMemoryWikiOkfResult> {
|
||||
await initializeMemoryWikiVault(params.config, { nowMs: params.nowMs });
|
||||
const bundlePath = path.resolve(params.bundlePath);
|
||||
const stat = await fs.stat(bundlePath);
|
||||
if (!stat.isDirectory()) {
|
||||
throw new Error("wiki okf import expects an unpacked OKF bundle directory.");
|
||||
}
|
||||
|
||||
const warnings: ImportMemoryWikiOkfWarning[] = [];
|
||||
const markdownFiles = await collectOkfMarkdownFiles(bundlePath, warnings);
|
||||
const concepts: OkfConceptDocument[] = [];
|
||||
let rootIndexContent: string | undefined;
|
||||
|
||||
for (const relativePath of markdownFiles) {
|
||||
if (relativePath === "index.md") {
|
||||
rootIndexContent =
|
||||
(await readOkfTextFile({ bundlePath, relativePath, warnings })) ?? undefined;
|
||||
}
|
||||
if (OKF_RESERVED_FILENAMES.has(path.posix.basename(relativePath))) {
|
||||
continue;
|
||||
}
|
||||
const content = await readOkfTextFile({ bundlePath, relativePath, warnings });
|
||||
if (content === null) {
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeOkfConcept({ bundlePath, relativePath, content });
|
||||
if (normalized.warning) {
|
||||
warnings.push(normalized.warning);
|
||||
continue;
|
||||
}
|
||||
if (normalized.concept) {
|
||||
concepts.push(normalized.concept);
|
||||
}
|
||||
}
|
||||
|
||||
const timestamp = resolveMemoryWikiTimestamp(params.nowMs);
|
||||
const bundleName = path.basename(bundlePath);
|
||||
const bundleMetadata = readRootOkfMetadata({
|
||||
rootIndex: rootIndexContent,
|
||||
bundleName,
|
||||
bundlePath,
|
||||
});
|
||||
const bundleKey = bundleMetadata.key;
|
||||
const pageByConceptId = new Map<string, { pageId: string; pagePath: string; title: string }>();
|
||||
for (const concept of concepts) {
|
||||
pageByConceptId.set(concept.conceptId, {
|
||||
...createOkfPageIdentity(bundleKey, concept.conceptId),
|
||||
title: concept.title,
|
||||
});
|
||||
}
|
||||
|
||||
const importedPages: OkfImportedPage[] = [];
|
||||
let updatedCount = 0;
|
||||
|
||||
await fs.mkdir(path.join(params.config.vault.path, "concepts"), { recursive: true });
|
||||
for (const concept of concepts.toSorted((left, right) =>
|
||||
left.conceptId.localeCompare(right.conceptId),
|
||||
)) {
|
||||
const page = pageByConceptId.get(concept.conceptId);
|
||||
if (!page) {
|
||||
continue;
|
||||
}
|
||||
const rewritten = rewriteOkfMarkdownLinks({
|
||||
body: concept.body,
|
||||
sourcePagePath: page.pagePath,
|
||||
sourceRelativePath: concept.relativePath,
|
||||
pageByConceptId,
|
||||
});
|
||||
const relationships = rewritten.linkedConceptIds.flatMap((conceptId) => {
|
||||
const target = pageByConceptId.get(conceptId);
|
||||
return target
|
||||
? [
|
||||
{
|
||||
targetId: target.pageId,
|
||||
targetPath: target.pagePath,
|
||||
targetTitle: target.title,
|
||||
kind: "okf-link",
|
||||
evidenceKind: "okf-markdown-link",
|
||||
},
|
||||
]
|
||||
: [];
|
||||
});
|
||||
|
||||
const frontmatter = {
|
||||
pageType: "concept",
|
||||
id: page.pageId,
|
||||
title: concept.title,
|
||||
sourceType: "okf",
|
||||
provenanceMode: "okf-import",
|
||||
sourcePath: concept.absolutePath,
|
||||
okfConceptId: concept.conceptId,
|
||||
okfType: concept.type,
|
||||
sourceIds: [`source.okf.${bundleKey}`],
|
||||
importedAt: timestamp,
|
||||
updatedAt: concept.timestamp ?? timestamp,
|
||||
status: "active",
|
||||
...(concept.description ? { description: concept.description } : {}),
|
||||
...(concept.resource ? { resource: concept.resource } : {}),
|
||||
...(concept.tags.length > 0 ? { tags: concept.tags } : {}),
|
||||
...(concept.timestamp ? { okfTimestamp: concept.timestamp } : {}),
|
||||
...(relationships.length > 0 ? { relationships } : {}),
|
||||
okf: {
|
||||
...(bundleMetadata.version ? { version: bundleMetadata.version } : {}),
|
||||
bundleName,
|
||||
bundleKey,
|
||||
conceptId: concept.conceptId,
|
||||
sourceRelativePath: concept.relativePath,
|
||||
frontmatter: concept.frontmatter,
|
||||
},
|
||||
};
|
||||
|
||||
const writeResult = await writeOkfConceptPage({
|
||||
vaultRoot: params.config.vault.path,
|
||||
pagePath: page.pagePath,
|
||||
content: renderWikiMarkdown({
|
||||
frontmatter,
|
||||
body: rewritten.body,
|
||||
}),
|
||||
});
|
||||
if (!writeResult.created && writeResult.changed) {
|
||||
updatedCount++;
|
||||
}
|
||||
importedPages.push({
|
||||
conceptId: concept.conceptId,
|
||||
sourcePath: concept.absolutePath,
|
||||
pageId: page.pageId,
|
||||
pagePath: page.pagePath,
|
||||
title: concept.title,
|
||||
created: writeResult.created,
|
||||
});
|
||||
}
|
||||
const currentPagePaths = new Set(importedPages.map((page) => page.pagePath));
|
||||
const removedPagePaths =
|
||||
warnings.length === 0
|
||||
? await removeStaleOkfConceptPages({
|
||||
vaultRoot: params.config.vault.path,
|
||||
bundleKey,
|
||||
currentPagePaths,
|
||||
})
|
||||
: [];
|
||||
|
||||
await appendMemoryWikiLog(params.config.vault.path, {
|
||||
type: "okf-import",
|
||||
timestamp,
|
||||
details: {
|
||||
bundlePath,
|
||||
bundleName,
|
||||
importedCount: importedPages.length,
|
||||
updatedCount,
|
||||
removedCount: removedPagePaths.length,
|
||||
skippedCount: warnings.length,
|
||||
pagePaths: importedPages.map((page) => page.pagePath),
|
||||
removedPagePaths,
|
||||
},
|
||||
});
|
||||
|
||||
const compile = await compileMemoryWikiVault(params.config);
|
||||
return {
|
||||
bundlePath,
|
||||
bundleName,
|
||||
...(bundleMetadata.version ? { okfVersion: bundleMetadata.version } : {}),
|
||||
importedCount: importedPages.length,
|
||||
updatedCount,
|
||||
removedCount: removedPagePaths.length,
|
||||
skippedCount: warnings.length,
|
||||
pagePaths: importedPages.map((page) => page.pagePath),
|
||||
removedPagePaths,
|
||||
warnings,
|
||||
indexUpdatedFiles: compile.updatedFiles,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user