From 7b62fcd87da82580438e06ae802271f30bcf9378 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 5 Apr 2026 21:00:52 +0100 Subject: [PATCH] feat(memory-wiki): add unsafe-local source sync --- .../skills/wiki-maintainer/SKILL.md | 1 + extensions/memory-wiki/src/cli.ts | 46 +++- extensions/memory-wiki/src/source-sync.ts | 24 ++ extensions/memory-wiki/src/tool.ts | 15 +- .../memory-wiki/src/unsafe-local.test.ts | 56 +++++ extensions/memory-wiki/src/unsafe-local.ts | 230 ++++++++++++++++++ 6 files changed, 359 insertions(+), 13 deletions(-) create mode 100644 extensions/memory-wiki/src/source-sync.ts create mode 100644 extensions/memory-wiki/src/unsafe-local.test.ts create mode 100644 extensions/memory-wiki/src/unsafe-local.ts diff --git a/extensions/memory-wiki/skills/wiki-maintainer/SKILL.md b/extensions/memory-wiki/skills/wiki-maintainer/SKILL.md index f04fa72df0a..ae8c18f33e6 100644 --- a/extensions/memory-wiki/skills/wiki-maintainer/SKILL.md +++ b/extensions/memory-wiki/skills/wiki-maintainer/SKILL.md @@ -9,6 +9,7 @@ Use this skill when working inside a memory-wiki vault. - Use `wiki_search` to discover candidate pages, then `wiki_get` to inspect the exact page before editing or citing it. - Use `openclaw wiki ingest`, `openclaw wiki compile`, and `openclaw wiki lint` as the default maintenance loop. - In `bridge` mode, run `openclaw wiki bridge import` before relying on search results if you need the latest public memory-core artifacts pulled in. +- In `unsafe-local` mode, use `openclaw wiki unsafe-local import` only when the user explicitly opted into private local path access. - Keep generated sections inside managed markers. Do not overwrite human note blocks. - Treat raw sources, memory artifacts, and daily notes as evidence. Do not let wiki pages become the only source of truth for new claims. - Keep page identity stable. Favor updating existing entities and concepts over spawning duplicates with slightly different names. diff --git a/extensions/memory-wiki/src/cli.ts b/extensions/memory-wiki/src/cli.ts index 64a6e1bc4b0..940d94320d8 100644 --- a/extensions/memory-wiki/src/cli.ts +++ b/extensions/memory-wiki/src/cli.ts @@ -1,6 +1,5 @@ import type { Command } from "commander"; import type { OpenClawConfig } from "../api.js"; -import { syncMemoryWikiBridgeSources } from "./bridge.js"; import { compileMemoryWikiVault } from "./compile.js"; import type { MemoryWikiPluginConfig, ResolvedMemoryWikiConfig } from "./config.js"; import { resolveMemoryWikiConfig } from "./config.js"; @@ -14,6 +13,7 @@ import { runObsidianSearch, } from "./obsidian.js"; import { getMemoryWikiPage, searchMemoryWiki } from "./query.js"; +import { syncMemoryWikiImportedSources } from "./source-sync.js"; import { renderMemoryWikiStatus, resolveMemoryWikiStatus } from "./status.js"; import { initializeMemoryWikiVault } from "./vault.js"; @@ -53,6 +53,10 @@ type WikiBridgeImportCommandOptions = { json?: boolean; }; +type WikiUnsafeLocalImportCommandOptions = { + json?: boolean; +}; + type WikiObsidianSearchCommandOptions = { json?: boolean; }; @@ -92,7 +96,7 @@ export async function runWikiStatus(params: { json?: boolean; stdout?: Pick; }) { - await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig }); + await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig }); const status = await resolveMemoryWikiStatus(params.config); writeOutput( params.json ? JSON.stringify(status, null, 2) : renderMemoryWikiStatus(status), @@ -120,7 +124,7 @@ export async function runWikiCompile(params: { json?: boolean; stdout?: Pick; }) { - await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig }); + await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig }); const result = await compileMemoryWikiVault(params.config); const summary = params.json ? JSON.stringify(result, null, 2) @@ -135,7 +139,7 @@ export async function runWikiLint(params: { json?: boolean; stdout?: Pick; }) { - await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig }); + await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig }); const result = await lintMemoryWikiVault(params.config); const summary = params.json ? JSON.stringify(result, null, 2) @@ -171,7 +175,7 @@ export async function runWikiSearch(params: { json?: boolean; stdout?: Pick; }) { - await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig }); + await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig }); const results = await searchMemoryWiki({ config: params.config, query: params.query, @@ -200,7 +204,7 @@ export async function runWikiGet(params: { json?: boolean; stdout?: Pick; }) { - await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig }); + await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig }); const result = await getMemoryWikiPage({ config: params.config, lookup: params.lookup, @@ -220,7 +224,7 @@ export async function runWikiBridgeImport(params: { json?: boolean; stdout?: Pick; }) { - const result = await syncMemoryWikiBridgeSources({ + const result = await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig, }); @@ -231,6 +235,23 @@ export async function runWikiBridgeImport(params: { return result; } +export async function runWikiUnsafeLocalImport(params: { + config: ResolvedMemoryWikiConfig; + appConfig?: OpenClawConfig; + json?: boolean; + stdout?: Pick; +}) { + const result = await syncMemoryWikiImportedSources({ + config: params.config, + appConfig: params.appConfig, + }); + const summary = params.json + ? JSON.stringify(result, null, 2) + : `Unsafe-local import synced ${result.artifactCount} artifacts (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged).`; + writeOutput(summary, params.stdout); + return result; +} + export async function runWikiObsidianStatus(params: { config: ResolvedMemoryWikiConfig; json?: boolean; @@ -396,6 +417,17 @@ export function registerWikiCli( await runWikiBridgeImport({ config, appConfig, json: opts.json }); }); + const unsafeLocal = wiki + .command("unsafe-local") + .description("Import explicitly configured private local paths into wiki source pages"); + unsafeLocal + .command("import") + .description("Sync unsafe-local configured paths into wiki source pages") + .option("--json", "Print JSON") + .action(async (opts: WikiUnsafeLocalImportCommandOptions) => { + await runWikiUnsafeLocalImport({ config, appConfig, json: opts.json }); + }); + const obsidian = wiki.command("obsidian").description("Run official Obsidian CLI helpers"); obsidian .command("status") diff --git a/extensions/memory-wiki/src/source-sync.ts b/extensions/memory-wiki/src/source-sync.ts new file mode 100644 index 00000000000..a1998ed1d2b --- /dev/null +++ b/extensions/memory-wiki/src/source-sync.ts @@ -0,0 +1,24 @@ +import type { OpenClawConfig } from "../api.js"; +import { syncMemoryWikiBridgeSources, type BridgeMemoryWikiResult } from "./bridge.js"; +import type { ResolvedMemoryWikiConfig } from "./config.js"; +import { syncMemoryWikiUnsafeLocalSources } from "./unsafe-local.js"; + +export async function syncMemoryWikiImportedSources(params: { + config: ResolvedMemoryWikiConfig; + appConfig?: OpenClawConfig; +}): Promise { + if (params.config.vaultMode === "bridge") { + return await syncMemoryWikiBridgeSources(params); + } + if (params.config.vaultMode === "unsafe-local") { + return await syncMemoryWikiUnsafeLocalSources(params.config); + } + return { + importedCount: 0, + updatedCount: 0, + skippedCount: 0, + artifactCount: 0, + workspaces: 0, + pagePaths: [], + }; +} diff --git a/extensions/memory-wiki/src/tool.ts b/extensions/memory-wiki/src/tool.ts index c6e0a69a7c3..f9f5a16c891 100644 --- a/extensions/memory-wiki/src/tool.ts +++ b/extensions/memory-wiki/src/tool.ts @@ -1,8 +1,8 @@ import { Type } from "@sinclair/typebox"; import type { AnyAgentTool, OpenClawConfig } from "../api.js"; -import { syncMemoryWikiBridgeSources } from "./bridge.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { getMemoryWikiPage, searchMemoryWiki } from "./query.js"; +import { syncMemoryWikiImportedSources } from "./source-sync.js"; import { renderMemoryWikiStatus, resolveMemoryWikiStatus } from "./status.js"; const WikiStatusSchema = Type.Object({}, { additionalProperties: false }); @@ -22,8 +22,11 @@ const WikiGetSchema = Type.Object( { additionalProperties: false }, ); -async function syncBridgeIfNeeded(config: ResolvedMemoryWikiConfig, appConfig?: OpenClawConfig) { - await syncMemoryWikiBridgeSources({ config, appConfig }); +async function syncImportedSourcesIfNeeded( + config: ResolvedMemoryWikiConfig, + appConfig?: OpenClawConfig, +) { + await syncMemoryWikiImportedSources({ config, appConfig }); } export function createWikiStatusTool( @@ -37,7 +40,7 @@ export function createWikiStatusTool( "Inspect the current memory wiki vault mode, health, and Obsidian CLI availability.", parameters: WikiStatusSchema, execute: async () => { - await syncBridgeIfNeeded(config, appConfig); + await syncImportedSourcesIfNeeded(config, appConfig); const status = await resolveMemoryWikiStatus(config); return { content: [{ type: "text", text: renderMemoryWikiStatus(status) }], @@ -58,7 +61,7 @@ export function createWikiSearchTool( parameters: WikiSearchSchema, execute: async (_toolCallId, rawParams) => { const params = rawParams as { query: string; maxResults?: number }; - await syncBridgeIfNeeded(config, appConfig); + await syncImportedSourcesIfNeeded(config, appConfig); const results = await searchMemoryWiki({ config, query: params.query, @@ -92,7 +95,7 @@ export function createWikiGetTool( parameters: WikiGetSchema, execute: async (_toolCallId, rawParams) => { const params = rawParams as { lookup: string; fromLine?: number; lineCount?: number }; - await syncBridgeIfNeeded(config, appConfig); + await syncImportedSourcesIfNeeded(config, appConfig); const result = await getMemoryWikiPage({ config, lookup: params.lookup, diff --git a/extensions/memory-wiki/src/unsafe-local.test.ts b/extensions/memory-wiki/src/unsafe-local.test.ts new file mode 100644 index 00000000000..8848bf28a6c --- /dev/null +++ b/extensions/memory-wiki/src/unsafe-local.test.ts @@ -0,0 +1,56 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveMemoryWikiConfig } from "./config.js"; +import { syncMemoryWikiUnsafeLocalSources } from "./unsafe-local.js"; + +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +describe("syncMemoryWikiUnsafeLocalSources", () => { + it("imports explicit private paths and preserves unsafe-local provenance", async () => { + const privateDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-private-")); + const vaultDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-unsafe-vault-")); + tempDirs.push(privateDir, vaultDir); + + await fs.mkdir(path.join(privateDir, "nested"), { recursive: true }); + await fs.writeFile(path.join(privateDir, "nested", "state.md"), "# internal state\n", "utf8"); + await fs.writeFile(path.join(privateDir, "nested", "cache.json"), '{"ok":true}\n', "utf8"); + await fs.writeFile(path.join(privateDir, "nested", "blob.bin"), "\u0000\u0001", "utf8"); + const directPath = path.join(privateDir, "events.log"); + await fs.writeFile(directPath, "private log\n", "utf8"); + + const config = resolveMemoryWikiConfig( + { + vaultMode: "unsafe-local", + vault: { path: vaultDir }, + unsafeLocal: { + allowPrivateMemoryCoreAccess: true, + paths: [path.join(privateDir, "nested"), directPath], + }, + }, + { homedir: "/Users/tester" }, + ); + + const first = await syncMemoryWikiUnsafeLocalSources(config); + + expect(first.artifactCount).toBe(3); + expect(first.importedCount).toBe(3); + expect(first.updatedCount).toBe(0); + expect(first.skippedCount).toBe(0); + + const page = await fs.readFile(path.join(vaultDir, first.pagePaths[0] ?? ""), "utf8"); + expect(page).toContain("sourceType: memory-unsafe-local"); + expect(page).toContain("provenanceMode: unsafe-local"); + + const second = await syncMemoryWikiUnsafeLocalSources(config); + + expect(second.importedCount).toBe(0); + expect(second.updatedCount).toBe(0); + expect(second.skippedCount).toBe(3); + }); +}); diff --git a/extensions/memory-wiki/src/unsafe-local.ts b/extensions/memory-wiki/src/unsafe-local.ts new file mode 100644 index 00000000000..af95b10d497 --- /dev/null +++ b/extensions/memory-wiki/src/unsafe-local.ts @@ -0,0 +1,230 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { BridgeMemoryWikiResult } from "./bridge.js"; +import type { ResolvedMemoryWikiConfig } from "./config.js"; +import { appendMemoryWikiLog } from "./log.js"; +import { renderMarkdownFence, renderWikiMarkdown, slugifyWikiSegment } from "./markdown.js"; +import { initializeMemoryWikiVault } from "./vault.js"; + +type UnsafeLocalArtifact = { + configuredPath: string; + absolutePath: string; + relativePath: string; +}; + +const DIRECTORY_TEXT_EXTENSIONS = new Set([".json", ".jsonl", ".md", ".txt", ".yaml", ".yml"]); + +async function pathExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function resolveArtifactKey(absolutePath: string): Promise { + const canonicalPath = await fs.realpath(absolutePath).catch(() => path.resolve(absolutePath)); + return process.platform === "win32" ? canonicalPath.toLowerCase() : canonicalPath; +} + +function detectFenceLanguage(filePath: string): string { + const ext = path.extname(filePath).toLowerCase(); + if (ext === ".json" || ext === ".jsonl") { + return "json"; + } + if (ext === ".yaml" || ext === ".yml") { + return "yaml"; + } + if (ext === ".txt") { + return "text"; + } + return "markdown"; +} + +async function listAllowedFilesRecursive(rootDir: string): Promise { + const entries = await fs.readdir(rootDir, { withFileTypes: true }).catch(() => []); + const files: string[] = []; + for (const entry of entries) { + const fullPath = path.join(rootDir, entry.name); + if (entry.isDirectory()) { + files.push(...(await listAllowedFilesRecursive(fullPath))); + continue; + } + if (entry.isFile() && DIRECTORY_TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) { + files.push(fullPath); + } + } + return files.toSorted((left, right) => left.localeCompare(right)); +} + +async function collectUnsafeLocalArtifacts( + configuredPaths: string[], +): Promise { + const artifacts: UnsafeLocalArtifact[] = []; + for (const configuredPath of configuredPaths) { + const absoluteConfiguredPath = path.resolve(configuredPath); + const stat = await fs.stat(absoluteConfiguredPath).catch(() => null); + if (!stat) { + continue; + } + if (stat.isDirectory()) { + const files = await listAllowedFilesRecursive(absoluteConfiguredPath); + for (const absolutePath of files) { + artifacts.push({ + configuredPath: absoluteConfiguredPath, + absolutePath, + relativePath: path.relative(absoluteConfiguredPath, absolutePath).replace(/\\/g, "/"), + }); + } + continue; + } + if (stat.isFile()) { + artifacts.push({ + configuredPath: absoluteConfiguredPath, + absolutePath: absoluteConfiguredPath, + relativePath: path.basename(absoluteConfiguredPath), + }); + } + } + + const deduped = new Map(); + for (const artifact of artifacts) { + deduped.set(await resolveArtifactKey(artifact.absolutePath), artifact); + } + return [...deduped.values()]; +} + +function resolveUnsafeLocalPagePath(params: { configuredPath: string; absolutePath: string }): { + pageId: string; + pagePath: string; +} { + const configuredBaseSlug = slugifyWikiSegment(path.basename(params.configuredPath)); + const configuredHash = createHash("sha1") + .update(path.resolve(params.configuredPath)) + .digest("hex") + .slice(0, 8); + const artifactBaseSlug = slugifyWikiSegment(path.basename(params.absolutePath)); + const artifactHash = createHash("sha1") + .update(path.resolve(params.absolutePath)) + .digest("hex") + .slice(0, 8); + const pageSlug = `${configuredBaseSlug}-${configuredHash}-${artifactBaseSlug}-${artifactHash}`; + return { + pageId: `source.unsafe-local.${pageSlug}`, + pagePath: path.join("sources", `unsafe-local-${pageSlug}.md`).replace(/\\/g, "/"), + }; +} + +function resolveUnsafeLocalTitle(artifact: UnsafeLocalArtifact): string { + return `Unsafe Local Import: ${artifact.relativePath}`; +} + +async function writeUnsafeLocalSourcePage(params: { + config: ResolvedMemoryWikiConfig; + artifact: UnsafeLocalArtifact; +}): Promise<{ pagePath: string; changed: boolean; created: boolean }> { + const { pageId, pagePath } = resolveUnsafeLocalPagePath({ + configuredPath: params.artifact.configuredPath, + absolutePath: params.artifact.absolutePath, + }); + const pageAbsPath = path.join(params.config.vault.path, pagePath); + const created = !(await pathExists(pageAbsPath)); + const raw = await fs.readFile(params.artifact.absolutePath, "utf8"); + const stats = await fs.stat(params.artifact.absolutePath); + const updatedAt = stats.mtime.toISOString(); + const title = resolveUnsafeLocalTitle(params.artifact); + const rendered = renderWikiMarkdown({ + frontmatter: { + pageType: "source", + id: pageId, + title, + sourceType: "memory-unsafe-local", + provenanceMode: "unsafe-local", + sourcePath: params.artifact.absolutePath, + unsafeLocalConfiguredPath: params.artifact.configuredPath, + unsafeLocalRelativePath: params.artifact.relativePath, + status: "active", + updatedAt, + }, + body: [ + `# ${title}`, + "", + "## Unsafe Local Source", + `- Configured path: \`${params.artifact.configuredPath}\``, + `- Relative path: \`${params.artifact.relativePath}\``, + `- Updated: ${updatedAt}`, + "", + "## Content", + renderMarkdownFence(raw, detectFenceLanguage(params.artifact.absolutePath)), + "", + "## Notes", + "", + "", + "", + ].join("\n"), + }); + const existing = await fs.readFile(pageAbsPath, "utf8").catch(() => ""); + if (existing === rendered) { + return { pagePath, changed: false, created }; + } + await fs.writeFile(pageAbsPath, rendered, "utf8"); + return { pagePath, changed: true, created }; +} + +export async function syncMemoryWikiUnsafeLocalSources( + config: ResolvedMemoryWikiConfig, +): Promise { + await initializeMemoryWikiVault(config); + if ( + config.vaultMode !== "unsafe-local" || + !config.unsafeLocal.allowPrivateMemoryCoreAccess || + config.unsafeLocal.paths.length === 0 + ) { + return { + importedCount: 0, + updatedCount: 0, + skippedCount: 0, + artifactCount: 0, + workspaces: 0, + pagePaths: [], + }; + } + + const artifacts = await collectUnsafeLocalArtifacts(config.unsafeLocal.paths); + const results = await Promise.all( + artifacts.map((artifact) => writeUnsafeLocalSourcePage({ config, artifact })), + ); + + const importedCount = results.filter((result) => result.changed && result.created).length; + const updatedCount = results.filter((result) => result.changed && !result.created).length; + const skippedCount = results.filter((result) => !result.changed).length; + const pagePaths = results + .map((result) => result.pagePath) + .toSorted((left, right) => left.localeCompare(right)); + + if (importedCount > 0 || updatedCount > 0) { + await appendMemoryWikiLog(config.vault.path, { + type: "ingest", + timestamp: new Date().toISOString(), + details: { + sourceType: "memory-unsafe-local", + configuredPathCount: config.unsafeLocal.paths.length, + artifactCount: artifacts.length, + importedCount, + updatedCount, + skippedCount, + }, + }); + } + + return { + importedCount, + updatedCount, + skippedCount, + artifactCount: artifacts.length, + workspaces: 0, + pagePaths, + }; +}