feat(memory-wiki): add unsafe-local source sync

extensions/memory-wiki/skills/wiki-maintainer/SKILL.md

@@ -9,6 +9,7 @@ Use this skill when working inside a memory-wiki vault.
 - Use `wiki_search` to discover candidate pages, then `wiki_get` to inspect the exact page before editing or citing it.
 - Use `openclaw wiki ingest`, `openclaw wiki compile`, and `openclaw wiki lint` as the default maintenance loop.
 - In `bridge` mode, run `openclaw wiki bridge import` before relying on search results if you need the latest public memory-core artifacts pulled in.
+- In `unsafe-local` mode, use `openclaw wiki unsafe-local import` only when the user explicitly opted into private local path access.
 - Keep generated sections inside managed markers. Do not overwrite human note blocks.
 - Treat raw sources, memory artifacts, and daily notes as evidence. Do not let wiki pages become the only source of truth for new claims.
 - Keep page identity stable. Favor updating existing entities and concepts over spawning duplicates with slightly different names.

extensions/memory-wiki/src/cli.ts

@@ -1,6 +1,5 @@
 import type { Command } from "commander";
 import type { OpenClawConfig } from "../api.js";
-import { syncMemoryWikiBridgeSources } from "./bridge.js";
 import { compileMemoryWikiVault } from "./compile.js";
 import type { MemoryWikiPluginConfig, ResolvedMemoryWikiConfig } from "./config.js";
 import { resolveMemoryWikiConfig } from "./config.js";
@@ -14,6 +13,7 @@ import {
   runObsidianSearch,
 } from "./obsidian.js";
 import { getMemoryWikiPage, searchMemoryWiki } from "./query.js";
+import { syncMemoryWikiImportedSources } from "./source-sync.js";
 import { renderMemoryWikiStatus, resolveMemoryWikiStatus } from "./status.js";
 import { initializeMemoryWikiVault } from "./vault.js";
 
@@ -53,6 +53,10 @@ type WikiBridgeImportCommandOptions = {
   json?: boolean;
 };
 
+type WikiUnsafeLocalImportCommandOptions = {
+  json?: boolean;
+};
+
 type WikiObsidianSearchCommandOptions = {
   json?: boolean;
 };
@@ -92,7 +96,7 @@ export async function runWikiStatus(params: {
   json?: boolean;
   stdout?: Pick<NodeJS.WriteStream, "write">;
 }) {
-  await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig });
+  await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
   const status = await resolveMemoryWikiStatus(params.config);
   writeOutput(
     params.json ? JSON.stringify(status, null, 2) : renderMemoryWikiStatus(status),
@@ -120,7 +124,7 @@ export async function runWikiCompile(params: {
   json?: boolean;
   stdout?: Pick<NodeJS.WriteStream, "write">;
 }) {
-  await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig });
+  await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
   const result = await compileMemoryWikiVault(params.config);
   const summary = params.json
     ? JSON.stringify(result, null, 2)
@@ -135,7 +139,7 @@ export async function runWikiLint(params: {
   json?: boolean;
   stdout?: Pick<NodeJS.WriteStream, "write">;
 }) {
-  await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig });
+  await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
   const result = await lintMemoryWikiVault(params.config);
   const summary = params.json
     ? JSON.stringify(result, null, 2)
@@ -171,7 +175,7 @@ export async function runWikiSearch(params: {
   json?: boolean;
   stdout?: Pick<NodeJS.WriteStream, "write">;
 }) {
-  await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig });
+  await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
   const results = await searchMemoryWiki({
     config: params.config,
     query: params.query,
@@ -200,7 +204,7 @@ export async function runWikiGet(params: {
   json?: boolean;
   stdout?: Pick<NodeJS.WriteStream, "write">;
 }) {
-  await syncMemoryWikiBridgeSources({ config: params.config, appConfig: params.appConfig });
+  await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
   const result = await getMemoryWikiPage({
     config: params.config,
     lookup: params.lookup,
@@ -220,7 +224,7 @@ export async function runWikiBridgeImport(params: {
   json?: boolean;
   stdout?: Pick<NodeJS.WriteStream, "write">;
 }) {
-  const result = await syncMemoryWikiBridgeSources({
+  const result = await syncMemoryWikiImportedSources({
     config: params.config,
     appConfig: params.appConfig,
   });
@@ -231,6 +235,23 @@ export async function runWikiBridgeImport(params: {
   return result;
 }
 
+export async function runWikiUnsafeLocalImport(params: {
+  config: ResolvedMemoryWikiConfig;
+  appConfig?: OpenClawConfig;
+  json?: boolean;
+  stdout?: Pick<NodeJS.WriteStream, "write">;
+}) {
+  const result = await syncMemoryWikiImportedSources({
+    config: params.config,
+    appConfig: params.appConfig,
+  });
+  const summary = params.json
+    ? JSON.stringify(result, null, 2)
+    : `Unsafe-local import synced ${result.artifactCount} artifacts (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged).`;
+  writeOutput(summary, params.stdout);
+  return result;
+}
+
 export async function runWikiObsidianStatus(params: {
   config: ResolvedMemoryWikiConfig;
   json?: boolean;
@@ -396,6 +417,17 @@ export function registerWikiCli(
       await runWikiBridgeImport({ config, appConfig, json: opts.json });
     });
 
+  const unsafeLocal = wiki
+    .command("unsafe-local")
+    .description("Import explicitly configured private local paths into wiki source pages");
+  unsafeLocal
+    .command("import")
+    .description("Sync unsafe-local configured paths into wiki source pages")
+    .option("--json", "Print JSON")
+    .action(async (opts: WikiUnsafeLocalImportCommandOptions) => {
+      await runWikiUnsafeLocalImport({ config, appConfig, json: opts.json });
+    });
+
   const obsidian = wiki.command("obsidian").description("Run official Obsidian CLI helpers");
   obsidian
     .command("status")

extensions/memory-wiki/src/source-sync.ts

@@ -0,0 +1,24 @@
+import type { OpenClawConfig } from "../api.js";
+import { syncMemoryWikiBridgeSources, type BridgeMemoryWikiResult } from "./bridge.js";
+import type { ResolvedMemoryWikiConfig } from "./config.js";
+import { syncMemoryWikiUnsafeLocalSources } from "./unsafe-local.js";
+
+export async function syncMemoryWikiImportedSources(params: {
+  config: ResolvedMemoryWikiConfig;
+  appConfig?: OpenClawConfig;
+}): Promise<BridgeMemoryWikiResult> {
+  if (params.config.vaultMode === "bridge") {
+    return await syncMemoryWikiBridgeSources(params);
+  }
+  if (params.config.vaultMode === "unsafe-local") {
+    return await syncMemoryWikiUnsafeLocalSources(params.config);
+  }
+  return {
+    importedCount: 0,
+    updatedCount: 0,
+    skippedCount: 0,
+    artifactCount: 0,
+    workspaces: 0,
+    pagePaths: [],
+  };
+}

extensions/memory-wiki/src/tool.ts

@@ -1,8 +1,8 @@
 import { Type } from "@sinclair/typebox";
 import type { AnyAgentTool, OpenClawConfig } from "../api.js";
-import { syncMemoryWikiBridgeSources } from "./bridge.js";
 import type { ResolvedMemoryWikiConfig } from "./config.js";
 import { getMemoryWikiPage, searchMemoryWiki } from "./query.js";
+import { syncMemoryWikiImportedSources } from "./source-sync.js";
 import { renderMemoryWikiStatus, resolveMemoryWikiStatus } from "./status.js";
 
 const WikiStatusSchema = Type.Object({}, { additionalProperties: false });
@@ -22,8 +22,11 @@ const WikiGetSchema = Type.Object(
   { additionalProperties: false },
 );
 
-async function syncBridgeIfNeeded(config: ResolvedMemoryWikiConfig, appConfig?: OpenClawConfig) {
-  await syncMemoryWikiBridgeSources({ config, appConfig });
+async function syncImportedSourcesIfNeeded(
+  config: ResolvedMemoryWikiConfig,
+  appConfig?: OpenClawConfig,
+) {
+  await syncMemoryWikiImportedSources({ config, appConfig });
 }
 
 export function createWikiStatusTool(
@@ -37,7 +40,7 @@ export function createWikiStatusTool(
       "Inspect the current memory wiki vault mode, health, and Obsidian CLI availability.",
     parameters: WikiStatusSchema,
     execute: async () => {
-      await syncBridgeIfNeeded(config, appConfig);
+      await syncImportedSourcesIfNeeded(config, appConfig);
       const status = await resolveMemoryWikiStatus(config);
       return {
         content: [{ type: "text", text: renderMemoryWikiStatus(status) }],
@@ -58,7 +61,7 @@ export function createWikiSearchTool(
     parameters: WikiSearchSchema,
     execute: async (_toolCallId, rawParams) => {
       const params = rawParams as { query: string; maxResults?: number };
-      await syncBridgeIfNeeded(config, appConfig);
+      await syncImportedSourcesIfNeeded(config, appConfig);
       const results = await searchMemoryWiki({
         config,
         query: params.query,
@@ -92,7 +95,7 @@ export function createWikiGetTool(
     parameters: WikiGetSchema,
     execute: async (_toolCallId, rawParams) => {
       const params = rawParams as { lookup: string; fromLine?: number; lineCount?: number };
-      await syncBridgeIfNeeded(config, appConfig);
+      await syncImportedSourcesIfNeeded(config, appConfig);
       const result = await getMemoryWikiPage({
         config,
         lookup: params.lookup,

extensions/memory-wiki/src/unsafe-local.test.ts

@@ -0,0 +1,56 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { resolveMemoryWikiConfig } from "./config.js";
+import { syncMemoryWikiUnsafeLocalSources } from "./unsafe-local.js";
+
+const tempDirs: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
+});
+
+describe("syncMemoryWikiUnsafeLocalSources", () => {
+  it("imports explicit private paths and preserves unsafe-local provenance", async () => {
+    const privateDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-private-"));
+    const vaultDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-unsafe-vault-"));
+    tempDirs.push(privateDir, vaultDir);
+
+    await fs.mkdir(path.join(privateDir, "nested"), { recursive: true });
+    await fs.writeFile(path.join(privateDir, "nested", "state.md"), "# internal state\n", "utf8");
+    await fs.writeFile(path.join(privateDir, "nested", "cache.json"), '{"ok":true}\n', "utf8");
+    await fs.writeFile(path.join(privateDir, "nested", "blob.bin"), "\u0000\u0001", "utf8");
+    const directPath = path.join(privateDir, "events.log");
+    await fs.writeFile(directPath, "private log\n", "utf8");
+
+    const config = resolveMemoryWikiConfig(
+      {
+        vaultMode: "unsafe-local",
+        vault: { path: vaultDir },
+        unsafeLocal: {
+          allowPrivateMemoryCoreAccess: true,
+          paths: [path.join(privateDir, "nested"), directPath],
+        },
+      },
+      { homedir: "/Users/tester" },
+    );
+
+    const first = await syncMemoryWikiUnsafeLocalSources(config);
+
+    expect(first.artifactCount).toBe(3);
+    expect(first.importedCount).toBe(3);
+    expect(first.updatedCount).toBe(0);
+    expect(first.skippedCount).toBe(0);
+
+    const page = await fs.readFile(path.join(vaultDir, first.pagePaths[0] ?? ""), "utf8");
+    expect(page).toContain("sourceType: memory-unsafe-local");
+    expect(page).toContain("provenanceMode: unsafe-local");
+
+    const second = await syncMemoryWikiUnsafeLocalSources(config);
+
+    expect(second.importedCount).toBe(0);
+    expect(second.updatedCount).toBe(0);
+    expect(second.skippedCount).toBe(3);
+  });
+});

extensions/memory-wiki/src/unsafe-local.ts

@@ -0,0 +1,230 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { BridgeMemoryWikiResult } from "./bridge.js";
+import type { ResolvedMemoryWikiConfig } from "./config.js";
+import { appendMemoryWikiLog } from "./log.js";
+import { renderMarkdownFence, renderWikiMarkdown, slugifyWikiSegment } from "./markdown.js";
+import { initializeMemoryWikiVault } from "./vault.js";
+
+type UnsafeLocalArtifact = {
+  configuredPath: string;
+  absolutePath: string;
+  relativePath: string;
+};
+
+const DIRECTORY_TEXT_EXTENSIONS = new Set([".json", ".jsonl", ".md", ".txt", ".yaml", ".yml"]);
+
+async function pathExists(filePath: string): Promise<boolean> {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function resolveArtifactKey(absolutePath: string): Promise<string> {
+  const canonicalPath = await fs.realpath(absolutePath).catch(() => path.resolve(absolutePath));
+  return process.platform === "win32" ? canonicalPath.toLowerCase() : canonicalPath;
+}
+
+function detectFenceLanguage(filePath: string): string {
+  const ext = path.extname(filePath).toLowerCase();
+  if (ext === ".json" || ext === ".jsonl") {
+    return "json";
+  }
+  if (ext === ".yaml" || ext === ".yml") {
+    return "yaml";
+  }
+  if (ext === ".txt") {
+    return "text";
+  }
+  return "markdown";
+}
+
+async function listAllowedFilesRecursive(rootDir: string): Promise<string[]> {
+  const entries = await fs.readdir(rootDir, { withFileTypes: true }).catch(() => []);
+  const files: string[] = [];
+  for (const entry of entries) {
+    const fullPath = path.join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...(await listAllowedFilesRecursive(fullPath)));
+      continue;
+    }
+    if (entry.isFile() && DIRECTORY_TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) {
+      files.push(fullPath);
+    }
+  }
+  return files.toSorted((left, right) => left.localeCompare(right));
+}
+
+async function collectUnsafeLocalArtifacts(
+  configuredPaths: string[],
+): Promise<UnsafeLocalArtifact[]> {
+  const artifacts: UnsafeLocalArtifact[] = [];
+  for (const configuredPath of configuredPaths) {
+    const absoluteConfiguredPath = path.resolve(configuredPath);
+    const stat = await fs.stat(absoluteConfiguredPath).catch(() => null);
+    if (!stat) {
+      continue;
+    }
+    if (stat.isDirectory()) {
+      const files = await listAllowedFilesRecursive(absoluteConfiguredPath);
+      for (const absolutePath of files) {
+        artifacts.push({
+          configuredPath: absoluteConfiguredPath,
+          absolutePath,
+          relativePath: path.relative(absoluteConfiguredPath, absolutePath).replace(/\\/g, "/"),
+        });
+      }
+      continue;
+    }
+    if (stat.isFile()) {
+      artifacts.push({
+        configuredPath: absoluteConfiguredPath,
+        absolutePath: absoluteConfiguredPath,
+        relativePath: path.basename(absoluteConfiguredPath),
+      });
+    }
+  }
+
+  const deduped = new Map<string, UnsafeLocalArtifact>();
+  for (const artifact of artifacts) {
+    deduped.set(await resolveArtifactKey(artifact.absolutePath), artifact);
+  }
+  return [...deduped.values()];
+}
+
+function resolveUnsafeLocalPagePath(params: { configuredPath: string; absolutePath: string }): {
+  pageId: string;
+  pagePath: string;
+} {
+  const configuredBaseSlug = slugifyWikiSegment(path.basename(params.configuredPath));
+  const configuredHash = createHash("sha1")
+    .update(path.resolve(params.configuredPath))
+    .digest("hex")
+    .slice(0, 8);
+  const artifactBaseSlug = slugifyWikiSegment(path.basename(params.absolutePath));
+  const artifactHash = createHash("sha1")
+    .update(path.resolve(params.absolutePath))
+    .digest("hex")
+    .slice(0, 8);
+  const pageSlug = `${configuredBaseSlug}-${configuredHash}-${artifactBaseSlug}-${artifactHash}`;
+  return {
+    pageId: `source.unsafe-local.${pageSlug}`,
+    pagePath: path.join("sources", `unsafe-local-${pageSlug}.md`).replace(/\\/g, "/"),
+  };
+}
+
+function resolveUnsafeLocalTitle(artifact: UnsafeLocalArtifact): string {
+  return `Unsafe Local Import: ${artifact.relativePath}`;
+}
+
+async function writeUnsafeLocalSourcePage(params: {
+  config: ResolvedMemoryWikiConfig;
+  artifact: UnsafeLocalArtifact;
+}): Promise<{ pagePath: string; changed: boolean; created: boolean }> {
+  const { pageId, pagePath } = resolveUnsafeLocalPagePath({
+    configuredPath: params.artifact.configuredPath,
+    absolutePath: params.artifact.absolutePath,
+  });
+  const pageAbsPath = path.join(params.config.vault.path, pagePath);
+  const created = !(await pathExists(pageAbsPath));
+  const raw = await fs.readFile(params.artifact.absolutePath, "utf8");
+  const stats = await fs.stat(params.artifact.absolutePath);
+  const updatedAt = stats.mtime.toISOString();
+  const title = resolveUnsafeLocalTitle(params.artifact);
+  const rendered = renderWikiMarkdown({
+    frontmatter: {
+      pageType: "source",
+      id: pageId,
+      title,
+      sourceType: "memory-unsafe-local",
+      provenanceMode: "unsafe-local",
+      sourcePath: params.artifact.absolutePath,
+      unsafeLocalConfiguredPath: params.artifact.configuredPath,
+      unsafeLocalRelativePath: params.artifact.relativePath,
+      status: "active",
+      updatedAt,
+    },
+    body: [
+      `# ${title}`,
+      "",
+      "## Unsafe Local Source",
+      `- Configured path: \`${params.artifact.configuredPath}\``,
+      `- Relative path: \`${params.artifact.relativePath}\``,
+      `- Updated: ${updatedAt}`,
+      "",
+      "## Content",
+      renderMarkdownFence(raw, detectFenceLanguage(params.artifact.absolutePath)),
+      "",
+      "## Notes",
+      "<!-- openclaw:human:start -->",
+      "<!-- openclaw:human:end -->",
+      "",
+    ].join("\n"),
+  });
+  const existing = await fs.readFile(pageAbsPath, "utf8").catch(() => "");
+  if (existing === rendered) {
+    return { pagePath, changed: false, created };
+  }
+  await fs.writeFile(pageAbsPath, rendered, "utf8");
+  return { pagePath, changed: true, created };
+}
+
+export async function syncMemoryWikiUnsafeLocalSources(
+  config: ResolvedMemoryWikiConfig,
+): Promise<BridgeMemoryWikiResult> {
+  await initializeMemoryWikiVault(config);
+  if (
+    config.vaultMode !== "unsafe-local" ||
+    !config.unsafeLocal.allowPrivateMemoryCoreAccess ||
+    config.unsafeLocal.paths.length === 0
+  ) {
+    return {
+      importedCount: 0,
+      updatedCount: 0,
+      skippedCount: 0,
+      artifactCount: 0,
+      workspaces: 0,
+      pagePaths: [],
+    };
+  }
+
+  const artifacts = await collectUnsafeLocalArtifacts(config.unsafeLocal.paths);
+  const results = await Promise.all(
+    artifacts.map((artifact) => writeUnsafeLocalSourcePage({ config, artifact })),
+  );
+
+  const importedCount = results.filter((result) => result.changed && result.created).length;
+  const updatedCount = results.filter((result) => result.changed && !result.created).length;
+  const skippedCount = results.filter((result) => !result.changed).length;
+  const pagePaths = results
+    .map((result) => result.pagePath)
+    .toSorted((left, right) => left.localeCompare(right));
+
+  if (importedCount > 0 || updatedCount > 0) {
+    await appendMemoryWikiLog(config.vault.path, {
+      type: "ingest",
+      timestamp: new Date().toISOString(),
+      details: {
+        sourceType: "memory-unsafe-local",
+        configuredPathCount: config.unsafeLocal.paths.length,
+        artifactCount: artifacts.length,
+        importedCount,
+        updatedCount,
+        skippedCount,
+      },
+    });
+  }
+
+  return {
+    importedCount,
+    updatedCount,
+    skippedCount,
+    artifactCount: artifacts.length,
+    workspaces: 0,
+    pagePaths,
+  };
+}