From a78c4de737c5a20fafdd38dc53cdfc7f9b0ebe1d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 5 Apr 2026 21:04:58 +0100 Subject: [PATCH] feat(memory-wiki): make imported source sync incremental --- extensions/memory-wiki/src/bridge.test.ts | 53 ++++++++ extensions/memory-wiki/src/bridge.ts | 87 +++++++++++-- extensions/memory-wiki/src/cli.ts | 4 +- .../memory-wiki/src/source-sync-state.ts | 123 ++++++++++++++++++ extensions/memory-wiki/src/source-sync.ts | 1 + .../memory-wiki/src/unsafe-local.test.ts | 36 +++++ extensions/memory-wiki/src/unsafe-local.ts | 86 ++++++++++-- 7 files changed, 370 insertions(+), 20 deletions(-) create mode 100644 extensions/memory-wiki/src/source-sync-state.ts diff --git a/extensions/memory-wiki/src/bridge.test.ts b/extensions/memory-wiki/src/bridge.test.ts index 1d4295945ff..ad2268fdcd6 100644 --- a/extensions/memory-wiki/src/bridge.test.ts +++ b/extensions/memory-wiki/src/bridge.test.ts @@ -67,6 +67,7 @@ describe("syncMemoryWikiBridgeSources", () => { expect(first.importedCount).toBe(3); expect(first.updatedCount).toBe(0); expect(first.skippedCount).toBe(0); + expect(first.removedCount).toBe(0); expect(first.pagePaths).toHaveLength(3); const sourcePages = await fs.readdir(path.join(vaultDir, "sources")); @@ -81,6 +82,7 @@ describe("syncMemoryWikiBridgeSources", () => { expect(second.importedCount).toBe(0); expect(second.updatedCount).toBe(0); expect(second.skippedCount).toBe(3); + expect(second.removedCount).toBe(0); const logLines = (await fs.readFile(path.join(vaultDir, ".openclaw-wiki", "log.jsonl"), "utf8")) .trim() @@ -102,6 +104,7 @@ describe("syncMemoryWikiBridgeSources", () => { importedCount: 0, updatedCount: 0, skippedCount: 0, + removedCount: 0, artifactCount: 0, workspaces: 0, pagePaths: [], @@ -157,8 +160,58 @@ describe("syncMemoryWikiBridgeSources", () => { expect(result.artifactCount).toBe(1); expect(result.importedCount).toBe(1); + expect(result.removedCount).toBe(0); const page = await fs.readFile(path.join(vaultDir, result.pagePaths[0] ?? ""), "utf8"); expect(page).toContain("sourceType: memory-bridge-events"); expect(page).toContain('"type":"memory.recall.recorded"'); }); + + it("prunes stale bridge pages when the source artifact disappears", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-bridge-prune-ws-")); + const vaultDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-bridge-prune-vault-")); + tempDirs.push(workspaceDir, vaultDir); + + await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8"); + + const config = resolveMemoryWikiConfig( + { + vaultMode: "bridge", + vault: { path: vaultDir }, + bridge: { + enabled: true, + indexMemoryRoot: true, + indexDailyNotes: false, + indexDreamReports: false, + followMemoryEvents: false, + }, + }, + { homedir: "/Users/tester" }, + ); + const appConfig: OpenClawConfig = { + plugins: { + entries: { + "memory-core": { + enabled: true, + config: {}, + }, + }, + }, + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir }], + }, + }; + + const first = await syncMemoryWikiBridgeSources({ config, appConfig }); + const firstPagePath = first.pagePaths[0] ?? ""; + await expect(fs.stat(path.join(vaultDir, firstPagePath))).resolves.toBeTruthy(); + + await fs.rm(path.join(workspaceDir, "MEMORY.md")); + const second = await syncMemoryWikiBridgeSources({ config, appConfig }); + + expect(second.artifactCount).toBe(0); + expect(second.removedCount).toBe(1); + await expect(fs.stat(path.join(vaultDir, firstPagePath))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); }); diff --git a/extensions/memory-wiki/src/bridge.ts b/extensions/memory-wiki/src/bridge.ts index 435f953dc05..385e90a86b8 100644 --- a/extensions/memory-wiki/src/bridge.ts +++ b/extensions/memory-wiki/src/bridge.ts @@ -10,9 +10,17 @@ import type { OpenClawConfig } from "../api.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { appendMemoryWikiLog } from "./log.js"; import { renderMarkdownFence, renderWikiMarkdown, slugifyWikiSegment } from "./markdown.js"; +import { + pruneImportedSourceEntries, + readMemoryWikiSourceSyncState, + setImportedSourceEntry, + shouldSkipImportedSourceWrite, + writeMemoryWikiSourceSyncState, +} from "./source-sync-state.js"; import { initializeMemoryWikiVault } from "./vault.js"; type BridgeArtifact = { + syncKey: string; artifactType: "markdown" | "memory-events"; workspaceDir: string; relativePath: string; @@ -23,6 +31,7 @@ export type BridgeMemoryWikiResult = { importedCount: number; updatedCount: number; skippedCount: number; + removedCount: number; artifactCount: number; workspaces: number; pagePaths: string[]; @@ -67,7 +76,9 @@ async function collectWorkspaceArtifacts( for (const relPath of ["MEMORY.md", "memory.md"]) { const absolutePath = path.join(workspaceDir, relPath); if (await pathExists(absolutePath)) { + const syncKey = await resolveArtifactKey(absolutePath); artifacts.push({ + syncKey, artifactType: "markdown", workspaceDir, relativePath: relPath, @@ -83,7 +94,9 @@ async function collectWorkspaceArtifacts( for (const absolutePath of files) { const relativePath = path.relative(workspaceDir, absolutePath).replace(/\\/g, "/"); if (!relativePath.startsWith("memory/dreaming/")) { + const syncKey = await resolveArtifactKey(absolutePath); artifacts.push({ + syncKey, artifactType: "markdown", workspaceDir, relativePath, @@ -98,7 +111,9 @@ async function collectWorkspaceArtifacts( const files = await listMarkdownFilesRecursive(dreamingDir); for (const absolutePath of files) { const relativePath = path.relative(workspaceDir, absolutePath).replace(/\\/g, "/"); + const syncKey = await resolveArtifactKey(absolutePath); artifacts.push({ + syncKey, artifactType: "markdown", workspaceDir, relativePath, @@ -110,7 +125,9 @@ async function collectWorkspaceArtifacts( if (bridgeConfig.followMemoryEvents) { const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir); if (await pathExists(eventLogPath)) { + const syncKey = await resolveArtifactKey(eventLogPath); artifacts.push({ + syncKey, artifactType: "memory-events", workspaceDir, relativePath: path.relative(workspaceDir, eventLogPath).replace(/\\/g, "/"), @@ -121,7 +138,7 @@ async function collectWorkspaceArtifacts( const deduped = new Map(); for (const artifact of artifacts) { - deduped.set(await resolveArtifactKey(artifact.absolutePath), artifact); + deduped.set(artifact.syncKey, artifact); } return [...deduped.values()]; } @@ -171,6 +188,9 @@ async function writeBridgeSourcePage(params: { config: ResolvedMemoryWikiConfig; artifact: BridgeArtifact; agentIds: string[]; + sourceUpdatedAtMs: number; + sourceSize: number; + state: Awaited>; }): Promise<{ pagePath: string; changed: boolean; created: boolean }> { const { pageId, pagePath } = resolveBridgePagePath({ workspaceDir: params.artifact.workspaceDir, @@ -179,9 +199,31 @@ async function writeBridgeSourcePage(params: { const title = resolveBridgeTitle(params.artifact, params.agentIds); const pageAbsPath = path.join(params.config.vault.path, pagePath); const created = !(await pathExists(pageAbsPath)); + const sourceUpdatedAt = new Date(params.sourceUpdatedAtMs).toISOString(); + const renderFingerprint = createHash("sha1") + .update( + JSON.stringify({ + artifactType: params.artifact.artifactType, + workspaceDir: params.artifact.workspaceDir, + relativePath: params.artifact.relativePath, + agentIds: params.agentIds, + }), + ) + .digest("hex"); + const shouldSkip = await shouldSkipImportedSourceWrite({ + vaultRoot: params.config.vault.path, + syncKey: params.artifact.syncKey, + expectedPagePath: pagePath, + expectedSourcePath: params.artifact.absolutePath, + sourceUpdatedAtMs: params.sourceUpdatedAtMs, + sourceSize: params.sourceSize, + renderFingerprint, + state: params.state, + }); + if (shouldSkip) { + return { pagePath, changed: false, created }; + } const raw = await fs.readFile(params.artifact.absolutePath, "utf8"); - const stats = await fs.stat(params.artifact.absolutePath); - const sourceUpdatedAt = stats.mtime.toISOString(); const contentLanguage = params.artifact.artifactType === "memory-events" ? "json" : "markdown"; const rendered = renderWikiMarkdown({ frontmatter: { @@ -217,11 +259,22 @@ async function writeBridgeSourcePage(params: { ].join("\n"), }); const existing = await fs.readFile(pageAbsPath, "utf8").catch(() => ""); - if (existing === rendered) { - return { pagePath, changed: false, created }; + if (existing !== rendered) { + await fs.writeFile(pageAbsPath, rendered, "utf8"); } - await fs.writeFile(pageAbsPath, rendered, "utf8"); - return { pagePath, changed: true, created }; + setImportedSourceEntry({ + syncKey: params.artifact.syncKey, + state: params.state, + entry: { + group: "bridge", + pagePath, + sourcePath: params.artifact.absolutePath, + sourceUpdatedAtMs: params.sourceUpdatedAtMs, + sourceSize: params.sourceSize, + renderFingerprint, + }, + }); + return { pagePath, changed: existing !== rendered, created }; } export async function syncMemoryWikiBridgeSources(params: { @@ -239,6 +292,7 @@ export async function syncMemoryWikiBridgeSources(params: { importedCount: 0, updatedCount: 0, skippedCount: 0, + removedCount: 0, artifactCount: 0, workspaces: 0, pagePaths: [], @@ -251,6 +305,7 @@ export async function syncMemoryWikiBridgeSources(params: { importedCount: 0, updatedCount: 0, skippedCount: 0, + removedCount: 0, artifactCount: 0, workspaces: 0, pagePaths: [], @@ -258,22 +313,36 @@ export async function syncMemoryWikiBridgeSources(params: { } const workspaces = resolveMemoryDreamingWorkspaces(params.appConfig); + const state = await readMemoryWikiSourceSyncState(params.config.vault.path); const results: Array<{ pagePath: string; changed: boolean; created: boolean }> = []; let artifactCount = 0; + const activeKeys = new Set(); for (const workspace of workspaces) { const artifacts = await collectWorkspaceArtifacts(workspace.workspaceDir, params.config.bridge); artifactCount += artifacts.length; for (const artifact of artifacts) { + const stats = await fs.stat(artifact.absolutePath); + activeKeys.add(artifact.syncKey); results.push( await writeBridgeSourcePage({ config: params.config, artifact, agentIds: workspace.agentIds, + sourceUpdatedAtMs: stats.mtimeMs, + sourceSize: stats.size, + state, }), ); } } + const removedCount = await pruneImportedSourceEntries({ + vaultRoot: params.config.vault.path, + group: "bridge", + activeKeys, + state, + }); + await writeMemoryWikiSourceSyncState(params.config.vault.path, state); const importedCount = results.filter((result) => result.changed && result.created).length; const updatedCount = results.filter((result) => result.changed && !result.created).length; const skippedCount = results.filter((result) => !result.changed).length; @@ -281,7 +350,7 @@ export async function syncMemoryWikiBridgeSources(params: { .map((result) => result.pagePath) .toSorted((left, right) => left.localeCompare(right)); - if (importedCount > 0 || updatedCount > 0) { + if (importedCount > 0 || updatedCount > 0 || removedCount > 0) { await appendMemoryWikiLog(params.config.vault.path, { type: "ingest", timestamp: new Date().toISOString(), @@ -292,6 +361,7 @@ export async function syncMemoryWikiBridgeSources(params: { importedCount, updatedCount, skippedCount, + removedCount, }, }); } @@ -300,6 +370,7 @@ export async function syncMemoryWikiBridgeSources(params: { importedCount, updatedCount, skippedCount, + removedCount, artifactCount, workspaces: workspaces.length, pagePaths, diff --git a/extensions/memory-wiki/src/cli.ts b/extensions/memory-wiki/src/cli.ts index 940d94320d8..c3c1c2ccd3a 100644 --- a/extensions/memory-wiki/src/cli.ts +++ b/extensions/memory-wiki/src/cli.ts @@ -230,7 +230,7 @@ export async function runWikiBridgeImport(params: { }); const summary = params.json ? JSON.stringify(result, null, 2) - : `Bridge import synced ${result.artifactCount} artifacts across ${result.workspaces} workspaces (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged).`; + : `Bridge import synced ${result.artifactCount} artifacts across ${result.workspaces} workspaces (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged, ${result.removedCount} removed).`; writeOutput(summary, params.stdout); return result; } @@ -247,7 +247,7 @@ export async function runWikiUnsafeLocalImport(params: { }); const summary = params.json ? JSON.stringify(result, null, 2) - : `Unsafe-local import synced ${result.artifactCount} artifacts (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged).`; + : `Unsafe-local import synced ${result.artifactCount} artifacts (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged, ${result.removedCount} removed).`; writeOutput(summary, params.stdout); return result; } diff --git a/extensions/memory-wiki/src/source-sync-state.ts b/extensions/memory-wiki/src/source-sync-state.ts new file mode 100644 index 00000000000..864499bdbb2 --- /dev/null +++ b/extensions/memory-wiki/src/source-sync-state.ts @@ -0,0 +1,123 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +export type MemoryWikiImportedSourceGroup = "bridge" | "unsafe-local"; + +export type MemoryWikiImportedSourceStateEntry = { + group: MemoryWikiImportedSourceGroup; + pagePath: string; + sourcePath: string; + sourceUpdatedAtMs: number; + sourceSize: number; + renderFingerprint: string; +}; + +type MemoryWikiImportedSourceState = { + version: 1; + entries: Record; +}; + +const EMPTY_STATE: MemoryWikiImportedSourceState = { + version: 1, + entries: {}, +}; + +export function resolveMemoryWikiSourceSyncStatePath(vaultRoot: string): string { + return path.join(vaultRoot, ".openclaw-wiki", "source-sync.json"); +} + +export async function readMemoryWikiSourceSyncState( + vaultRoot: string, +): Promise { + const statePath = resolveMemoryWikiSourceSyncStatePath(vaultRoot); + const raw = await fs.readFile(statePath, "utf8").catch((err: unknown) => { + if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { + return ""; + } + throw err; + }); + if (!raw.trim()) { + return { + version: EMPTY_STATE.version, + entries: {}, + }; + } + try { + const parsed = JSON.parse(raw) as Partial; + return { + version: 1, + entries: { ...(parsed.entries ?? {}) }, + }; + } catch { + return { + version: EMPTY_STATE.version, + entries: {}, + }; + } +} + +export async function writeMemoryWikiSourceSyncState( + vaultRoot: string, + state: MemoryWikiImportedSourceState, +): Promise { + const statePath = resolveMemoryWikiSourceSyncStatePath(vaultRoot); + await fs.mkdir(path.dirname(statePath), { recursive: true }); + await fs.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8"); +} + +export async function shouldSkipImportedSourceWrite(params: { + vaultRoot: string; + syncKey: string; + expectedPagePath: string; + expectedSourcePath: string; + sourceUpdatedAtMs: number; + sourceSize: number; + renderFingerprint: string; + state: MemoryWikiImportedSourceState; +}): Promise { + const entry = params.state.entries[params.syncKey]; + if (!entry) { + return false; + } + if ( + entry.pagePath !== params.expectedPagePath || + entry.sourcePath !== params.expectedSourcePath || + entry.sourceUpdatedAtMs !== params.sourceUpdatedAtMs || + entry.sourceSize !== params.sourceSize || + entry.renderFingerprint !== params.renderFingerprint + ) { + return false; + } + const pagePath = path.join(params.vaultRoot, params.expectedPagePath); + return await fs + .access(pagePath) + .then(() => true) + .catch(() => false); +} + +export async function pruneImportedSourceEntries(params: { + vaultRoot: string; + group: MemoryWikiImportedSourceGroup; + activeKeys: Set; + state: MemoryWikiImportedSourceState; +}): Promise { + let removedCount = 0; + for (const [syncKey, entry] of Object.entries(params.state.entries)) { + if (entry.group !== params.group || params.activeKeys.has(syncKey)) { + continue; + } + const pageAbsPath = path.join(params.vaultRoot, entry.pagePath); + await fs.rm(pageAbsPath, { force: true }).catch(() => undefined); + delete params.state.entries[syncKey]; + removedCount += 1; + } + return removedCount; +} + +export function setImportedSourceEntry(params: { + syncKey: string; + entry: MemoryWikiImportedSourceStateEntry; + state: MemoryWikiImportedSourceState; +}): void { + params.state.entries[params.syncKey] = params.entry; +} diff --git a/extensions/memory-wiki/src/source-sync.ts b/extensions/memory-wiki/src/source-sync.ts index a1998ed1d2b..82df64eebc9 100644 --- a/extensions/memory-wiki/src/source-sync.ts +++ b/extensions/memory-wiki/src/source-sync.ts @@ -17,6 +17,7 @@ export async function syncMemoryWikiImportedSources(params: { importedCount: 0, updatedCount: 0, skippedCount: 0, + removedCount: 0, artifactCount: 0, workspaces: 0, pagePaths: [], diff --git a/extensions/memory-wiki/src/unsafe-local.test.ts b/extensions/memory-wiki/src/unsafe-local.test.ts index 8848bf28a6c..5a07d23805c 100644 --- a/extensions/memory-wiki/src/unsafe-local.test.ts +++ b/extensions/memory-wiki/src/unsafe-local.test.ts @@ -42,6 +42,7 @@ describe("syncMemoryWikiUnsafeLocalSources", () => { expect(first.importedCount).toBe(3); expect(first.updatedCount).toBe(0); expect(first.skippedCount).toBe(0); + expect(first.removedCount).toBe(0); const page = await fs.readFile(path.join(vaultDir, first.pagePaths[0] ?? ""), "utf8"); expect(page).toContain("sourceType: memory-unsafe-local"); @@ -52,5 +53,40 @@ describe("syncMemoryWikiUnsafeLocalSources", () => { expect(second.importedCount).toBe(0); expect(second.updatedCount).toBe(0); expect(second.skippedCount).toBe(3); + expect(second.removedCount).toBe(0); + }); + + it("prunes stale unsafe-local pages when configured files disappear", async () => { + const privateDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-private-prune-")); + const vaultDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-unsafe-prune-vault-")); + tempDirs.push(privateDir, vaultDir); + + const secretPath = path.join(privateDir, "secret.md"); + await fs.writeFile(secretPath, "# private\n", "utf8"); + + const config = resolveMemoryWikiConfig( + { + vaultMode: "unsafe-local", + vault: { path: vaultDir }, + unsafeLocal: { + allowPrivateMemoryCoreAccess: true, + paths: [secretPath], + }, + }, + { homedir: "/Users/tester" }, + ); + + const first = await syncMemoryWikiUnsafeLocalSources(config); + const firstPagePath = first.pagePaths[0] ?? ""; + await expect(fs.stat(path.join(vaultDir, firstPagePath))).resolves.toBeTruthy(); + + await fs.rm(secretPath); + const second = await syncMemoryWikiUnsafeLocalSources(config); + + expect(second.artifactCount).toBe(0); + expect(second.removedCount).toBe(1); + await expect(fs.stat(path.join(vaultDir, firstPagePath))).rejects.toMatchObject({ + code: "ENOENT", + }); }); }); diff --git a/extensions/memory-wiki/src/unsafe-local.ts b/extensions/memory-wiki/src/unsafe-local.ts index af95b10d497..863d246fdb3 100644 --- a/extensions/memory-wiki/src/unsafe-local.ts +++ b/extensions/memory-wiki/src/unsafe-local.ts @@ -5,9 +5,17 @@ import type { BridgeMemoryWikiResult } from "./bridge.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { appendMemoryWikiLog } from "./log.js"; import { renderMarkdownFence, renderWikiMarkdown, slugifyWikiSegment } from "./markdown.js"; +import { + pruneImportedSourceEntries, + readMemoryWikiSourceSyncState, + setImportedSourceEntry, + shouldSkipImportedSourceWrite, + writeMemoryWikiSourceSyncState, +} from "./source-sync-state.js"; import { initializeMemoryWikiVault } from "./vault.js"; type UnsafeLocalArtifact = { + syncKey: string; configuredPath: string; absolutePath: string; relativePath: string; @@ -73,6 +81,7 @@ async function collectUnsafeLocalArtifacts( const files = await listAllowedFilesRecursive(absoluteConfiguredPath); for (const absolutePath of files) { artifacts.push({ + syncKey: await resolveArtifactKey(absolutePath), configuredPath: absoluteConfiguredPath, absolutePath, relativePath: path.relative(absoluteConfiguredPath, absolutePath).replace(/\\/g, "/"), @@ -82,6 +91,7 @@ async function collectUnsafeLocalArtifacts( } if (stat.isFile()) { artifacts.push({ + syncKey: await resolveArtifactKey(absoluteConfiguredPath), configuredPath: absoluteConfiguredPath, absolutePath: absoluteConfiguredPath, relativePath: path.basename(absoluteConfiguredPath), @@ -91,7 +101,7 @@ async function collectUnsafeLocalArtifacts( const deduped = new Map(); for (const artifact of artifacts) { - deduped.set(await resolveArtifactKey(artifact.absolutePath), artifact); + deduped.set(artifact.syncKey, artifact); } return [...deduped.values()]; } @@ -124,6 +134,9 @@ function resolveUnsafeLocalTitle(artifact: UnsafeLocalArtifact): string { async function writeUnsafeLocalSourcePage(params: { config: ResolvedMemoryWikiConfig; artifact: UnsafeLocalArtifact; + sourceUpdatedAtMs: number; + sourceSize: number; + state: Awaited>; }): Promise<{ pagePath: string; changed: boolean; created: boolean }> { const { pageId, pagePath } = resolveUnsafeLocalPagePath({ configuredPath: params.artifact.configuredPath, @@ -131,10 +144,30 @@ async function writeUnsafeLocalSourcePage(params: { }); const pageAbsPath = path.join(params.config.vault.path, pagePath); const created = !(await pathExists(pageAbsPath)); - const raw = await fs.readFile(params.artifact.absolutePath, "utf8"); - const stats = await fs.stat(params.artifact.absolutePath); - const updatedAt = stats.mtime.toISOString(); + const updatedAt = new Date(params.sourceUpdatedAtMs).toISOString(); const title = resolveUnsafeLocalTitle(params.artifact); + const renderFingerprint = createHash("sha1") + .update( + JSON.stringify({ + configuredPath: params.artifact.configuredPath, + relativePath: params.artifact.relativePath, + }), + ) + .digest("hex"); + const shouldSkip = await shouldSkipImportedSourceWrite({ + vaultRoot: params.config.vault.path, + syncKey: params.artifact.syncKey, + expectedPagePath: pagePath, + expectedSourcePath: params.artifact.absolutePath, + sourceUpdatedAtMs: params.sourceUpdatedAtMs, + sourceSize: params.sourceSize, + renderFingerprint, + state: params.state, + }); + if (shouldSkip) { + return { pagePath, changed: false, created }; + } + const raw = await fs.readFile(params.artifact.absolutePath, "utf8"); const rendered = renderWikiMarkdown({ frontmatter: { pageType: "source", @@ -166,11 +199,22 @@ async function writeUnsafeLocalSourcePage(params: { ].join("\n"), }); const existing = await fs.readFile(pageAbsPath, "utf8").catch(() => ""); - if (existing === rendered) { - return { pagePath, changed: false, created }; + if (existing !== rendered) { + await fs.writeFile(pageAbsPath, rendered, "utf8"); } - await fs.writeFile(pageAbsPath, rendered, "utf8"); - return { pagePath, changed: true, created }; + setImportedSourceEntry({ + syncKey: params.artifact.syncKey, + state: params.state, + entry: { + group: "unsafe-local", + pagePath, + sourcePath: params.artifact.absolutePath, + sourceUpdatedAtMs: params.sourceUpdatedAtMs, + sourceSize: params.sourceSize, + renderFingerprint, + }, + }); + return { pagePath, changed: existing !== rendered, created }; } export async function syncMemoryWikiUnsafeLocalSources( @@ -186,6 +230,7 @@ export async function syncMemoryWikiUnsafeLocalSources( importedCount: 0, updatedCount: 0, skippedCount: 0, + removedCount: 0, artifactCount: 0, workspaces: 0, pagePaths: [], @@ -193,10 +238,29 @@ export async function syncMemoryWikiUnsafeLocalSources( } const artifacts = await collectUnsafeLocalArtifacts(config.unsafeLocal.paths); + const state = await readMemoryWikiSourceSyncState(config.vault.path); + const activeKeys = new Set(); const results = await Promise.all( - artifacts.map((artifact) => writeUnsafeLocalSourcePage({ config, artifact })), + artifacts.map(async (artifact) => { + const stats = await fs.stat(artifact.absolutePath); + activeKeys.add(artifact.syncKey); + return await writeUnsafeLocalSourcePage({ + config, + artifact, + sourceUpdatedAtMs: stats.mtimeMs, + sourceSize: stats.size, + state, + }); + }), ); + const removedCount = await pruneImportedSourceEntries({ + vaultRoot: config.vault.path, + group: "unsafe-local", + activeKeys, + state, + }); + await writeMemoryWikiSourceSyncState(config.vault.path, state); const importedCount = results.filter((result) => result.changed && result.created).length; const updatedCount = results.filter((result) => result.changed && !result.created).length; const skippedCount = results.filter((result) => !result.changed).length; @@ -204,7 +268,7 @@ export async function syncMemoryWikiUnsafeLocalSources( .map((result) => result.pagePath) .toSorted((left, right) => left.localeCompare(right)); - if (importedCount > 0 || updatedCount > 0) { + if (importedCount > 0 || updatedCount > 0 || removedCount > 0) { await appendMemoryWikiLog(config.vault.path, { type: "ingest", timestamp: new Date().toISOString(), @@ -215,6 +279,7 @@ export async function syncMemoryWikiUnsafeLocalSources( importedCount, updatedCount, skippedCount, + removedCount, }, }); } @@ -223,6 +288,7 @@ export async function syncMemoryWikiUnsafeLocalSources( importedCount, updatedCount, skippedCount, + removedCount, artifactCount: artifacts.length, workspaces: 0, pagePaths,