vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-12 09:41:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(memory-wiki): make imported source sync incremental

Browse Source
This commit is contained in:
Vincent Koc
2026-04-05 21:04:58 +01:00
parent 7b62fcd87d
commit a78c4de737
7 changed files with 370 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
extensions/memory-wiki/src/bridge.test.ts
View File

@@ -67,6 +67,7 @@ describe("syncMemoryWikiBridgeSources", () => {
expect(first.importedCount).toBe(3);
expect(first.updatedCount).toBe(0);
expect(first.skippedCount).toBe(0);
expect(first.removedCount).toBe(0);
expect(first.pagePaths).toHaveLength(3);
const sourcePages = await fs.readdir(path.join(vaultDir, "sources"));
@@ -81,6 +82,7 @@ describe("syncMemoryWikiBridgeSources", () => {
expect(second.importedCount).toBe(0);
expect(second.updatedCount).toBe(0);
expect(second.skippedCount).toBe(3);
expect(second.removedCount).toBe(0);
const logLines = (await fs.readFile(path.join(vaultDir, ".openclaw-wiki", "log.jsonl"), "utf8"))
.trim()
@@ -102,6 +104,7 @@ describe("syncMemoryWikiBridgeSources", () => {
importedCount: 0,
updatedCount: 0,
skippedCount: 0,
removedCount: 0,
artifactCount: 0,
workspaces: 0,
pagePaths: [],
@@ -157,8 +160,58 @@ describe("syncMemoryWikiBridgeSources", () => {
expect(result.artifactCount).toBe(1);
expect(result.importedCount).toBe(1);
expect(result.removedCount).toBe(0);
const page = await fs.readFile(path.join(vaultDir, result.pagePaths[0] ?? ""), "utf8");
expect(page).toContain("sourceType: memory-bridge-events");
expect(page).toContain('"type":"memory.recall.recorded"');
});
it("prunes stale bridge pages when the source artifact disappears", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-bridge-prune-ws-"));
const vaultDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-bridge-prune-vault-"));
tempDirs.push(workspaceDir, vaultDir);
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8");
const config = resolveMemoryWikiConfig(
{
vaultMode: "bridge",
vault: { path: vaultDir },
bridge: {
enabled: true,
indexMemoryRoot: true,
indexDailyNotes: false,
indexDreamReports: false,
followMemoryEvents: false,
},
},
{ homedir: "/Users/tester" },
);
const appConfig: OpenClawConfig = {
plugins: {
entries: {
"memory-core": {
enabled: true,
config: {},
},
},
},
agents: {
list: [{ id: "main", default: true, workspace: workspaceDir }],
},
};
const first = await syncMemoryWikiBridgeSources({ config, appConfig });
const firstPagePath = first.pagePaths[0] ?? "";
await expect(fs.stat(path.join(vaultDir, firstPagePath))).resolves.toBeTruthy();
await fs.rm(path.join(workspaceDir, "MEMORY.md"));
const second = await syncMemoryWikiBridgeSources({ config, appConfig });
expect(second.artifactCount).toBe(0);
expect(second.removedCount).toBe(1);
await expect(fs.stat(path.join(vaultDir, firstPagePath))).rejects.toMatchObject({
code: "ENOENT",
});
});
});

87
extensions/memory-wiki/src/bridge.ts
View File

@@ -10,9 +10,17 @@ import type { OpenClawConfig } from "../api.js";
import type { ResolvedMemoryWikiConfig } from "./config.js";
import { appendMemoryWikiLog } from "./log.js";
import { renderMarkdownFence, renderWikiMarkdown, slugifyWikiSegment } from "./markdown.js";
import {
pruneImportedSourceEntries,
readMemoryWikiSourceSyncState,
setImportedSourceEntry,
shouldSkipImportedSourceWrite,
writeMemoryWikiSourceSyncState,
} from "./source-sync-state.js";
import { initializeMemoryWikiVault } from "./vault.js";
type BridgeArtifact = {
syncKey: string;
artifactType: "markdown" | "memory-events";
workspaceDir: string;
relativePath: string;
@@ -23,6 +31,7 @@ export type BridgeMemoryWikiResult = {
importedCount: number;
updatedCount: number;
skippedCount: number;
removedCount: number;
artifactCount: number;
workspaces: number;
pagePaths: string[];
@@ -67,7 +76,9 @@ async function collectWorkspaceArtifacts(
for (const relPath of ["MEMORY.md", "memory.md"]) {
const absolutePath = path.join(workspaceDir, relPath);
if (await pathExists(absolutePath)) {
const syncKey = await resolveArtifactKey(absolutePath);
artifacts.push({
syncKey,
artifactType: "markdown",
workspaceDir,
relativePath: relPath,
@@ -83,7 +94,9 @@ async function collectWorkspaceArtifacts(
for (const absolutePath of files) {
const relativePath = path.relative(workspaceDir, absolutePath).replace(/\\/g, "/");
if (!relativePath.startsWith("memory/dreaming/")) {
const syncKey = await resolveArtifactKey(absolutePath);
artifacts.push({
syncKey,
artifactType: "markdown",
workspaceDir,
relativePath,
@@ -98,7 +111,9 @@ async function collectWorkspaceArtifacts(
const files = await listMarkdownFilesRecursive(dreamingDir);
for (const absolutePath of files) {
const relativePath = path.relative(workspaceDir, absolutePath).replace(/\\/g, "/");
const syncKey = await resolveArtifactKey(absolutePath);
artifacts.push({
syncKey,
artifactType: "markdown",
workspaceDir,
relativePath,
@@ -110,7 +125,9 @@ async function collectWorkspaceArtifacts(
if (bridgeConfig.followMemoryEvents) {
const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir);
if (await pathExists(eventLogPath)) {
const syncKey = await resolveArtifactKey(eventLogPath);
artifacts.push({
syncKey,
artifactType: "memory-events",
workspaceDir,
relativePath: path.relative(workspaceDir, eventLogPath).replace(/\\/g, "/"),
@@ -121,7 +138,7 @@ async function collectWorkspaceArtifacts(
const deduped = new Map<string, BridgeArtifact>();
for (const artifact of artifacts) {
deduped.set(await resolveArtifactKey(artifact.absolutePath), artifact);
deduped.set(artifact.syncKey, artifact);
}
return [...deduped.values()];
}
@@ -171,6 +188,9 @@ async function writeBridgeSourcePage(params: {
config: ResolvedMemoryWikiConfig;
artifact: BridgeArtifact;
agentIds: string[];
sourceUpdatedAtMs: number;
sourceSize: number;
state: Awaited<ReturnType<typeof readMemoryWikiSourceSyncState>>;
}): Promise<{ pagePath: string; changed: boolean; created: boolean }> {
const { pageId, pagePath } = resolveBridgePagePath({
workspaceDir: params.artifact.workspaceDir,
@@ -179,9 +199,31 @@ async function writeBridgeSourcePage(params: {
const title = resolveBridgeTitle(params.artifact, params.agentIds);
const pageAbsPath = path.join(params.config.vault.path, pagePath);
const created = !(await pathExists(pageAbsPath));
const sourceUpdatedAt = new Date(params.sourceUpdatedAtMs).toISOString();
const renderFingerprint = createHash("sha1")
.update(
JSON.stringify({
artifactType: params.artifact.artifactType,
workspaceDir: params.artifact.workspaceDir,
relativePath: params.artifact.relativePath,
agentIds: params.agentIds,
}),
)
.digest("hex");
const shouldSkip = await shouldSkipImportedSourceWrite({
vaultRoot: params.config.vault.path,
syncKey: params.artifact.syncKey,
expectedPagePath: pagePath,
expectedSourcePath: params.artifact.absolutePath,
sourceUpdatedAtMs: params.sourceUpdatedAtMs,
sourceSize: params.sourceSize,
renderFingerprint,
state: params.state,
});
if (shouldSkip) {
return { pagePath, changed: false, created };
}
const raw = await fs.readFile(params.artifact.absolutePath, "utf8");
const stats = await fs.stat(params.artifact.absolutePath);
const sourceUpdatedAt = stats.mtime.toISOString();
const contentLanguage = params.artifact.artifactType === "memory-events" ? "json" : "markdown";
const rendered = renderWikiMarkdown({
frontmatter: {
@@ -217,11 +259,22 @@ async function writeBridgeSourcePage(params: {
].join("\n"),
});
const existing = await fs.readFile(pageAbsPath, "utf8").catch(() => "");
if (existing === rendered) {
return { pagePath, changed: false, created };
if (existing !== rendered) {
await fs.writeFile(pageAbsPath, rendered, "utf8");
}
await fs.writeFile(pageAbsPath, rendered, "utf8");
return { pagePath, changed: true, created };
setImportedSourceEntry({
syncKey: params.artifact.syncKey,
state: params.state,
entry: {
group: "bridge",
pagePath,
sourcePath: params.artifact.absolutePath,
sourceUpdatedAtMs: params.sourceUpdatedAtMs,
sourceSize: params.sourceSize,
renderFingerprint,
},
});
return { pagePath, changed: existing !== rendered, created };
}
export async function syncMemoryWikiBridgeSources(params: {
@@ -239,6 +292,7 @@ export async function syncMemoryWikiBridgeSources(params: {
importedCount: 0,
updatedCount: 0,
skippedCount: 0,
removedCount: 0,
artifactCount: 0,
workspaces: 0,
pagePaths: [],
@@ -251,6 +305,7 @@ export async function syncMemoryWikiBridgeSources(params: {
importedCount: 0,
updatedCount: 0,
skippedCount: 0,
removedCount: 0,
artifactCount: 0,
workspaces: 0,
pagePaths: [],
@@ -258,22 +313,36 @@ export async function syncMemoryWikiBridgeSources(params: {
}
const workspaces = resolveMemoryDreamingWorkspaces(params.appConfig);
const state = await readMemoryWikiSourceSyncState(params.config.vault.path);
const results: Array<{ pagePath: string; changed: boolean; created: boolean }> = [];
let artifactCount = 0;
const activeKeys = new Set<string>();
for (const workspace of workspaces) {
const artifacts = await collectWorkspaceArtifacts(workspace.workspaceDir, params.config.bridge);
artifactCount += artifacts.length;
for (const artifact of artifacts) {
const stats = await fs.stat(artifact.absolutePath);
activeKeys.add(artifact.syncKey);
results.push(
await writeBridgeSourcePage({
config: params.config,
artifact,
agentIds: workspace.agentIds,
sourceUpdatedAtMs: stats.mtimeMs,
sourceSize: stats.size,
state,
}),
);
}
}
const removedCount = await pruneImportedSourceEntries({
vaultRoot: params.config.vault.path,
group: "bridge",
activeKeys,
state,
});
await writeMemoryWikiSourceSyncState(params.config.vault.path, state);
const importedCount = results.filter((result) => result.changed && result.created).length;
const updatedCount = results.filter((result) => result.changed && !result.created).length;
const skippedCount = results.filter((result) => !result.changed).length;
@@ -281,7 +350,7 @@ export async function syncMemoryWikiBridgeSources(params: {
.map((result) => result.pagePath)
.toSorted((left, right) => left.localeCompare(right));
if (importedCount > 0 || updatedCount > 0) {
if (importedCount > 0 || updatedCount > 0 || removedCount > 0) {
await appendMemoryWikiLog(params.config.vault.path, {
type: "ingest",
timestamp: new Date().toISOString(),
@@ -292,6 +361,7 @@ export async function syncMemoryWikiBridgeSources(params: {
importedCount,
updatedCount,
skippedCount,
removedCount,
},
});
}
@@ -300,6 +370,7 @@ export async function syncMemoryWikiBridgeSources(params: {
importedCount,
updatedCount,
skippedCount,
removedCount,
artifactCount,
workspaces: workspaces.length,
pagePaths,

4
extensions/memory-wiki/src/cli.ts
View File

@@ -230,7 +230,7 @@ export async function runWikiBridgeImport(params: {
});
const summary = params.json
? JSON.stringify(result, null, 2)
: `Bridge import synced ${result.artifactCount} artifacts across ${result.workspaces} workspaces (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged).`;
: `Bridge import synced ${result.artifactCount} artifacts across ${result.workspaces} workspaces (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged, ${result.removedCount} removed).`;
writeOutput(summary, params.stdout);
return result;
}
@@ -247,7 +247,7 @@ export async function runWikiUnsafeLocalImport(params: {
});
const summary = params.json
? JSON.stringify(result, null, 2)
: `Unsafe-local import synced ${result.artifactCount} artifacts (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged).`;
: `Unsafe-local import synced ${result.artifactCount} artifacts (${result.importedCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged, ${result.removedCount} removed).`;
writeOutput(summary, params.stdout);
return result;
}

123
extensions/memory-wiki/src/source-sync-state.ts Normal file
View File

@@ -0,0 +1,123 @@
import fs from "node:fs/promises";
import path from "node:path";
export type MemoryWikiImportedSourceGroup = "bridge" | "unsafe-local";
export type MemoryWikiImportedSourceStateEntry = {
group: MemoryWikiImportedSourceGroup;
pagePath: string;
sourcePath: string;
sourceUpdatedAtMs: number;
sourceSize: number;
renderFingerprint: string;
};
type MemoryWikiImportedSourceState = {
version: 1;
entries: Record<string, MemoryWikiImportedSourceStateEntry>;
};
const EMPTY_STATE: MemoryWikiImportedSourceState = {
version: 1,
entries: {},
};
export function resolveMemoryWikiSourceSyncStatePath(vaultRoot: string): string {
return path.join(vaultRoot, ".openclaw-wiki", "source-sync.json");
}
export async function readMemoryWikiSourceSyncState(
vaultRoot: string,
): Promise<MemoryWikiImportedSourceState> {
const statePath = resolveMemoryWikiSourceSyncStatePath(vaultRoot);
const raw = await fs.readFile(statePath, "utf8").catch((err: unknown) => {
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
return "";
}
throw err;
});
if (!raw.trim()) {
return {
version: EMPTY_STATE.version,
entries: {},
};
}
try {
const parsed = JSON.parse(raw) as Partial<MemoryWikiImportedSourceState>;
return {
version: 1,
entries: { ...(parsed.entries ?? {}) },
};
} catch {
return {
version: EMPTY_STATE.version,
entries: {},
};
}
}
export async function writeMemoryWikiSourceSyncState(
vaultRoot: string,
state: MemoryWikiImportedSourceState,
): Promise<void> {
const statePath = resolveMemoryWikiSourceSyncStatePath(vaultRoot);
await fs.mkdir(path.dirname(statePath), { recursive: true });
await fs.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8");
}
export async function shouldSkipImportedSourceWrite(params: {
vaultRoot: string;
syncKey: string;
expectedPagePath: string;
expectedSourcePath: string;
sourceUpdatedAtMs: number;
sourceSize: number;
renderFingerprint: string;
state: MemoryWikiImportedSourceState;
}): Promise<boolean> {
const entry = params.state.entries[params.syncKey];
if (!entry) {
return false;
}
if (
entry.pagePath !== params.expectedPagePath ||
entry.sourcePath !== params.expectedSourcePath ||
entry.sourceUpdatedAtMs !== params.sourceUpdatedAtMs ||
entry.sourceSize !== params.sourceSize ||
entry.renderFingerprint !== params.renderFingerprint
) {
return false;
}
const pagePath = path.join(params.vaultRoot, params.expectedPagePath);
return await fs
.access(pagePath)
.then(() => true)
.catch(() => false);
}
export async function pruneImportedSourceEntries(params: {
vaultRoot: string;
group: MemoryWikiImportedSourceGroup;
activeKeys: Set<string>;
state: MemoryWikiImportedSourceState;
}): Promise<number> {
let removedCount = 0;
for (const [syncKey, entry] of Object.entries(params.state.entries)) {
if (entry.group !== params.group || params.activeKeys.has(syncKey)) {
continue;
}
const pageAbsPath = path.join(params.vaultRoot, entry.pagePath);
await fs.rm(pageAbsPath, { force: true }).catch(() => undefined);
delete params.state.entries[syncKey];
removedCount += 1;
}
return removedCount;
}
export function setImportedSourceEntry(params: {
syncKey: string;
entry: MemoryWikiImportedSourceStateEntry;
state: MemoryWikiImportedSourceState;
}): void {
params.state.entries[params.syncKey] = params.entry;
}

1
extensions/memory-wiki/src/source-sync.ts
View File

@@ -17,6 +17,7 @@ export async function syncMemoryWikiImportedSources(params: {
importedCount: 0,
updatedCount: 0,
skippedCount: 0,
removedCount: 0,
artifactCount: 0,
workspaces: 0,
pagePaths: [],

36
extensions/memory-wiki/src/unsafe-local.test.ts
View File

@@ -42,6 +42,7 @@ describe("syncMemoryWikiUnsafeLocalSources", () => {
expect(first.importedCount).toBe(3);
expect(first.updatedCount).toBe(0);
expect(first.skippedCount).toBe(0);
expect(first.removedCount).toBe(0);
const page = await fs.readFile(path.join(vaultDir, first.pagePaths[0] ?? ""), "utf8");
expect(page).toContain("sourceType: memory-unsafe-local");
@@ -52,5 +53,40 @@ describe("syncMemoryWikiUnsafeLocalSources", () => {
expect(second.importedCount).toBe(0);
expect(second.updatedCount).toBe(0);
expect(second.skippedCount).toBe(3);
expect(second.removedCount).toBe(0);
});
it("prunes stale unsafe-local pages when configured files disappear", async () => {
const privateDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-private-prune-"));
const vaultDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-unsafe-prune-vault-"));
tempDirs.push(privateDir, vaultDir);
const secretPath = path.join(privateDir, "secret.md");
await fs.writeFile(secretPath, "# private\n", "utf8");
const config = resolveMemoryWikiConfig(
{
vaultMode: "unsafe-local",
vault: { path: vaultDir },
unsafeLocal: {
allowPrivateMemoryCoreAccess: true,
paths: [secretPath],
},
},
{ homedir: "/Users/tester" },
);
const first = await syncMemoryWikiUnsafeLocalSources(config);
const firstPagePath = first.pagePaths[0] ?? "";
await expect(fs.stat(path.join(vaultDir, firstPagePath))).resolves.toBeTruthy();
await fs.rm(secretPath);
const second = await syncMemoryWikiUnsafeLocalSources(config);
expect(second.artifactCount).toBe(0);
expect(second.removedCount).toBe(1);
await expect(fs.stat(path.join(vaultDir, firstPagePath))).rejects.toMatchObject({
code: "ENOENT",
});
});
});

86
extensions/memory-wiki/src/unsafe-local.ts
View File

@@ -5,9 +5,17 @@ import type { BridgeMemoryWikiResult } from "./bridge.js";
import type { ResolvedMemoryWikiConfig } from "./config.js";
import { appendMemoryWikiLog } from "./log.js";
import { renderMarkdownFence, renderWikiMarkdown, slugifyWikiSegment } from "./markdown.js";
import {
pruneImportedSourceEntries,
readMemoryWikiSourceSyncState,
setImportedSourceEntry,
shouldSkipImportedSourceWrite,
writeMemoryWikiSourceSyncState,
} from "./source-sync-state.js";
import { initializeMemoryWikiVault } from "./vault.js";
type UnsafeLocalArtifact = {
syncKey: string;
configuredPath: string;
absolutePath: string;
relativePath: string;
@@ -73,6 +81,7 @@ async function collectUnsafeLocalArtifacts(
const files = await listAllowedFilesRecursive(absoluteConfiguredPath);
for (const absolutePath of files) {
artifacts.push({
syncKey: await resolveArtifactKey(absolutePath),
configuredPath: absoluteConfiguredPath,
absolutePath,
relativePath: path.relative(absoluteConfiguredPath, absolutePath).replace(/\\/g, "/"),
@@ -82,6 +91,7 @@ async function collectUnsafeLocalArtifacts(
}
if (stat.isFile()) {
artifacts.push({
syncKey: await resolveArtifactKey(absoluteConfiguredPath),
configuredPath: absoluteConfiguredPath,
absolutePath: absoluteConfiguredPath,
relativePath: path.basename(absoluteConfiguredPath),
@@ -91,7 +101,7 @@ async function collectUnsafeLocalArtifacts(
const deduped = new Map<string, UnsafeLocalArtifact>();
for (const artifact of artifacts) {
deduped.set(await resolveArtifactKey(artifact.absolutePath), artifact);
deduped.set(artifact.syncKey, artifact);
}
return [...deduped.values()];
}
@@ -124,6 +134,9 @@ function resolveUnsafeLocalTitle(artifact: UnsafeLocalArtifact): string {
async function writeUnsafeLocalSourcePage(params: {
config: ResolvedMemoryWikiConfig;
artifact: UnsafeLocalArtifact;
sourceUpdatedAtMs: number;
sourceSize: number;
state: Awaited<ReturnType<typeof readMemoryWikiSourceSyncState>>;
}): Promise<{ pagePath: string; changed: boolean; created: boolean }> {
const { pageId, pagePath } = resolveUnsafeLocalPagePath({
configuredPath: params.artifact.configuredPath,
@@ -131,10 +144,30 @@ async function writeUnsafeLocalSourcePage(params: {
});
const pageAbsPath = path.join(params.config.vault.path, pagePath);
const created = !(await pathExists(pageAbsPath));
const raw = await fs.readFile(params.artifact.absolutePath, "utf8");
const stats = await fs.stat(params.artifact.absolutePath);
const updatedAt = stats.mtime.toISOString();
const updatedAt = new Date(params.sourceUpdatedAtMs).toISOString();
const title = resolveUnsafeLocalTitle(params.artifact);
const renderFingerprint = createHash("sha1")
.update(
JSON.stringify({
configuredPath: params.artifact.configuredPath,
relativePath: params.artifact.relativePath,
}),
)
.digest("hex");
const shouldSkip = await shouldSkipImportedSourceWrite({
vaultRoot: params.config.vault.path,
syncKey: params.artifact.syncKey,
expectedPagePath: pagePath,
expectedSourcePath: params.artifact.absolutePath,
sourceUpdatedAtMs: params.sourceUpdatedAtMs,
sourceSize: params.sourceSize,
renderFingerprint,
state: params.state,
});
if (shouldSkip) {
return { pagePath, changed: false, created };
}
const raw = await fs.readFile(params.artifact.absolutePath, "utf8");
const rendered = renderWikiMarkdown({
frontmatter: {
pageType: "source",
@@ -166,11 +199,22 @@ async function writeUnsafeLocalSourcePage(params: {
].join("\n"),
});
const existing = await fs.readFile(pageAbsPath, "utf8").catch(() => "");
if (existing === rendered) {
return { pagePath, changed: false, created };
if (existing !== rendered) {
await fs.writeFile(pageAbsPath, rendered, "utf8");
}
await fs.writeFile(pageAbsPath, rendered, "utf8");
return { pagePath, changed: true, created };
setImportedSourceEntry({
syncKey: params.artifact.syncKey,
state: params.state,
entry: {
group: "unsafe-local",
pagePath,
sourcePath: params.artifact.absolutePath,
sourceUpdatedAtMs: params.sourceUpdatedAtMs,
sourceSize: params.sourceSize,
renderFingerprint,
},
});
return { pagePath, changed: existing !== rendered, created };
}
export async function syncMemoryWikiUnsafeLocalSources(
@@ -186,6 +230,7 @@ export async function syncMemoryWikiUnsafeLocalSources(
importedCount: 0,
updatedCount: 0,
skippedCount: 0,
removedCount: 0,
artifactCount: 0,
workspaces: 0,
pagePaths: [],
@@ -193,10 +238,29 @@ export async function syncMemoryWikiUnsafeLocalSources(
}
const artifacts = await collectUnsafeLocalArtifacts(config.unsafeLocal.paths);
const state = await readMemoryWikiSourceSyncState(config.vault.path);
const activeKeys = new Set<string>();
const results = await Promise.all(
artifacts.map((artifact) => writeUnsafeLocalSourcePage({ config, artifact })),
artifacts.map(async (artifact) => {
const stats = await fs.stat(artifact.absolutePath);
activeKeys.add(artifact.syncKey);
return await writeUnsafeLocalSourcePage({
config,
artifact,
sourceUpdatedAtMs: stats.mtimeMs,
sourceSize: stats.size,
state,
});
}),
);
const removedCount = await pruneImportedSourceEntries({
vaultRoot: config.vault.path,
group: "unsafe-local",
activeKeys,
state,
});
await writeMemoryWikiSourceSyncState(config.vault.path, state);
const importedCount = results.filter((result) => result.changed && result.created).length;
const updatedCount = results.filter((result) => result.changed && !result.created).length;
const skippedCount = results.filter((result) => !result.changed).length;
@@ -204,7 +268,7 @@ export async function syncMemoryWikiUnsafeLocalSources(
.map((result) => result.pagePath)
.toSorted((left, right) => left.localeCompare(right));
if (importedCount > 0 || updatedCount > 0) {
if (importedCount > 0 || updatedCount > 0 || removedCount > 0) {
await appendMemoryWikiLog(config.vault.path, {
type: "ingest",
timestamp: new Date().toISOString(),
@@ -215,6 +279,7 @@ export async function syncMemoryWikiUnsafeLocalSources(
importedCount,
updatedCount,
skippedCount,
removedCount,
},
});
}
@@ -223,6 +288,7 @@ export async function syncMemoryWikiUnsafeLocalSources(
importedCount,
updatedCount,
skippedCount,
removedCount,
artifactCount: artifacts.length,
workspaces: 0,
pagePaths,