From 0d3cd4ac42b3a21b2cea088bda934f59e12fe919 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 7 Apr 2026 08:14:49 +0100 Subject: [PATCH] feat(memory-wiki): use digests for retrieval --- CHANGELOG.md | 1 + extensions/memory-wiki/src/query.test.ts | 41 ++++ extensions/memory-wiki/src/query.ts | 264 ++++++++++++++++++++++- 3 files changed, 301 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44437e69b4f..d76f588cb8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Docs: https://docs.openclaw.ai - Plugins/memory: add a public memory-artifact export seam to the unified memory capability so companion plugins like `memory-wiki` can bridge the active memory plugin without reaching into `memory-core` internals. Thanks @vincentkoc. - Memory/wiki: add structured claim/evidence fields plus compiled agent digest artifacts so `memory-wiki` behaves more like a persistent knowledge layer and less like markdown-only page storage. Thanks @vincentkoc. - Memory/wiki: add claim-health linting, contradiction clustering, staleness-aware dashboards, and freshness-weighted wiki search so `memory-wiki` can act more like a maintained belief layer than a passive markdown dump. Thanks @vincentkoc. +- Memory/wiki: use compiled digest artifacts as the first-pass wiki index for search/get flows, and resolve claim ids back to owning pages so agents can retrieve knowledge by belief identity instead of only by file path. Thanks @vincentkoc. ### Fixes diff --git a/extensions/memory-wiki/src/query.test.ts b/extensions/memory-wiki/src/query.test.ts index c03f29c6b67..1b6c2f1a6b5 100644 --- a/extensions/memory-wiki/src/query.test.ts +++ b/extensions/memory-wiki/src/query.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../api.js"; +import { compileMemoryWikiVault } from "./compile.js"; import type { MemoryWikiPluginConfig } from "./config.js"; import { renderWikiMarkdown } from "./markdown.js"; import { getMemoryWikiPage, searchMemoryWiki } from "./query.js"; @@ -449,6 +450,46 @@ describe("getMemoryWikiPage", () => { expect(result?.content).not.toContain("line three"); }); + it("resolves compiled claim ids back to the owning page", async () => { + const { rootDir, config } = await createQueryVault({ + initialize: true, + }); + await fs.writeFile( + path.join(rootDir, "entities", "alpha.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "entity", + id: "entity.alpha", + title: "Alpha", + claims: [ + { + id: "claim.alpha.db", + text: "Alpha uses PostgreSQL for production writes.", + status: "supported", + evidence: [{ sourceId: "source.alpha", lines: "1-2" }], + }, + ], + }, + body: "# Alpha\n\nline one\nline two\n", + }), + "utf8", + ); + await compileMemoryWikiVault(config); + + const result = await getMemoryWikiPage({ + config, + lookup: "claim.alpha.db", + }); + + expect(result).toMatchObject({ + corpus: "wiki", + path: "entities/alpha.md", + title: "Alpha", + id: "entity.alpha", + }); + expect(result?.content).toContain("line one"); + }); + it("returns provenance for imported wiki source pages", async () => { const { rootDir, config } = await createQueryVault({ initialize: true, diff --git a/extensions/memory-wiki/src/query.ts b/extensions/memory-wiki/src/query.ts index 0af109f2f9a..006d608b224 100644 --- a/extensions/memory-wiki/src/query.ts +++ b/extensions/memory-wiki/src/query.ts @@ -15,6 +15,37 @@ import { import { initializeMemoryWikiVault } from "./vault.js"; const QUERY_DIRS = ["entities", "concepts", "sources", "syntheses", "reports"] as const; +const AGENT_DIGEST_PATH = ".openclaw-wiki/cache/agent-digest.json"; +const CLAIMS_DIGEST_PATH = ".openclaw-wiki/cache/claims.jsonl"; + +type QueryDigestPage = { + id?: string; + title: string; + kind: WikiPageSummary["kind"]; + path: string; + sourceIds: string[]; + questions: string[]; + contradictions: string[]; +}; + +type QueryDigestClaim = { + id?: string; + pageId?: string; + pageTitle: string; + pageKind: WikiPageSummary["kind"]; + pagePath: string; + text: string; + status?: string; + confidence?: number; + sourceIds?: string[]; + freshnessLevel?: string; + lastTouchedAt?: string; +}; + +type QueryDigestBundle = { + pages: QueryDigestPage[]; + claims: QueryDigestClaim[]; +}; export type WikiSearchResult = { corpus: "wiki" | "memory"; @@ -79,6 +110,13 @@ async function listWikiMarkdownFiles(rootDir: string): Promise { export async function readQueryableWikiPages(rootDir: string): Promise { const files = await listWikiMarkdownFiles(rootDir); + return readQueryableWikiPagesByPaths(rootDir, files); +} + +async function readQueryableWikiPagesByPaths( + rootDir: string, + files: string[], +): Promise { const pages = await Promise.all( files.map(async (relativePath) => { const absolutePath = path.join(rootDir, relativePath); @@ -90,6 +128,53 @@ export async function readQueryableWikiPages(rootDir: string): Promise (page ? [page] : [])); } +function parseClaimsDigest(raw: string): QueryDigestClaim[] { + return raw.split(/\r?\n/).flatMap((line) => { + const trimmed = line.trim(); + if (!trimmed) { + return []; + } + try { + const parsed = JSON.parse(trimmed) as QueryDigestClaim; + if (!parsed || typeof parsed !== "object" || typeof parsed.pagePath !== "string") { + return []; + } + return [parsed]; + } catch { + return []; + } + }); +} + +async function readQueryDigestBundle(rootDir: string): Promise { + const [agentDigestRaw, claimsDigestRaw] = await Promise.all([ + fs.readFile(path.join(rootDir, AGENT_DIGEST_PATH), "utf8").catch(() => null), + fs.readFile(path.join(rootDir, CLAIMS_DIGEST_PATH), "utf8").catch(() => null), + ]); + if (!agentDigestRaw && !claimsDigestRaw) { + return null; + } + + const pages = (() => { + if (!agentDigestRaw) { + return []; + } + try { + const parsed = JSON.parse(agentDigestRaw) as { pages?: QueryDigestPage[] }; + return Array.isArray(parsed.pages) ? parsed.pages : []; + } catch { + return []; + } + })(); + const claims = claimsDigestRaw ? parseClaimsDigest(claimsDigestRaw) : []; + + if (pages.length === 0 && claims.length === 0) { + return null; + } + + return { pages, claims }; +} + function buildSnippet(raw: string, query: string): string { const queryLower = query.toLowerCase(); const matchingLine = raw @@ -120,6 +205,116 @@ function buildPageSearchText(page: QueryableWikiPage): string { .join("\n"); } +function buildDigestPageSearchText(page: QueryDigestPage, claims: QueryDigestClaim[]): string { + return [ + page.title, + page.path, + page.id ?? "", + page.sourceIds.join(" "), + page.questions.join(" "), + page.contradictions.join(" "), + claims.map((claim) => claim.text).join(" "), + claims.map((claim) => claim.id ?? "").join(" "), + ] + .filter(Boolean) + .join("\n"); +} + +function scoreDigestClaimMatch(claim: QueryDigestClaim, queryLower: string): number { + let score = 0; + if (claim.text.toLowerCase().includes(queryLower)) { + score += 25; + } + if (claim.id?.toLowerCase().includes(queryLower)) { + score += 10; + } + if (typeof claim.confidence === "number") { + score += Math.round(claim.confidence * 10); + } + switch (claim.freshnessLevel) { + case "fresh": + score += 8; + break; + case "aging": + score += 4; + break; + case "stale": + score -= 2; + break; + case "unknown": + score -= 4; + break; + } + score += isClaimContestedStatus(claim.status) ? -6 : 4; + return score; +} + +function buildDigestCandidatePaths(params: { + digest: QueryDigestBundle; + query: string; + maxResults: number; +}): string[] { + const queryLower = params.query.toLowerCase(); + const claimsByPage = new Map(); + for (const claim of params.digest.claims) { + const current = claimsByPage.get(claim.pagePath) ?? []; + current.push(claim); + claimsByPage.set(claim.pagePath, current); + } + + return params.digest.pages + .map((page) => { + const claims = claimsByPage.get(page.path) ?? []; + const metadataLower = buildDigestPageSearchText(page, claims).toLowerCase(); + if (!metadataLower.includes(queryLower)) { + return { path: page.path, score: 0 }; + } + let score = 1; + const titleLower = page.title.toLowerCase(); + const pathLower = page.path.toLowerCase(); + const idLower = page.id?.toLowerCase() ?? ""; + if (titleLower === queryLower) { + score += 50; + } else if (titleLower.includes(queryLower)) { + score += 20; + } + if (pathLower.includes(queryLower)) { + score += 10; + } + if (idLower.includes(queryLower)) { + score += 20; + } + if (page.sourceIds.some((sourceId) => sourceId.toLowerCase().includes(queryLower))) { + score += 12; + } + const matchingClaims = claims + .filter((claim) => { + if (claim.text.toLowerCase().includes(queryLower)) { + return true; + } + return claim.id?.toLowerCase().includes(queryLower) ?? false; + }) + .toSorted( + (left, right) => + scoreDigestClaimMatch(right, queryLower) - scoreDigestClaimMatch(left, queryLower), + ); + if (matchingClaims.length > 0) { + score += scoreDigestClaimMatch(matchingClaims[0], queryLower); + score += Math.min(10, (matchingClaims.length - 1) * 2); + } + return { path: page.path, score }; + }) + .filter((candidate) => candidate.score > 0) + .toSorted((left, right) => { + if (left.score !== right.score) { + return right.score - left.score; + } + return left.path.localeCompare(right.path); + }) + .slice(0, Math.max(params.maxResults * 4, 20)) + .map((candidate) => candidate.path); +} + function isClaimMatch(claim: WikiClaim, queryLower: string): boolean { if (claim.text.toLowerCase().includes(queryLower)) { return true; @@ -360,6 +555,54 @@ function toMemoryWikiSearchResult(result: MemorySearchResult): WikiSearchResult }; } +async function searchWikiCorpus(params: { + rootDir: string; + query: string; + maxResults: number; +}): Promise { + const digest = await readQueryDigestBundle(params.rootDir); + const candidatePaths = digest + ? buildDigestCandidatePaths({ + digest, + query: params.query, + maxResults: params.maxResults, + }) + : []; + const seenPaths = new Set(); + const candidatePages = + candidatePaths.length > 0 + ? await readQueryableWikiPagesByPaths(params.rootDir, candidatePaths) + : await readQueryableWikiPages(params.rootDir); + for (const page of candidatePages) { + seenPaths.add(page.relativePath); + } + + const results = candidatePages + .map((page) => toWikiSearchResult(page, params.query)) + .filter((page) => page.score > 0); + if (candidatePaths.length === 0 || results.length >= params.maxResults) { + return results; + } + + const remainingPaths = (await listWikiMarkdownFiles(params.rootDir)).filter( + (relativePath) => !seenPaths.has(relativePath), + ); + const remainingPages = await readQueryableWikiPagesByPaths(params.rootDir, remainingPaths); + return [ + ...results, + ...remainingPages + .map((page) => toWikiSearchResult(page, params.query)) + .filter((page) => page.score > 0), + ]; +} + +function resolveDigestClaimLookup(digest: QueryDigestBundle, lookup: string): string | null { + const trimmed = lookup.trim(); + const claimId = trimmed.replace(/^claim:/i, ""); + const match = digest.claims.find((claim) => claim.id === claimId); + return match?.pagePath ?? null; +} + export function resolveQueryableWikiPageByLookup( pages: QueryableWikiPage[], lookup: string, @@ -391,9 +634,11 @@ export async function searchMemoryWiki(params: { const maxResults = Math.max(1, params.maxResults ?? 10); const wikiResults = shouldSearchWiki(effectiveConfig) - ? (await readQueryableWikiPages(effectiveConfig.vault.path)) - .map((page) => toWikiSearchResult(page, params.query)) - .filter((page) => page.score > 0) + ? await searchWikiCorpus({ + rootDir: effectiveConfig.vault.path, + query: params.query, + maxResults, + }) : []; const sharedMemoryManager = shouldSearchSharedMemory(effectiveConfig, params.appConfig) @@ -436,8 +681,17 @@ export async function getMemoryWikiPage(params: { const lineCount = Math.max(1, params.lineCount ?? 200); if (shouldSearchWiki(effectiveConfig)) { - const pages = await readQueryableWikiPages(effectiveConfig.vault.path); - const page = resolveQueryableWikiPageByLookup(pages, params.lookup); + const digest = await readQueryDigestBundle(effectiveConfig.vault.path); + const digestClaimPagePath = digest ? resolveDigestClaimLookup(digest, params.lookup) : null; + const digestLookupPage = digestClaimPagePath + ? (( + await readQueryableWikiPagesByPaths(effectiveConfig.vault.path, [digestClaimPagePath]) + )[0] ?? null) + : null; + const pages = digestLookupPage + ? [digestLookupPage] + : await readQueryableWikiPages(effectiveConfig.vault.path); + const page = digestLookupPage ?? resolveQueryableWikiPageByLookup(pages, params.lookup); if (page) { const parsed = parseWikiMarkdown(page.raw); const lines = parsed.body.split(/\r?\n/);