From 64693d2e96ab8b3e3e998fd9b256ea04d435bcc6 Mon Sep 17 00:00:00 2001 From: Mariano Date: Sat, 11 Apr 2026 07:04:08 +0200 Subject: [PATCH] [codex] Dreaming: surface memory wiki imports and palace (#64505) Merged via squash. Prepared head SHA: 12d5e372229dbb841762a03c9ef6cfa787fc58c0 Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com> Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com> Reviewed-by: @mbelinky --- CHANGELOG.md | 1 + extensions/memory-wiki/src/chatgpt-import.ts | 903 ++++++++++++++++++ extensions/memory-wiki/src/cli.test.ts | 96 +- extensions/memory-wiki/src/cli.ts | 99 ++ extensions/memory-wiki/src/gateway.test.ts | 197 ++++ extensions/memory-wiki/src/gateway.ts | 42 + .../memory-wiki/src/import-insights.test.ts | 142 +++ extensions/memory-wiki/src/import-insights.ts | 438 +++++++++ extensions/memory-wiki/src/import-runs.ts | 138 +++ .../memory-wiki/src/memory-palace.test.ts | 91 ++ extensions/memory-wiki/src/memory-palace.ts | 148 +++ extensions/memory-wiki/src/query.test.ts | 2 + extensions/memory-wiki/src/query.ts | 6 + ui/src/styles/dreams.css | 241 ++++- ui/src/ui/app-render.ts | 60 +- ui/src/ui/app-settings.test.ts | 6 + ui/src/ui/app-settings.ts | 15 +- ui/src/ui/app-view-state.ts | 6 + ui/src/ui/app.ts | 12 +- ui/src/ui/controllers/dreaming.test.ts | 126 +++ ui/src/ui/controllers/dreaming.ts | 356 +++++++ ui/src/ui/views/dreaming.test.ts | 206 +++- ui/src/ui/views/dreaming.ts | 751 +++++++++++++-- 23 files changed, 4002 insertions(+), 80 deletions(-) create mode 100644 extensions/memory-wiki/src/chatgpt-import.ts create mode 100644 extensions/memory-wiki/src/import-insights.test.ts create mode 100644 extensions/memory-wiki/src/import-insights.ts create mode 100644 extensions/memory-wiki/src/import-runs.ts create mode 100644 extensions/memory-wiki/src/memory-palace.test.ts create mode 100644 extensions/memory-wiki/src/memory-palace.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e83cb2554bb..00aa72ae0f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Docs: https://docs.openclaw.ai - Agents: add an opt-in strict-agentic embedded Pi execution contract for GPT-5-family runs so plan-only or filler turns keep acting until they hit a real blocker. (#64241) Thanks @100yenadmin. - Agents/OpenAI: add provider-owned OpenAI/Codex tool schema compatibility and surface embedded-run replay/liveness state for long-running runs. (#64300) Thanks @100yenadmin. - Docs i18n: chunk raw doc translation, reject truncated tagged outputs, avoid ambiguous body-only wrapper unwrapping, and recover from terminated Pi translation sessions without changing the default `openai/gpt-5.4` path. (#62969, #63808) Thanks @hxy91819. +- Dreaming/memory-wiki: add ChatGPT import ingestion plus new `Imported Insights` and `Memory Palace` diary subtabs so Dreaming can inspect imported source chats, compiled wiki pages, and full source pages directly from the UI. (#64505) ### Fixes diff --git a/extensions/memory-wiki/src/chatgpt-import.ts b/extensions/memory-wiki/src/chatgpt-import.ts new file mode 100644 index 00000000000..d4fb268b643 --- /dev/null +++ b/extensions/memory-wiki/src/chatgpt-import.ts @@ -0,0 +1,903 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { + replaceManagedMarkdownBlock, + withTrailingNewline, +} from "openclaw/plugin-sdk/memory-host-markdown"; +import { compileMemoryWikiVault } from "./compile.js"; +import type { ResolvedMemoryWikiConfig } from "./config.js"; +import { appendMemoryWikiLog } from "./log.js"; +import { + parseWikiMarkdown, + renderWikiMarkdown, + WIKI_RELATED_END_MARKER, + WIKI_RELATED_START_MARKER, +} from "./markdown.js"; +import { initializeMemoryWikiVault } from "./vault.js"; + +const CHATGPT_PREFERENCE_SIGNAL_RE = + /\b(prefer|prefers|preference|want|wants|need|needs|avoid|avoids|hate|hates|love|loves|default to|should default to|always use|don't want|does not want|likes|dislikes)\b/i; +const HUMAN_START_MARKER = ""; +const HUMAN_END_MARKER = ""; + +const CHATGPT_RISK_RULES: Array<{ label: string; pattern: RegExp }> = [ + { + label: "relationships", + pattern: + /\b(relationship|dating|breakup|jealous|sex|intimacy|partner|apology|trust|boyfriend|girlfriend|husband|wife)\b/i, + }, + { + label: "health", + pattern: + /\b(supplement|medication|diagnosis|symptom|therapy|depression|anxiety|mri|migraine|injury|pain|cortisol|sleep)\b/i, + }, + { + label: "legal_tax", + pattern: + /\b(contract|tax|legal|law|lawsuit|visa|immigration|license|insurance|claim|non-residence|residency)\b/i, + }, + { + label: "finance", + pattern: + /\b(investment|invest|portfolio|dividend|yield|coupon|valuation|mortgage|loan|crypto|covered call|call option|put option)\b/i, + }, + { + label: "drugs", + pattern: /\b(vape|weed|cannabis|nicotine|opioid|ketamine)\b/i, + }, +]; + +type ChatGptMessage = { + role: string; + text: string; +}; + +type ChatGptRiskAssessment = { + level: "low" | "medium" | "high"; + reasons: string[]; +}; + +type ChatGptConversationRecord = { + conversationId: string; + title: string; + createdAt?: string; + updatedAt?: string; + sourcePath: string; + pageId: string; + pagePath: string; + labels: string[]; + risk: ChatGptRiskAssessment; + userMessageCount: number; + assistantMessageCount: number; + preferenceSignals: string[]; + firstUserLine?: string; + lastUserLine?: string; + transcript: ChatGptMessage[]; +}; + +type ChatGptImportOperation = "create" | "update" | "skip"; + +export type ChatGptImportAction = { + conversationId: string; + title: string; + pagePath: string; + operation: ChatGptImportOperation; + riskLevel: ChatGptRiskAssessment["level"]; + labels: string[]; + userMessageCount: number; + assistantMessageCount: number; + preferenceSignals: string[]; +}; + +type ChatGptImportRunEntry = { + path: string; + snapshotPath?: string; +}; + +type ChatGptImportRunRecord = { + version: 1; + runId: string; + importType: "chatgpt"; + exportPath: string; + sourcePath: string; + appliedAt: string; + conversationCount: number; + createdCount: number; + updatedCount: number; + skippedCount: number; + createdPaths: string[]; + updatedPaths: ChatGptImportRunEntry[]; + rolledBackAt?: string; +}; + +export type ChatGptImportResult = { + dryRun: boolean; + exportPath: string; + sourcePath: string; + conversationCount: number; + createdCount: number; + updatedCount: number; + skippedCount: number; + actions: ChatGptImportAction[]; + pagePaths: string[]; + runId?: string; + indexUpdatedFiles: string[]; +}; + +export type ChatGptRollbackResult = { + runId: string; + removedCount: number; + restoredCount: number; + pagePaths: string[]; + indexUpdatedFiles: string[]; + alreadyRolledBack: boolean; +}; + +function asRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + return value as Record; +} + +function normalizeWhitespace(value: string): string { + return value.trim().replace(/\s+/g, " "); +} + +function resolveConversationSourcePath(exportInputPath: string): { + exportPath: string; + conversationsPath: string; +} { + const resolved = path.resolve(exportInputPath); + const conversationsPath = resolved.endsWith(".json") + ? resolved + : path.join(resolved, "conversations.json"); + return { + exportPath: resolved, + conversationsPath, + }; +} + +async function loadConversations(exportInputPath: string): Promise<{ + exportPath: string; + conversationsPath: string; + conversations: Record[]; +}> { + const { exportPath, conversationsPath } = resolveConversationSourcePath(exportInputPath); + const raw = await fs.readFile(conversationsPath, "utf8"); + const parsed = JSON.parse(raw) as unknown; + if (Array.isArray(parsed)) { + return { + exportPath, + conversationsPath, + conversations: parsed.filter( + (entry): entry is Record => asRecord(entry) !== null, + ), + }; + } + const record = asRecord(parsed); + if (record) { + for (const value of Object.values(record)) { + if (Array.isArray(value)) { + return { + exportPath, + conversationsPath, + conversations: value.filter( + (entry): entry is Record => asRecord(entry) !== null, + ), + }; + } + } + } + throw new Error(`Unrecognized ChatGPT conversations export format: ${conversationsPath}`); +} + +function isoFromUnix(raw: unknown): string | undefined { + if (typeof raw !== "number" && typeof raw !== "string") { + return undefined; + } + const numeric = Number(raw); + if (!Number.isFinite(numeric)) { + return undefined; + } + return new Date(numeric * 1000).toISOString(); +} + +function cleanMessageText(value: string): string { + const trimmed = value.trim(); + if (!trimmed) { + return ""; + } + if ( + (trimmed.includes("asset_pointer") || + trimmed.includes("image_asset_pointer") || + trimmed.includes("dalle") || + trimmed.includes("file_service")) && + trimmed.length > 40 + ) { + return ""; + } + if ( + trimmed.startsWith("{") && + trimmed.length > 80 && + (trimmed.includes(":") || trimmed.includes("content_type")) + ) { + const textMatch = trimmed.match(/["']text["']\s*:\s*(["'])(.+?)\1/s); + return textMatch?.[2] ? normalizeWhitespace(textMatch[2]) : ""; + } + return trimmed; +} + +function extractMessageText(message: Record): string { + const content = asRecord(message.content); + if (content) { + const parts = content.parts; + if (Array.isArray(parts)) { + const collected: string[] = []; + for (const part of parts) { + if (typeof part === "string") { + const cleaned = cleanMessageText(part); + if (cleaned) { + collected.push(cleaned); + } + continue; + } + const partRecord = asRecord(part); + if (partRecord && typeof partRecord.text === "string" && partRecord.text.trim()) { + collected.push(partRecord.text.trim()); + } + } + return collected.join("\n").trim(); + } + if (typeof content.text === "string") { + return cleanMessageText(content.text); + } + } + return typeof message.text === "string" ? cleanMessageText(message.text) : ""; +} + +function activeBranchMessages(conversation: Record): ChatGptMessage[] { + const mapping = asRecord(conversation.mapping); + if (!mapping) { + return []; + } + let currentNode = + typeof conversation.current_node === "string" ? conversation.current_node : undefined; + const seen = new Set(); + const chain: ChatGptMessage[] = []; + while (currentNode && !seen.has(currentNode)) { + seen.add(currentNode); + const node = asRecord(mapping[currentNode]); + if (!node) { + break; + } + const message = asRecord(node.message); + if (message) { + const author = asRecord(message.author); + const role = typeof author?.role === "string" ? author.role : "unknown"; + const text = extractMessageText(message); + if (text) { + chain.push({ role, text }); + } + } + currentNode = typeof node.parent === "string" ? node.parent : undefined; + } + return chain.reverse(); +} + +function inferRisk(title: string, sampleText: string): ChatGptRiskAssessment { + const blob = `${title}\n${sampleText}`.toLowerCase(); + const reasons = CHATGPT_RISK_RULES.filter((rule) => rule.pattern.test(blob)).map( + (rule) => rule.label, + ); + if (reasons.length > 0) { + return { level: "high", reasons: [...new Set(reasons)] }; + } + if (/\b(career|job|salary|interview|offer|resume|cover letter)\b/i.test(blob)) { + return { level: "medium", reasons: ["work_career"] }; + } + return { level: "low", reasons: [] }; +} + +function inferLabels(title: string, sampleText: string): string[] { + const blob = `${title}\n${sampleText}`.toLowerCase(); + const labels = new Set(["domain/personal"]); + const addAreaTopic = (area: string, topics: string[]) => { + labels.add(area); + for (const topic of topics) { + labels.add(topic); + } + }; + const hasTranslation = + /\b(translate|translation|traduc\w*|traducc\w*|traduç\w*|traducci[oó]n|traduccio|traducció|traduzione)\b/i.test( + blob, + ); + const hasLearning = + /\b(anki|flashcards?|grammar|conjugat\w*|declension|pronunciation|vocab(?:ular(?:y|io))?|lesson|tutor|teacher|jlpt|kanji|hiragana|katakana|study|learn|practice)\b/i.test( + blob, + ); + const hasLanguageName = + /\b(japanese|portuguese|catalan|castellano|espa[nñ]ol|franc[eé]s|french|italian|german|spanish)\b/i.test( + blob, + ); + if (hasTranslation) { + labels.add("topic/translation"); + } + if ( + hasLearning || + (hasLanguageName && /\b(learn|study|practice|lesson|tutor|grammar)\b/i.test(blob)) + ) { + addAreaTopic("area/language-learning", ["topic/language-learning"]); + } + if ( + /\b(hike|trail|hotel|flight|trip|travel|airport|itinerary|booking|airbnb|train|stay)\b/i.test( + blob, + ) + ) { + labels.add("area/travel"); + labels.add("topic/travel"); + } + if ( + /\b(recipe|cook|cooking|bread|sourdough|pizza|espresso|coffee|mousse|cast iron|meatballs?)\b/i.test( + blob, + ) + ) { + addAreaTopic("area/cooking", ["topic/cooking"]); + } + if ( + /\b(garden|orchard|plant|soil|compost|agroforestry|permaculture|mulch|beds?|irrigation|seeds?)\b/i.test( + blob, + ) + ) { + addAreaTopic("area/gardening", ["topic/gardening"]); + } + if (/\b(dating|relationship|partner|jealous|breakup|trust)\b/i.test(blob)) { + addAreaTopic("area/relationships", ["topic/relationships"]); + } + if ( + /\b(investment|invest|portfolio|dividend|yield|coupon|valuation|return|mortgage|loan|kraken|crypto|covered call|call option|put option|option chain|bond|stocks?)\b/i.test( + blob, + ) + ) { + addAreaTopic("area/finance", ["topic/finance"]); + } + if ( + /\b(contract|mou|tax|impuesto|legal|law|lawsuit|visa|immigration|license|licencia|dispute|claim|insurance|non-residence|residency)\b/i.test( + blob, + ) + ) { + addAreaTopic("area/legal-tax", ["topic/legal-tax"]); + } + if ( + /\b(supplement|medication|diagnos(?:is|e)|symptom|therapy|depress(?:ion|ed)|anxiet(?:y|ies)|mri|migraine|injur(?:y|ies)|pain|cortisol|sleep|dentist|dermatolog(?:ist|y))\b/i.test( + blob, + ) + ) { + addAreaTopic("area/health", ["topic/health"]); + } + if ( + /\b(book (an )?appointment|rebook|open (a )?new account|driving test|exam|gestor(?:a)?|itv)\b/i.test( + blob, + ) + ) { + addAreaTopic("area/life-admin", ["topic/life-admin"]); + } + if (/\b(frc|robot|robotics|wpilib|limelight|chiefdelphi)\b/i.test(blob)) { + addAreaTopic("area/work", ["topic/robotics"]); + } else if ( + /\b(docker|git|python|node|npm|pip|sql|postgres|api|bug|stack trace|permission denied)\b/i.test( + blob, + ) + ) { + addAreaTopic("area/work", ["topic/software"]); + } else if (/\b(job|interview|cover letter|resume|cv)\b/i.test(blob)) { + addAreaTopic("area/work", ["topic/career"]); + } + if (/\b(wifi|wi-fi|starlink|router|mesh|network|orbi|milesight|coverage)\b/i.test(blob)) { + addAreaTopic("area/home", ["topic/home-infrastructure"]); + } + if ( + /\b(p38|range rover|porsche|bmw|bobcat|excavator|auger|trailer|chainsaw|stihl)\b/i.test(blob) + ) { + addAreaTopic("area/vehicles", ["topic/vehicles"]); + } + if (![...labels].some((label) => label.startsWith("area/"))) { + labels.add("area/other"); + } + return [...labels]; +} + +function collectPreferenceSignals(userTexts: string[]): string[] { + const signals: string[] = []; + const seen = new Set(); + for (const text of userTexts.slice(0, 25)) { + for (const rawLine of text.split(/\r?\n/)) { + const line = normalizeWhitespace(rawLine); + if (!line || !CHATGPT_PREFERENCE_SIGNAL_RE.test(line)) { + continue; + } + const key = line.toLowerCase(); + if (seen.has(key)) { + continue; + } + seen.add(key); + signals.push(line); + if (signals.length >= 10) { + return signals; + } + } + } + return signals; +} + +function buildTranscript(messages: ChatGptMessage[]): string { + if (messages.length === 0) { + return "_No active-branch transcript could be reconstructed._"; + } + return messages + .flatMap((message) => [ + `### ${message.role[0]?.toUpperCase() ?? "U"}${message.role.slice(1)}`, + "", + message.text, + "", + ]) + .join("\n") + .trim(); +} + +function resolveConversationPagePath(record: { conversationId: string; createdAt?: string }): { + pageId: string; + pagePath: string; +} { + const conversationSlug = record.conversationId.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); + const pageId = `source.chatgpt.${conversationSlug || createHash("sha1").update(record.conversationId).digest("hex").slice(0, 12)}`; + const datePrefix = record.createdAt?.slice(0, 10) ?? "undated"; + const shortId = conversationSlug.slice(0, 8) || "export"; + return { + pageId, + pagePath: path + .join("sources", `chatgpt-${datePrefix}-${conversationSlug || shortId}.md`) + .replace(/\\/g, "/"), + }; +} + +function toConversationRecord( + conversation: Record, + sourcePath: string, +): ChatGptConversationRecord | null { + const conversationId = + typeof conversation.conversation_id === "string" ? conversation.conversation_id.trim() : ""; + if (!conversationId) { + return null; + } + const title = + typeof conversation.title === "string" && conversation.title.trim() + ? conversation.title.trim() + : "Untitled conversation"; + const transcript = activeBranchMessages(conversation); + const userTexts = transcript.filter((entry) => entry.role === "user").map((entry) => entry.text); + const assistantTexts = transcript.filter((entry) => entry.role === "assistant"); + const sampleText = userTexts.slice(0, 6).join("\n"); + const risk = inferRisk(title, sampleText); + const labels = inferLabels(title, sampleText); + const { pageId, pagePath } = resolveConversationPagePath({ + conversationId, + createdAt: isoFromUnix(conversation.create_time), + }); + return { + conversationId, + title, + createdAt: isoFromUnix(conversation.create_time), + updatedAt: isoFromUnix(conversation.update_time) ?? isoFromUnix(conversation.create_time), + sourcePath, + pageId, + pagePath, + labels, + risk, + userMessageCount: userTexts.length, + assistantMessageCount: assistantTexts.length, + preferenceSignals: risk.level === "low" ? collectPreferenceSignals(userTexts) : [], + firstUserLine: userTexts[0]?.split(/\r?\n/)[0]?.trim(), + lastUserLine: userTexts.at(-1)?.split(/\r?\n/)[0]?.trim(), + transcript, + }; +} + +function renderConversationPage(record: ChatGptConversationRecord): string { + const autoDigestLines = + record.risk.level === "low" + ? [ + `- User messages: ${record.userMessageCount}`, + `- Assistant messages: ${record.assistantMessageCount}`, + ...(record.firstUserLine ? [`- First user line: ${record.firstUserLine}`] : []), + ...(record.lastUserLine ? [`- Last user line: ${record.lastUserLine}`] : []), + ...(record.preferenceSignals.length > 0 + ? ["- Preference signals:", ...record.preferenceSignals.map((line) => ` - ${line}`)] + : ["- Preference signals: none detected"]), + ] + : [ + "- Auto digest withheld from durable-candidate generation until reviewed.", + `- Risk reasons: ${record.risk.reasons.length > 0 ? record.risk.reasons.join(", ") : "none recorded"}`, + ]; + return renderWikiMarkdown({ + frontmatter: { + pageType: "source", + id: record.pageId, + title: `ChatGPT Export: ${record.title}`, + sourceType: "chatgpt-export", + sourceSystem: "chatgpt", + sourcePath: record.sourcePath, + conversationId: record.conversationId, + riskLevel: record.risk.level, + riskReasons: record.risk.reasons, + labels: record.labels, + status: "draft", + ...(record.createdAt ? { createdAt: record.createdAt } : {}), + ...(record.updatedAt ? { updatedAt: record.updatedAt } : {}), + }, + body: [ + `# ChatGPT Export: ${record.title}`, + "", + "## Source", + `- Conversation id: \`${record.conversationId}\``, + `- Export file: \`${record.sourcePath}\``, + ...(record.createdAt ? [`- Created: ${record.createdAt}`] : []), + ...(record.updatedAt ? [`- Updated: ${record.updatedAt}`] : []), + "", + "## Auto Triage", + `- Risk level: \`${record.risk.level}\``, + `- Labels: ${record.labels.join(", ")}`, + `- Active-branch messages: ${record.transcript.length}`, + "", + "## Auto Digest", + ...autoDigestLines, + "", + "## Active Branch Transcript", + buildTranscript(record.transcript), + "", + "## Notes", + HUMAN_START_MARKER, + HUMAN_END_MARKER, + "", + ].join("\n"), + }); +} + +function replaceSimpleManagedBlock(params: { + original: string; + startMarker: string; + endMarker: string; + replacement: string; +}): string { + const escapedStart = params.startMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const escapedEnd = params.endMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const blockPattern = new RegExp(`${escapedStart}[\\s\\S]*?${escapedEnd}`); + return params.original.replace(blockPattern, params.replacement); +} + +function extractSimpleManagedBlock(params: { + body: string; + startMarker: string; + endMarker: string; +}): string | null { + const escapedStart = params.startMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const escapedEnd = params.endMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const blockPattern = new RegExp(`${escapedStart}[\\s\\S]*?${escapedEnd}`); + return params.body.match(blockPattern)?.[0] ?? null; +} + +function extractManagedBlockBody(params: { + body: string; + startMarker: string; + endMarker: string; +}): string | null { + const escapedStart = params.startMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const escapedEnd = params.endMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const blockPattern = new RegExp(`${escapedStart}\\n?([\\s\\S]*?)\\n?${escapedEnd}`); + const captured = params.body.match(blockPattern)?.[1]; + return typeof captured === "string" ? captured.trim() : null; +} + +function preserveExistingPageBlocks(rendered: string, existing: string): string { + if (!existing.trim()) { + return withTrailingNewline(rendered); + } + const parsedExisting = parseWikiMarkdown(existing); + const parsedRendered = parseWikiMarkdown(rendered); + let nextBody = parsedRendered.body; + + const humanBlock = extractSimpleManagedBlock({ + body: parsedExisting.body, + startMarker: HUMAN_START_MARKER, + endMarker: HUMAN_END_MARKER, + }); + if (humanBlock) { + nextBody = replaceSimpleManagedBlock({ + original: nextBody, + startMarker: HUMAN_START_MARKER, + endMarker: HUMAN_END_MARKER, + replacement: humanBlock, + }); + } + + const relatedBody = extractManagedBlockBody({ + body: parsedExisting.body, + startMarker: WIKI_RELATED_START_MARKER, + endMarker: WIKI_RELATED_END_MARKER, + }); + if (relatedBody) { + nextBody = replaceManagedMarkdownBlock({ + original: nextBody, + heading: "## Related", + startMarker: WIKI_RELATED_START_MARKER, + endMarker: WIKI_RELATED_END_MARKER, + body: relatedBody, + }); + } + + return withTrailingNewline( + renderWikiMarkdown({ + frontmatter: parsedRendered.frontmatter, + body: nextBody, + }), + ); +} + +function buildRunId(exportPath: string, nowIso: string): string { + const seed = `${exportPath}:${nowIso}:${Math.random()}`; + return `chatgpt-${createHash("sha1").update(seed).digest("hex").slice(0, 12)}`; +} + +function resolveImportRunsDir(vaultRoot: string): string { + return path.join(vaultRoot, ".openclaw-wiki", "import-runs"); +} + +function resolveImportRunPath(vaultRoot: string, runId: string): string { + return path.join(resolveImportRunsDir(vaultRoot), `${runId}.json`); +} + +function normalizeConversationActions( + records: ChatGptConversationRecord[], + operations: Map, +): ChatGptImportAction[] { + return records.map((record) => ({ + conversationId: record.conversationId, + title: record.title, + pagePath: record.pagePath, + operation: operations.get(record.pagePath) ?? "skip", + riskLevel: record.risk.level, + labels: record.labels, + userMessageCount: record.userMessageCount, + assistantMessageCount: record.assistantMessageCount, + preferenceSignals: record.preferenceSignals, + })); +} + +async function writeImportRunRecord( + vaultRoot: string, + record: ChatGptImportRunRecord, +): Promise { + const recordPath = resolveImportRunPath(vaultRoot, record.runId); + await fs.mkdir(path.dirname(recordPath), { recursive: true }); + await fs.writeFile(recordPath, `${JSON.stringify(record, null, 2)}\n`, "utf8"); +} + +async function readImportRunRecord( + vaultRoot: string, + runId: string, +): Promise { + const recordPath = resolveImportRunPath(vaultRoot, runId); + const raw = await fs.readFile(recordPath, "utf8"); + return JSON.parse(raw) as ChatGptImportRunRecord; +} + +async function writeTrackedImportPage(params: { + vaultRoot: string; + runDir: string; + relativePath: string; + content: string; + record: ChatGptImportRunRecord; +}): Promise { + const absolutePath = path.join(params.vaultRoot, params.relativePath); + const existing = await fs.readFile(absolutePath, "utf8").catch(() => ""); + const rendered = preserveExistingPageBlocks(params.content, existing); + if (existing === rendered) { + return "skip"; + } + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + if (!existing) { + await fs.writeFile(absolutePath, rendered, "utf8"); + params.record.createdPaths.push(params.relativePath); + return "create"; + } + const snapshotHash = createHash("sha1").update(params.relativePath).digest("hex").slice(0, 12); + const snapshotRelativePath = path.join("snapshots", `${snapshotHash}.md`).replace(/\\/g, "/"); + const snapshotAbsolutePath = path.join(params.runDir, snapshotRelativePath); + await fs.mkdir(path.dirname(snapshotAbsolutePath), { recursive: true }); + await fs.writeFile(snapshotAbsolutePath, existing, "utf8"); + await fs.writeFile(absolutePath, rendered, "utf8"); + params.record.updatedPaths.push({ + path: params.relativePath, + snapshotPath: snapshotRelativePath, + }); + return "update"; +} + +export async function importChatGptConversations(params: { + config: ResolvedMemoryWikiConfig; + exportPath: string; + dryRun?: boolean; + nowMs?: number; +}): Promise { + await initializeMemoryWikiVault(params.config, { nowMs: params.nowMs }); + const { exportPath, conversationsPath, conversations } = await loadConversations( + params.exportPath, + ); + const records = conversations + .map((conversation) => toConversationRecord(conversation, conversationsPath)) + .filter((entry): entry is ChatGptConversationRecord => entry !== null) + .toSorted((left, right) => left.pagePath.localeCompare(right.pagePath)); + + const operations = new Map(); + let createdCount = 0; + let updatedCount = 0; + let skippedCount = 0; + let runId: string | undefined; + const nowIso = new Date(params.nowMs ?? Date.now()).toISOString(); + + let importRunRecord: ChatGptImportRunRecord | undefined; + let importRunDir = ""; + + if (!params.dryRun) { + runId = buildRunId(exportPath, nowIso); + importRunDir = path.join(resolveImportRunsDir(params.config.vault.path), runId); + importRunRecord = { + version: 1, + runId, + importType: "chatgpt", + exportPath, + sourcePath: conversationsPath, + appliedAt: nowIso, + conversationCount: records.length, + createdCount: 0, + updatedCount: 0, + skippedCount: 0, + createdPaths: [], + updatedPaths: [], + }; + } + + for (const record of records) { + const rendered = renderConversationPage(record); + const absolutePath = path.join(params.config.vault.path, record.pagePath); + const existing = await fs.readFile(absolutePath, "utf8").catch(() => ""); + const stabilized = preserveExistingPageBlocks(rendered, existing); + const operation: ChatGptImportOperation = + existing === stabilized ? "skip" : existing ? "update" : "create"; + operations.set(record.pagePath, operation); + if (operation === "create") { + createdCount += 1; + } else if (operation === "update") { + updatedCount += 1; + } else { + skippedCount += 1; + } + if (!params.dryRun && importRunRecord) { + await writeTrackedImportPage({ + vaultRoot: params.config.vault.path, + runDir: importRunDir, + relativePath: record.pagePath, + content: rendered, + record: importRunRecord, + }); + } + } + + let indexUpdatedFiles: string[] = []; + if (!params.dryRun && importRunRecord) { + importRunRecord.createdCount = createdCount; + importRunRecord.updatedCount = updatedCount; + importRunRecord.skippedCount = skippedCount; + if (importRunRecord.createdPaths.length > 0 || importRunRecord.updatedPaths.length > 0) { + const compile = await compileMemoryWikiVault(params.config); + indexUpdatedFiles = compile.updatedFiles; + await writeImportRunRecord(params.config.vault.path, importRunRecord); + await appendMemoryWikiLog(params.config.vault.path, { + type: "ingest", + timestamp: nowIso, + details: { + sourceType: "chatgpt-export", + runId: importRunRecord.runId, + exportPath, + sourcePath: conversationsPath, + conversationCount: records.length, + createdCount: importRunRecord.createdPaths.length, + updatedCount: importRunRecord.updatedPaths.length, + skippedCount, + }, + }); + } else { + runId = undefined; + } + } + + return { + dryRun: Boolean(params.dryRun), + exportPath, + sourcePath: conversationsPath, + conversationCount: records.length, + createdCount, + updatedCount, + skippedCount, + actions: normalizeConversationActions(records, operations), + pagePaths: records.map((record) => record.pagePath), + ...(runId ? { runId } : {}), + indexUpdatedFiles, + }; +} + +export async function rollbackChatGptImportRun(params: { + config: ResolvedMemoryWikiConfig; + runId: string; +}): Promise { + await initializeMemoryWikiVault(params.config); + const record = await readImportRunRecord(params.config.vault.path, params.runId); + if (record.rolledBackAt) { + return { + runId: record.runId, + removedCount: 0, + restoredCount: 0, + pagePaths: [ + ...record.createdPaths, + ...record.updatedPaths.map((entry) => entry.path), + ].toSorted((left, right) => left.localeCompare(right)), + indexUpdatedFiles: [], + alreadyRolledBack: true, + }; + } + let removedCount = 0; + for (const relativePath of record.createdPaths) { + await fs + .rm(path.join(params.config.vault.path, relativePath), { force: true }) + .catch(() => undefined); + removedCount += 1; + } + let restoredCount = 0; + const runDir = path.join(resolveImportRunsDir(params.config.vault.path), record.runId); + for (const entry of record.updatedPaths) { + if (!entry.snapshotPath) { + continue; + } + const snapshotPath = path.join(runDir, entry.snapshotPath); + const snapshot = await fs.readFile(snapshotPath, "utf8"); + const targetPath = path.join(params.config.vault.path, entry.path); + await fs.mkdir(path.dirname(targetPath), { recursive: true }); + await fs.writeFile(targetPath, snapshot, "utf8"); + restoredCount += 1; + } + const compile = await compileMemoryWikiVault(params.config); + record.rolledBackAt = new Date().toISOString(); + await writeImportRunRecord(params.config.vault.path, record); + await appendMemoryWikiLog(params.config.vault.path, { + type: "ingest", + timestamp: record.rolledBackAt, + details: { + sourceType: "chatgpt-export", + runId: record.runId, + rollback: true, + removedCount, + restoredCount, + }, + }); + return { + runId: record.runId, + removedCount, + restoredCount, + pagePaths: [...record.createdPaths, ...record.updatedPaths.map((entry) => entry.path)].toSorted( + (left, right) => left.localeCompare(right), + ), + indexUpdatedFiles: compile.updatedFiles, + alreadyRolledBack: false, + }; +} diff --git a/extensions/memory-wiki/src/cli.test.ts b/extensions/memory-wiki/src/cli.test.ts index 60716dd9828..b806e170d79 100644 --- a/extensions/memory-wiki/src/cli.test.ts +++ b/extensions/memory-wiki/src/cli.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { Command } from "commander"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { registerWikiCli } from "./cli.js"; +import { registerWikiCli, runWikiChatGptImport, runWikiChatGptRollback } from "./cli.js"; import type { MemoryWikiPluginConfig } from "./config.js"; import { parseWikiMarkdown, renderWikiMarkdown } from "./markdown.js"; import { createMemoryWikiTestHarness } from "./test-helpers.js"; @@ -47,6 +47,47 @@ describe("memory-wiki cli", () => { }); } + async function createChatGptExport(rootDir: string) { + const exportDir = path.join(rootDir, "chatgpt-export"); + await fs.mkdir(exportDir, { recursive: true }); + const conversations = [ + { + conversation_id: "12345678-1234-1234-1234-1234567890ab", + title: "Travel preference check", + create_time: 1_712_363_200, + update_time: 1_712_366_800, + current_node: "assistant-1", + mapping: { + root: {}, + "user-1": { + parent: "root", + message: { + author: { role: "user" }, + content: { + parts: ["I prefer aisle seats and I don't want a hotel far from the airport."], + }, + }, + }, + "assistant-1": { + parent: "user-1", + message: { + author: { role: "assistant" }, + content: { + parts: ["Noted. I will keep travel options close to the airport."], + }, + }, + }, + }, + }, + ]; + await fs.writeFile( + path.join(exportDir, "conversations.json"), + `${JSON.stringify(conversations, null, 2)}\n`, + "utf8", + ); + return exportDir; + } + it("registers apply synthesis and writes a synthesis page", async () => { const { rootDir, config } = await createCliVault(); const program = new Command(); @@ -153,4 +194,57 @@ cli note expect(process.exitCode).toBe(1); }); + + it("imports ChatGPT exports with dry-run, apply, and rollback", async () => { + const { rootDir, config } = await createCliVault({ initialize: true }); + const exportDir = await createChatGptExport(rootDir); + + const dryRun = await runWikiChatGptImport({ + config, + exportPath: exportDir, + dryRun: true, + json: true, + }); + expect(dryRun.dryRun).toBe(true); + expect(dryRun.createdCount).toBe(1); + await expect(fs.readdir(path.join(rootDir, "sources"))).resolves.toEqual([]); + + const applied = await runWikiChatGptImport({ + config, + exportPath: exportDir, + json: true, + }); + expect(applied.runId).toBeTruthy(); + expect(applied.createdCount).toBe(1); + const sourceFiles = (await fs.readdir(path.join(rootDir, "sources"))).filter( + (entry) => entry !== "index.md", + ); + expect(sourceFiles).toHaveLength(1); + const pageContent = await fs.readFile(path.join(rootDir, "sources", sourceFiles[0]!), "utf8"); + expect(pageContent).toContain("ChatGPT Export: Travel preference check"); + expect(pageContent).toContain("I prefer aisle seats"); + expect(pageContent).toContain("Preference signals:"); + + const secondDryRun = await runWikiChatGptImport({ + config, + exportPath: exportDir, + dryRun: true, + json: true, + }); + expect(secondDryRun.createdCount).toBe(0); + expect(secondDryRun.updatedCount).toBe(0); + expect(secondDryRun.skippedCount).toBe(1); + + const rollback = await runWikiChatGptRollback({ + config, + runId: applied.runId!, + json: true, + }); + expect(rollback.alreadyRolledBack).toBe(false); + await expect( + fs + .readdir(path.join(rootDir, "sources")) + .then((entries) => entries.filter((entry) => entry !== "index.md")), + ).resolves.toEqual([]); + }); }); diff --git a/extensions/memory-wiki/src/cli.ts b/extensions/memory-wiki/src/cli.ts index be6ce8cd203..461a3f1e19b 100644 --- a/extensions/memory-wiki/src/cli.ts +++ b/extensions/memory-wiki/src/cli.ts @@ -2,6 +2,12 @@ import fs from "node:fs/promises"; import type { Command } from "commander"; import type { OpenClawConfig } from "../api.js"; import { applyMemoryWikiMutation } from "./apply.js"; +import { + importChatGptConversations, + rollbackChatGptImportRun, + type ChatGptImportResult, + type ChatGptRollbackResult, +} from "./chatgpt-import.js"; import { compileMemoryWikiVault } from "./compile.js"; import { resolveMemoryWikiConfig, @@ -98,6 +104,16 @@ type WikiUnsafeLocalImportCommandOptions = { json?: boolean; }; +type WikiChatGptImportCommandOptions = { + json?: boolean; + dryRun?: boolean; + export?: string; +}; + +type WikiChatGptRollbackCommandOptions = { + json?: boolean; +}; + type WikiObsidianSearchCommandOptions = { json?: boolean; }; @@ -592,6 +608,59 @@ export async function runWikiObsidianDailyCli(params: { }); } +function formatChatGptImportSummary(result: ChatGptImportResult): string { + if (result.dryRun) { + return `ChatGPT import dry run scanned ${result.conversationCount} conversations (${result.createdCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged).`; + } + const runSuffix = result.runId ? ` Run id: ${result.runId}.` : ""; + return `ChatGPT import applied ${result.conversationCount} conversations (${result.createdCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged). Refreshed ${result.indexUpdatedFiles.length} index file${result.indexUpdatedFiles.length === 1 ? "" : "s"}.${runSuffix}`; +} + +function formatChatGptRollbackSummary(result: ChatGptRollbackResult): string { + if (result.alreadyRolledBack) { + return `ChatGPT import run ${result.runId} was already rolled back.`; + } + return `Rolled back ChatGPT import run ${result.runId} (${result.removedCount} removed, ${result.restoredCount} restored). Refreshed ${result.indexUpdatedFiles.length} index file${result.indexUpdatedFiles.length === 1 ? "" : "s"}.`; +} + +export async function runWikiChatGptImport(params: { + config: ResolvedMemoryWikiConfig; + exportPath: string; + dryRun?: boolean; + json?: boolean; + stdout?: Pick; +}) { + return runWikiCommandWithSummary({ + json: params.json, + stdout: params.stdout, + run: () => + importChatGptConversations({ + config: params.config, + exportPath: params.exportPath, + dryRun: params.dryRun, + }), + render: formatChatGptImportSummary, + }); +} + +export async function runWikiChatGptRollback(params: { + config: ResolvedMemoryWikiConfig; + runId: string; + json?: boolean; + stdout?: Pick; +}) { + return runWikiCommandWithSummary({ + json: params.json, + stdout: params.stdout, + run: () => + rollbackChatGptImportRun({ + config: params.config, + runId: params.runId, + }), + render: formatChatGptRollbackSummary, + }); +} + export function registerWikiCli( program: Command, pluginConfig?: MemoryWikiPluginConfig | ResolvedMemoryWikiConfig, @@ -764,6 +833,36 @@ export function registerWikiCli( await runWikiUnsafeLocalImport({ config, appConfig, json: opts.json }); }); + const chatgpt = wiki + .command("chatgpt") + .description("Import ChatGPT export history into wiki source pages"); + chatgpt + .command("import") + .description("Import a ChatGPT export into draft wiki source pages") + .requiredOption("--export ", "ChatGPT export directory or conversations.json path") + .option("--dry-run", "Preview changes without writing", false) + .option("--json", "Print JSON") + .action(async (opts: WikiChatGptImportCommandOptions) => { + await runWikiChatGptImport({ + config, + exportPath: opts.export!, + dryRun: opts.dryRun, + json: opts.json, + }); + }); + chatgpt + .command("rollback") + .description("Roll back a previously applied ChatGPT import run") + .argument("", "Import run id") + .option("--json", "Print JSON") + .action(async (runId: string, opts: WikiChatGptRollbackCommandOptions) => { + await runWikiChatGptRollback({ + config, + runId, + json: opts.json, + }); + }); + const obsidian = wiki.command("obsidian").description("Run official Obsidian CLI helpers"); obsidian .command("status") diff --git a/extensions/memory-wiki/src/gateway.test.ts b/extensions/memory-wiki/src/gateway.test.ts index b05d0209348..5003a409479 100644 --- a/extensions/memory-wiki/src/gateway.test.ts +++ b/extensions/memory-wiki/src/gateway.test.ts @@ -5,7 +5,10 @@ import { type ApplyMemoryWikiMutation, } from "./apply.js"; import { registerMemoryWikiGatewayMethods } from "./gateway.js"; +import { listMemoryWikiImportInsights } from "./import-insights.js"; +import { listMemoryWikiImportRuns } from "./import-runs.js"; import { ingestMemoryWikiSource } from "./ingest.js"; +import { listMemoryWikiPalace } from "./memory-palace.js"; import { searchMemoryWiki } from "./query.js"; import { syncMemoryWikiImportedSources } from "./source-sync.js"; import { resolveMemoryWikiStatus } from "./status.js"; @@ -24,10 +27,22 @@ vi.mock("./ingest.js", () => ({ ingestMemoryWikiSource: vi.fn(), })); +vi.mock("./import-insights.js", () => ({ + listMemoryWikiImportInsights: vi.fn(), +})); + +vi.mock("./import-runs.js", () => ({ + listMemoryWikiImportRuns: vi.fn(), +})); + vi.mock("./lint.js", () => ({ lintMemoryWikiVault: vi.fn(), })); +vi.mock("./memory-palace.js", () => ({ + listMemoryWikiPalace: vi.fn(), +})); + vi.mock("./obsidian.js", () => ({ probeObsidianCli: vi.fn(), runObsidianCommand: vi.fn(), @@ -90,6 +105,25 @@ describe("memory-wiki gateway methods", () => { vi.mocked(ingestMemoryWikiSource).mockResolvedValue({ pagePath: "sources/alpha-notes.md", } as never); + vi.mocked(listMemoryWikiImportRuns).mockResolvedValue({ + runs: [], + totalRuns: 0, + activeRuns: 0, + rolledBackRuns: 0, + } as never); + vi.mocked(listMemoryWikiImportInsights).mockResolvedValue({ + sourceType: "chatgpt", + totalItems: 0, + totalClusters: 0, + clusters: [], + } as never); + vi.mocked(listMemoryWikiPalace).mockResolvedValue({ + totalItems: 0, + totalClaims: 0, + totalQuestions: 0, + totalContradictions: 0, + clusters: [], + } as never); vi.mocked(normalizeMemoryWikiMutationInput).mockReturnValue({ op: "create_synthesis", title: "Gateway Alpha", @@ -135,6 +169,169 @@ describe("memory-wiki gateway methods", () => { ); }); + it("returns recent import runs over the gateway", async () => { + const { config } = await createVault({ prefix: "memory-wiki-gateway-" }); + const { api, registerGatewayMethod } = createPluginApi(); + vi.mocked(listMemoryWikiImportRuns).mockResolvedValue({ + runs: [ + { + runId: "chatgpt-abc123", + importType: "chatgpt", + appliedAt: "2026-04-10T10:00:00.000Z", + exportPath: "/tmp/chatgpt", + sourcePath: "/tmp/chatgpt/conversations.json", + conversationCount: 12, + createdCount: 4, + updatedCount: 2, + skippedCount: 6, + status: "applied", + pagePaths: ["sources/chatgpt-2026-04-10-alpha.md"], + samplePaths: ["sources/chatgpt-2026-04-10-alpha.md"], + }, + ], + totalRuns: 1, + activeRuns: 1, + rolledBackRuns: 0, + } as never); + + registerMemoryWikiGatewayMethods({ api, config }); + const handler = findGatewayHandler(registerGatewayMethod, "wiki.importRuns"); + if (!handler) { + throw new Error("wiki.importRuns handler missing"); + } + const respond = vi.fn(); + + await handler({ + params: { + limit: 5, + }, + respond, + }); + + expect(listMemoryWikiImportRuns).toHaveBeenCalledWith(config, { limit: 5 }); + expect(respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ + totalRuns: 1, + activeRuns: 1, + }), + ); + }); + + it("returns import insights over the gateway", async () => { + const { config } = await createVault({ prefix: "memory-wiki-gateway-" }); + const { api, registerGatewayMethod } = createPluginApi(); + vi.mocked(listMemoryWikiImportInsights).mockResolvedValue({ + sourceType: "chatgpt", + totalItems: 2, + totalClusters: 1, + clusters: [ + { + key: "topic/travel", + label: "Travel", + itemCount: 2, + highRiskCount: 1, + withheldCount: 1, + preferenceSignalCount: 0, + updatedAt: "2026-04-10T10:00:00.000Z", + items: [ + { + pagePath: "sources/chatgpt-2026-04-10-alpha.md", + title: "BA flight receipts process", + riskLevel: "low", + labels: ["domain/personal", "area/travel", "topic/travel"], + topicKey: "topic/travel", + topicLabel: "Travel", + digestStatus: "available", + firstUserLine: "how do i get receipts?", + lastUserLine: "that option does not exist", + preferenceSignals: [], + }, + ], + }, + ], + } as never); + + registerMemoryWikiGatewayMethods({ api, config }); + const handler = findGatewayHandler(registerGatewayMethod, "wiki.importInsights"); + if (!handler) { + throw new Error("wiki.importInsights handler missing"); + } + const respond = vi.fn(); + + await handler({ + params: {}, + respond, + }); + + expect(syncMemoryWikiImportedSources).toHaveBeenCalledWith({ config, appConfig: undefined }); + expect(listMemoryWikiImportInsights).toHaveBeenCalledWith(config); + expect(respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ + sourceType: "chatgpt", + totalItems: 2, + totalClusters: 1, + }), + ); + }); + + it("returns memory palace overview over the gateway", async () => { + const { config } = await createVault({ prefix: "memory-wiki-gateway-" }); + const { api, registerGatewayMethod } = createPluginApi(); + vi.mocked(listMemoryWikiPalace).mockResolvedValue({ + totalItems: 3, + totalClaims: 4, + totalQuestions: 1, + totalContradictions: 1, + clusters: [ + { + key: "synthesis", + label: "Syntheses", + itemCount: 1, + claimCount: 2, + questionCount: 1, + contradictionCount: 0, + items: [ + { + pagePath: "syntheses/travel-system.md", + title: "Travel system", + kind: "synthesis", + claimCount: 2, + questionCount: 1, + contradictionCount: 0, + claims: ["prefers direct receipts"], + questions: ["should this become a playbook?"], + contradictions: [], + }, + ], + }, + ], + } as never); + + registerMemoryWikiGatewayMethods({ api, config }); + const handler = findGatewayHandler(registerGatewayMethod, "wiki.palace"); + if (!handler) { + throw new Error("wiki.palace handler missing"); + } + const respond = vi.fn(); + + await handler({ + params: {}, + respond, + }); + + expect(syncMemoryWikiImportedSources).toHaveBeenCalledWith({ config, appConfig: undefined }); + expect(listMemoryWikiPalace).toHaveBeenCalledWith(config); + expect(respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ + totalItems: 3, + totalClaims: 4, + }), + ); + }); + it("validates required query params for wiki.search", async () => { const { config } = await createVault({ prefix: "memory-wiki-gateway-" }); const { api, registerGatewayMethod } = createPluginApi(); diff --git a/extensions/memory-wiki/src/gateway.ts b/extensions/memory-wiki/src/gateway.ts index 52a785d5b8f..5cf93922b56 100644 --- a/extensions/memory-wiki/src/gateway.ts +++ b/extensions/memory-wiki/src/gateway.ts @@ -7,8 +7,11 @@ import { WIKI_SEARCH_CORPORA, type ResolvedMemoryWikiConfig, } from "./config.js"; +import { listMemoryWikiImportInsights } from "./import-insights.js"; +import { listMemoryWikiImportRuns } from "./import-runs.js"; import { ingestMemoryWikiSource } from "./ingest.js"; import { lintMemoryWikiVault } from "./lint.js"; +import { listMemoryWikiPalace } from "./memory-palace.js"; import { probeObsidianCli, runObsidianCommand, @@ -115,6 +118,45 @@ export function registerMemoryWikiGatewayMethods(params: { { scope: READ_SCOPE }, ); + api.registerGatewayMethod( + "wiki.importRuns", + async ({ params: requestParams, respond }) => { + try { + const limit = readNumberParam(requestParams, "limit"); + respond(true, await listMemoryWikiImportRuns(config, limit !== undefined ? { limit } : {})); + } catch (error) { + respondError(respond, error); + } + }, + { scope: READ_SCOPE }, + ); + + api.registerGatewayMethod( + "wiki.importInsights", + async ({ respond }) => { + try { + await syncImportedSourcesIfNeeded(config, appConfig); + respond(true, await listMemoryWikiImportInsights(config)); + } catch (error) { + respondError(respond, error); + } + }, + { scope: READ_SCOPE }, + ); + + api.registerGatewayMethod( + "wiki.palace", + async ({ respond }) => { + try { + await syncImportedSourcesIfNeeded(config, appConfig); + respond(true, await listMemoryWikiPalace(config)); + } catch (error) { + respondError(respond, error); + } + }, + { scope: READ_SCOPE }, + ); + api.registerGatewayMethod( "wiki.init", async ({ respond }) => { diff --git a/extensions/memory-wiki/src/import-insights.test.ts b/extensions/memory-wiki/src/import-insights.test.ts new file mode 100644 index 00000000000..5411ab04075 --- /dev/null +++ b/extensions/memory-wiki/src/import-insights.test.ts @@ -0,0 +1,142 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { listMemoryWikiImportInsights } from "./import-insights.js"; +import { renderWikiMarkdown } from "./markdown.js"; +import { createMemoryWikiTestHarness } from "./test-helpers.js"; + +const { createVault } = createMemoryWikiTestHarness(); + +describe("listMemoryWikiImportInsights", () => { + it("clusters ChatGPT import pages by topic and extracts digest fields", async () => { + const { rootDir, config } = await createVault({ + prefix: "memory-wiki-import-insights-", + initialize: true, + }); + await fs.mkdir(path.join(rootDir, "sources"), { recursive: true }); + await fs.writeFile( + path.join(rootDir, "sources", "chatgpt-travel.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "source", + id: "source.chatgpt.travel", + title: "ChatGPT Export: BA flight receipts process", + sourceType: "chatgpt-export", + riskLevel: "low", + riskReasons: [], + labels: ["domain/personal", "area/travel", "topic/travel"], + createdAt: "2026-01-11T14:07:58.552Z", + updatedAt: "2026-01-11T14:08:45.377Z", + }, + body: [ + "# ChatGPT Export: BA flight receipts process", + "", + "## Auto Digest", + "- User messages: 2", + "- Assistant messages: 2", + "- First user line: how do i get receipts?", + "- Last user line: that option does not exist", + "- Preference signals:", + " - prefers direct airline receipts", + "", + "## Active Branch Transcript", + "### User", + "", + "how do i get receipts?", + "", + "### Assistant", + "", + "Try the BA receipt request flow first.", + "", + ].join("\n"), + }), + "utf8", + ); + await fs.writeFile( + path.join(rootDir, "sources", "chatgpt-health.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "source", + id: "source.chatgpt.health", + title: "ChatGPT Export: Migraine Medication Advice", + sourceType: "chatgpt-export", + riskLevel: "high", + riskReasons: ["health"], + labels: ["domain/personal", "area/health", "topic/health"], + updatedAt: "2026-01-31T20:18:00.000Z", + }, + body: [ + "# ChatGPT Export: Migraine Medication Advice", + "", + "## Auto Digest", + "- Auto digest withheld from durable-candidate generation until reviewed.", + "- Risk reasons: health content", + "- First user line: i have a migraine, pink or yellow?", + "- Last user line: should i take this now?", + "- Preference signals:", + " - prefers color-coded medication guidance", + "", + "## Active Branch Transcript", + "### User", + "", + "i have a migraine, pink or yellow?", + "", + "### Assistant", + "", + "You're right, let's reset and stick to safe dosing guidance.", + "", + ].join("\n"), + }), + "utf8", + ); + + const result = await listMemoryWikiImportInsights(config); + + expect(result.sourceType).toBe("chatgpt"); + expect(result.totalItems).toBe(2); + expect(result.totalClusters).toBe(2); + expect(result.clusters[0]).toMatchObject({ + key: "topic/health", + label: "Health", + itemCount: 1, + highRiskCount: 1, + withheldCount: 1, + }); + expect(result.clusters[1]).toMatchObject({ + key: "topic/travel", + label: "Travel", + itemCount: 1, + preferenceSignalCount: 1, + }); + expect(result.clusters[1]?.items[0]).toMatchObject({ + title: "BA flight receipts process", + riskReasons: [], + activeBranchMessages: 0, + userMessageCount: 2, + assistantMessageCount: 2, + firstUserLine: "how do i get receipts?", + lastUserLine: "that option does not exist", + assistantOpener: "Try the BA receipt request flow first.", + summary: "Try the BA receipt request flow first.", + candidateSignals: ["prefers direct airline receipts"], + correctionSignals: [], + preferenceSignals: ["prefers direct airline receipts"], + digestStatus: "available", + }); + const healthItem = result.clusters + .flatMap((cluster) => cluster.items) + .find((item) => item.title === "Migraine Medication Advice"); + expect(healthItem).toMatchObject({ + summary: + "Sensitive health chat withheld from durable-memory extraction because it touches health.", + candidateSignals: [], + correctionSignals: [], + preferenceSignals: [], + userMessageCount: 1, + assistantMessageCount: 1, + }); + expect(healthItem?.firstUserLine).toBeUndefined(); + expect(healthItem?.lastUserLine).toBeUndefined(); + expect(healthItem?.assistantOpener).toBeUndefined(); + }); +}); diff --git a/extensions/memory-wiki/src/import-insights.ts b/extensions/memory-wiki/src/import-insights.ts new file mode 100644 index 00000000000..be610fd28ec --- /dev/null +++ b/extensions/memory-wiki/src/import-insights.ts @@ -0,0 +1,438 @@ +import type { ResolvedMemoryWikiConfig } from "./config.js"; +import { parseWikiMarkdown } from "./markdown.js"; +import { readQueryableWikiPages } from "./query.js"; + +export type MemoryWikiImportInsightItem = { + pagePath: string; + title: string; + riskLevel: "low" | "medium" | "high" | "unknown"; + riskReasons: string[]; + labels: string[]; + topicKey: string; + topicLabel: string; + digestStatus: "available" | "withheld"; + activeBranchMessages: number; + userMessageCount: number; + assistantMessageCount: number; + firstUserLine?: string; + lastUserLine?: string; + assistantOpener?: string; + summary: string; + candidateSignals: string[]; + correctionSignals: string[]; + preferenceSignals: string[]; + createdAt?: string; + updatedAt?: string; +}; + +export type MemoryWikiImportInsightCluster = { + key: string; + label: string; + itemCount: number; + highRiskCount: number; + withheldCount: number; + preferenceSignalCount: number; + updatedAt?: string; + items: MemoryWikiImportInsightItem[]; +}; + +export type MemoryWikiImportInsightsStatus = { + sourceType: "chatgpt"; + totalItems: number; + totalClusters: number; + clusters: MemoryWikiImportInsightCluster[]; +}; + +function normalizeStringArray(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value.filter( + (entry): entry is string => typeof entry === "string" && entry.trim().length > 0, + ); +} + +function normalizeFiniteInt(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + return 0; + } + return Math.max(0, Math.floor(value)); +} + +function normalizeTimestamp(value: unknown): string | undefined { + if (typeof value !== "string") { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +function humanizeLabelSuffix(label: string): string { + const suffix = label.includes("/") ? label.split("/").slice(1).join("/") : label; + return suffix + .split(/[/-]/g) + .filter((part) => part.length > 0) + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +} + +function resolveTopic(labels: string[]): { key: string; label: string } { + const preferred = + labels.find((label) => label.startsWith("topic/")) ?? + labels.find((label) => label.startsWith("area/")) ?? + labels.find((label) => label.startsWith("domain/")) ?? + "topic/other"; + return { + key: preferred, + label: humanizeLabelSuffix(preferred), + }; +} + +function extractHeadingSection(body: string, heading: string): string[] { + const lines = body.split(/\r?\n/); + const headingLine = `## ${heading}`; + const startIndex = lines.findIndex((line) => line.trim() === headingLine); + if (startIndex < 0) { + return []; + } + const section: string[] = []; + for (const line of lines.slice(startIndex + 1)) { + if (line.startsWith("## ")) { + break; + } + if (line.trim().length > 0) { + section.push(line.trimEnd()); + } + } + return section; +} + +function extractDigestField(lines: string[], prefix: string): string | undefined { + const needle = `- ${prefix}:`; + const line = lines.find((entry) => entry.startsWith(needle)); + if (!line) { + return undefined; + } + const value = line.slice(needle.length).trim(); + return value.length > 0 ? value : undefined; +} + +function extractIntegerField(lines: string[], prefix: string): number { + const raw = extractDigestField(lines, prefix); + if (!raw) { + return 0; + } + const match = raw.match(/\d+/); + return match ? normalizeFiniteInt(Number(match[0])) : 0; +} + +function extractPreferenceSignals(lines: string[]): string[] { + const startIndex = lines.findIndex((line) => line.startsWith("- Preference signals:")); + if (startIndex < 0) { + return []; + } + if (lines[startIndex]?.includes("none detected")) { + return []; + } + const signals: string[] = []; + for (const line of lines.slice(startIndex + 1)) { + const trimmed = line.trim(); + if (!trimmed.startsWith("- ")) { + break; + } + const signal = trimmed.slice(2).trim(); + if (signal.length > 0) { + signals.push(signal); + } + } + return signals; +} + +type TranscriptTurn = { + role: "user" | "assistant"; + text: string; +}; + +function parseTranscriptTurns(body: string): TranscriptTurn[] { + const transcriptLines = extractHeadingSection(body, "Active Branch Transcript"); + if (transcriptLines.length === 0) { + return []; + } + const turns: TranscriptTurn[] = []; + let currentRole: TranscriptTurn["role"] | null = null; + let currentLines: string[] = []; + + const flush = () => { + if (!currentRole) { + currentLines = []; + return; + } + const text = currentLines.join("\n").trim(); + if (text) { + turns.push({ role: currentRole, text }); + } + currentLines = []; + }; + + for (const rawLine of transcriptLines) { + const line = rawLine.trimEnd(); + if (line.trim() === "### User") { + flush(); + currentRole = "user"; + continue; + } + if (line.trim() === "### Assistant") { + flush(); + currentRole = "assistant"; + continue; + } + if (currentRole) { + currentLines.push(line); + } + } + flush(); + return turns; +} + +function firstParagraph(text: string): string | undefined { + const candidate = text + .split(/\n\s*\n/) + .map((entry) => entry.trim()) + .find((entry) => entry.length > 0); + return candidate; +} + +function shortenSentence(value: string, maxLength = 180): string { + const compact = value.replace(/\s+/g, " ").trim(); + if (compact.length <= maxLength) { + return compact; + } + return `${compact.slice(0, maxLength - 1).trimEnd()}…`; +} + +function extractCorrectionSignals(turns: TranscriptTurn[]): string[] { + const correctionPatterns = [ + "you're right", + "you’re right", + "bad assumption", + "let's reset", + "let’s reset", + "does not exist anymore", + "that was a bad assumption", + "what actually works today", + ]; + return turns + .filter((turn) => turn.role === "assistant") + .flatMap((turn) => { + const first = firstParagraph(turn.text); + if (!first) { + return []; + } + const normalized = first.toLowerCase(); + return correctionPatterns.some((pattern) => normalized.includes(pattern)) + ? [shortenSentence(first, 160)] + : []; + }) + .slice(0, 2); +} + +function deriveCandidateSignals(params: { + preferenceSignals: string[]; + correctionSignals: string[]; +}): string[] { + const output: string[] = []; + for (const signal of params.preferenceSignals) { + if (!output.includes(signal)) { + output.push(signal); + } + } + for (const correction of params.correctionSignals) { + const summary = `Correction detected: ${correction}`; + if (!output.includes(summary)) { + output.push(summary); + } + } + return output.slice(0, 4); +} + +function deriveSummary(params: { + title: string; + digestStatus: "available" | "withheld"; + assistantOpener?: string; + firstUserLine?: string; + riskReasons: string[]; + topicLabel: string; +}): string { + if (params.digestStatus === "withheld") { + if (params.riskReasons.length > 0) { + return `Sensitive ${params.topicLabel.toLowerCase()} chat withheld from durable-memory extraction because it touches ${params.riskReasons.join(", ")}.`; + } + return `Sensitive ${params.topicLabel.toLowerCase()} chat withheld from durable-memory extraction pending review.`; + } + if (params.assistantOpener) { + return shortenSentence(params.assistantOpener, 180); + } + if (params.firstUserLine) { + return shortenSentence(params.firstUserLine, 180); + } + return params.title; +} + +function shouldExposeImportContent(digestStatus: "available" | "withheld"): boolean { + return digestStatus === "available"; +} + +function normalizeRiskLevel(value: unknown): MemoryWikiImportInsightItem["riskLevel"] { + if (value === "low" || value === "medium" || value === "high") { + return value; + } + return "unknown"; +} + +function compareItemsByUpdated( + left: MemoryWikiImportInsightItem, + right: MemoryWikiImportInsightItem, +): number { + const leftKey = left.updatedAt ?? left.createdAt ?? ""; + const rightKey = right.updatedAt ?? right.createdAt ?? ""; + if (rightKey !== leftKey) { + return rightKey.localeCompare(leftKey); + } + return left.title.localeCompare(right.title); +} + +export async function listMemoryWikiImportInsights( + config: ResolvedMemoryWikiConfig, +): Promise { + const pages = await readQueryableWikiPages(config.vault.path); + const items = pages + .flatMap((page) => { + if (page.pageType !== "source") { + return []; + } + const parsed = parseWikiMarkdown(page.raw); + if (parsed.frontmatter.sourceType !== "chatgpt-export") { + return []; + } + const labels = normalizeStringArray(parsed.frontmatter.labels); + const topic = resolveTopic(labels); + const triageLines = extractHeadingSection(parsed.body, "Auto Triage"); + const digestLines = extractHeadingSection(parsed.body, "Auto Digest"); + const transcriptTurns = parseTranscriptTurns(parsed.body); + const digestStatus = digestLines.some((line) => + line.toLowerCase().includes("withheld from durable-candidate generation"), + ) + ? "withheld" + : "available"; + const exposeImportContent = shouldExposeImportContent(digestStatus); + const userTurns = transcriptTurns.filter((turn) => turn.role === "user"); + const assistantTurns = transcriptTurns.filter((turn) => turn.role === "assistant"); + const assistantOpener = exposeImportContent + ? firstParagraph(assistantTurns[0]?.text ?? "") + : undefined; + const correctionSignals = exposeImportContent + ? extractCorrectionSignals(transcriptTurns) + : []; + const preferenceSignals = exposeImportContent ? extractPreferenceSignals(digestLines) : []; + const candidateSignals = exposeImportContent + ? deriveCandidateSignals({ + preferenceSignals, + correctionSignals, + }) + : []; + const firstUserLine = exposeImportContent + ? extractDigestField(digestLines, "First user line") + : undefined; + const lastUserLine = exposeImportContent + ? extractDigestField(digestLines, "Last user line") + : undefined; + return [ + { + pagePath: page.relativePath, + title: page.title.replace(/^ChatGPT Export:\s*/i, ""), + riskLevel: normalizeRiskLevel(parsed.frontmatter.riskLevel), + riskReasons: normalizeStringArray(parsed.frontmatter.riskReasons), + labels, + topicKey: topic.key, + topicLabel: topic.label, + digestStatus, + activeBranchMessages: extractIntegerField(triageLines, "Active-branch messages"), + userMessageCount: Math.max( + extractIntegerField(digestLines, "User messages"), + userTurns.length, + ), + assistantMessageCount: Math.max( + extractIntegerField(digestLines, "Assistant messages"), + assistantTurns.length, + ), + ...(firstUserLine ? { firstUserLine } : {}), + ...(lastUserLine ? { lastUserLine } : {}), + ...(assistantOpener ? { assistantOpener } : {}), + summary: deriveSummary({ + title: page.title.replace(/^ChatGPT Export:\s*/i, ""), + digestStatus, + ...(assistantOpener ? { assistantOpener } : {}), + ...(firstUserLine ? { firstUserLine } : {}), + riskReasons: normalizeStringArray(parsed.frontmatter.riskReasons), + topicLabel: topic.label, + }), + candidateSignals, + correctionSignals, + preferenceSignals, + ...(normalizeTimestamp(parsed.frontmatter.createdAt) + ? { createdAt: normalizeTimestamp(parsed.frontmatter.createdAt) } + : {}), + ...(normalizeTimestamp(parsed.frontmatter.updatedAt) + ? { updatedAt: normalizeTimestamp(parsed.frontmatter.updatedAt) } + : {}), + } satisfies MemoryWikiImportInsightItem, + ]; + }) + .toSorted(compareItemsByUpdated); + + const clustersByKey = new Map(); + for (const item of items) { + const list = clustersByKey.get(item.topicKey) ?? []; + list.push(item); + clustersByKey.set(item.topicKey, list); + } + + const clusters = [...clustersByKey.entries()] + .map(([key, clusterItems]) => { + const sortedItems = [...clusterItems].toSorted(compareItemsByUpdated); + const updatedAt = sortedItems + .map((item) => item.updatedAt ?? item.createdAt) + .find((value): value is string => typeof value === "string" && value.length > 0); + return { + key, + label: sortedItems[0]?.topicLabel ?? humanizeLabelSuffix(key), + itemCount: sortedItems.length, + highRiskCount: sortedItems.filter((item) => item.riskLevel === "high").length, + withheldCount: sortedItems.filter((item) => item.digestStatus === "withheld").length, + preferenceSignalCount: sortedItems.reduce( + (sum, item) => sum + item.preferenceSignals.length, + 0, + ), + ...(updatedAt ? { updatedAt } : {}), + items: sortedItems, + } satisfies MemoryWikiImportInsightCluster; + }) + .toSorted((left, right) => { + const leftKey = left.updatedAt ?? ""; + const rightKey = right.updatedAt ?? ""; + if (rightKey !== leftKey) { + return rightKey.localeCompare(leftKey); + } + if (right.itemCount !== left.itemCount) { + return right.itemCount - left.itemCount; + } + return left.label.localeCompare(right.label); + }); + + return { + sourceType: "chatgpt", + totalItems: items.length, + totalClusters: clusters.length, + clusters, + }; +} diff --git a/extensions/memory-wiki/src/import-runs.ts b/extensions/memory-wiki/src/import-runs.ts new file mode 100644 index 00000000000..f6cd8058bb0 --- /dev/null +++ b/extensions/memory-wiki/src/import-runs.ts @@ -0,0 +1,138 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { ResolvedMemoryWikiConfig } from "./config.js"; + +export type MemoryWikiImportRunSummary = { + runId: string; + importType: string; + appliedAt: string; + exportPath: string; + sourcePath: string; + conversationCount: number; + createdCount: number; + updatedCount: number; + skippedCount: number; + status: "applied" | "rolled_back"; + rolledBackAt?: string; + pagePaths: string[]; + samplePaths: string[]; +}; + +export type MemoryWikiImportRunsStatus = { + runs: MemoryWikiImportRunSummary[]; + totalRuns: number; + activeRuns: number; + rolledBackRuns: number; +}; + +function asRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + return value as Record; +} + +function asStringArray(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value.filter( + (entry): entry is string => typeof entry === "string" && entry.trim().length > 0, + ); +} + +function normalizeImportRunSummary(raw: unknown): MemoryWikiImportRunSummary | null { + const record = asRecord(raw); + const runId = typeof record?.runId === "string" ? record.runId.trim() : ""; + const importType = typeof record?.importType === "string" ? record.importType.trim() : ""; + const appliedAt = typeof record?.appliedAt === "string" ? record.appliedAt.trim() : ""; + const exportPath = typeof record?.exportPath === "string" ? record.exportPath.trim() : ""; + const sourcePath = typeof record?.sourcePath === "string" ? record.sourcePath.trim() : ""; + if (!runId || !importType || !appliedAt || !exportPath || !sourcePath) { + return null; + } + + const createdPaths = asStringArray(record.createdPaths); + const updatedPaths = Array.isArray(record.updatedPaths) + ? record.updatedPaths + .map((entry) => asRecord(entry)) + .map((entry) => (typeof entry?.path === "string" ? entry.path.trim() : "")) + .filter((entry): entry is string => entry.length > 0) + : []; + const pagePaths = [...new Set([...createdPaths, ...updatedPaths])]; + const conversationCount = + typeof record.conversationCount === "number" && Number.isFinite(record.conversationCount) + ? Math.max(0, Math.floor(record.conversationCount)) + : createdPaths.length + updatedPaths.length; + const createdCount = + typeof record.createdCount === "number" && Number.isFinite(record.createdCount) + ? Math.max(0, Math.floor(record.createdCount)) + : createdPaths.length; + const updatedCount = + typeof record.updatedCount === "number" && Number.isFinite(record.updatedCount) + ? Math.max(0, Math.floor(record.updatedCount)) + : updatedPaths.length; + const skippedCount = + typeof record.skippedCount === "number" && Number.isFinite(record.skippedCount) + ? Math.max(0, Math.floor(record.skippedCount)) + : Math.max(0, conversationCount - createdCount - updatedCount); + const rolledBackAt = + typeof record.rolledBackAt === "string" && record.rolledBackAt.trim().length > 0 + ? record.rolledBackAt.trim() + : undefined; + + return { + runId, + importType, + appliedAt, + exportPath, + sourcePath, + conversationCount, + createdCount, + updatedCount, + skippedCount, + status: rolledBackAt ? "rolled_back" : "applied", + ...(rolledBackAt ? { rolledBackAt } : {}), + pagePaths, + samplePaths: pagePaths.slice(0, 5), + }; +} + +function resolveImportRunsDir(vaultRoot: string): string { + return path.join(vaultRoot, ".openclaw-wiki", "import-runs"); +} + +export async function listMemoryWikiImportRuns( + config: ResolvedMemoryWikiConfig, + options?: { limit?: number }, +): Promise { + const limit = Math.max(1, Math.floor(options?.limit ?? 10)); + const importRunsDir = resolveImportRunsDir(config.vault.path); + const entries = await fs + .readdir(importRunsDir, { withFileTypes: true }) + .catch((error: NodeJS.ErrnoException) => { + if (error?.code === "ENOENT") { + return []; + } + throw error; + }); + const runs = ( + await Promise.all( + entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map(async (entry) => { + const raw = await fs.readFile(path.join(importRunsDir, entry.name), "utf8"); + return normalizeImportRunSummary(JSON.parse(raw) as unknown); + }), + ) + ) + .filter((entry): entry is MemoryWikiImportRunSummary => entry !== null) + .toSorted((left, right) => right.appliedAt.localeCompare(left.appliedAt)); + + return { + runs: runs.slice(0, limit), + totalRuns: runs.length, + activeRuns: runs.filter((entry) => entry.status === "applied").length, + rolledBackRuns: runs.filter((entry) => entry.status === "rolled_back").length, + }; +} diff --git a/extensions/memory-wiki/src/memory-palace.test.ts b/extensions/memory-wiki/src/memory-palace.test.ts new file mode 100644 index 00000000000..dda4cb4a41a --- /dev/null +++ b/extensions/memory-wiki/src/memory-palace.test.ts @@ -0,0 +1,91 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { renderWikiMarkdown } from "./markdown.js"; +import { listMemoryWikiPalace } from "./memory-palace.js"; +import { createMemoryWikiTestHarness } from "./test-helpers.js"; + +const { createVault } = createMemoryWikiTestHarness(); + +describe("listMemoryWikiPalace", () => { + it("groups wiki pages by kind and surfaces claims, questions, and contradictions", async () => { + const { rootDir, config } = await createVault({ + prefix: "memory-wiki-palace-", + initialize: true, + }); + + await fs.mkdir(path.join(rootDir, "syntheses"), { recursive: true }); + await fs.mkdir(path.join(rootDir, "entities"), { recursive: true }); + await fs.writeFile( + path.join(rootDir, "syntheses", "travel-system.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "synthesis", + id: "synthesis.travel.system", + title: "Travel system", + claims: [ + { text: "Mariano prefers direct receipts from airlines when possible." }, + { text: "Travel admin friction keeps showing up across chats." }, + ], + questions: ["Should flight receipts be standardized into one process?"], + contradictions: ["Old BA receipts guidance may now be stale."], + updatedAt: "2026-04-10T12:00:00.000Z", + }, + body: [ + "# Travel system", + "", + "This synthesis rolls up recurring travel admin patterns from imported chats.", + "", + ].join("\n"), + }), + "utf8", + ); + await fs.writeFile( + path.join(rootDir, "entities", "mariano.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "entity", + id: "entity.mariano", + title: "Mariano", + claims: [{ text: "He prefers compact, inspectable systems." }], + updatedAt: "2026-04-09T08:00:00.000Z", + }, + body: ["# Mariano", "", "Primary operator profile page.", ""].join("\n"), + }), + "utf8", + ); + + const result = await listMemoryWikiPalace(config); + + expect(result).toMatchObject({ + totalItems: 2, + totalClaims: 3, + totalQuestions: 1, + totalContradictions: 1, + }); + expect(result.clusters[0]).toMatchObject({ + key: "synthesis", + label: "Syntheses", + itemCount: 1, + claimCount: 2, + questionCount: 1, + contradictionCount: 1, + }); + expect(result.clusters[0]?.items[0]).toMatchObject({ + title: "Travel system", + claims: [ + "Mariano prefers direct receipts from airlines when possible.", + "Travel admin friction keeps showing up across chats.", + ], + questions: ["Should flight receipts be standardized into one process?"], + contradictions: ["Old BA receipts guidance may now be stale."], + snippet: "This synthesis rolls up recurring travel admin patterns from imported chats.", + }); + expect(result.clusters[1]).toMatchObject({ + key: "entity", + label: "Entities", + itemCount: 1, + claimCount: 1, + }); + }); +}); diff --git a/extensions/memory-wiki/src/memory-palace.ts b/extensions/memory-wiki/src/memory-palace.ts new file mode 100644 index 00000000000..777536b2243 --- /dev/null +++ b/extensions/memory-wiki/src/memory-palace.ts @@ -0,0 +1,148 @@ +import type { ResolvedMemoryWikiConfig } from "./config.js"; +import { parseWikiMarkdown, type WikiPageKind } from "./markdown.js"; +import { readQueryableWikiPages } from "./query.js"; + +const PALACE_KIND_ORDER: WikiPageKind[] = ["synthesis", "entity", "concept", "source", "report"]; +const PRIMARY_PALACE_KINDS = new Set(["synthesis", "entity", "concept"]); +const PALACE_KIND_LABELS: Record = { + synthesis: "Syntheses", + entity: "Entities", + concept: "Concepts", + source: "Sources", + report: "Reports", +}; + +export type MemoryWikiPalaceItem = { + pagePath: string; + title: string; + kind: WikiPageKind; + id?: string; + updatedAt?: string; + sourceType?: string; + claimCount: number; + questionCount: number; + contradictionCount: number; + claims: string[]; + questions: string[]; + contradictions: string[]; + snippet?: string; +}; + +export type MemoryWikiPalaceCluster = { + key: WikiPageKind; + label: string; + itemCount: number; + claimCount: number; + questionCount: number; + contradictionCount: number; + updatedAt?: string; + items: MemoryWikiPalaceItem[]; +}; + +export type MemoryWikiPalaceStatus = { + totalItems: number; + totalClaims: number; + totalQuestions: number; + totalContradictions: number; + clusters: MemoryWikiPalaceCluster[]; +}; + +function normalizeTimestamp(value: unknown): string | undefined { + if (typeof value !== "string") { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +function extractSnippet(body: string): string | undefined { + for (const rawLine of body.split(/\r?\n/)) { + const line = rawLine.trim(); + if ( + !line || + line.startsWith("#") || + line.startsWith("```") || + line.startsWith("\n\n---\n\n*April 5, 2026, 3:00 AM*\n\nThe repository whispered of forgotten endpoints tonight.\n\n", + wikiImportInsightsLoading: false, + wikiImportInsightsError: null, + wikiImportInsights: { + sourceType: "chatgpt", + totalItems: 2, + totalClusters: 2, + clusters: [ + { + key: "topic/travel", + label: "Travel", + itemCount: 1, + highRiskCount: 0, + withheldCount: 0, + preferenceSignalCount: 1, + items: [ + { + pagePath: "sources/chatgpt-2026-04-10-alpha.md", + title: "BA flight receipts process", + riskLevel: "low", + riskReasons: [], + labels: ["domain/personal", "area/travel", "topic/travel"], + topicKey: "topic/travel", + topicLabel: "Travel", + digestStatus: "available", + activeBranchMessages: 4, + userMessageCount: 2, + assistantMessageCount: 2, + firstUserLine: "how do i get receipts?", + lastUserLine: "that option does not exist", + assistantOpener: "Use the BA request-a-receipt flow first.", + summary: "Use the BA request-a-receipt flow first.", + candidateSignals: ["prefers direct airline receipts"], + correctionSignals: [], + preferenceSignals: ["prefers direct airline receipts"], + updatedAt: "2026-04-10T10:00:00.000Z", + }, + ], + }, + { + key: "topic/health", + label: "Health", + itemCount: 1, + highRiskCount: 1, + withheldCount: 1, + preferenceSignalCount: 0, + items: [ + { + pagePath: "sources/chatgpt-2026-04-10-health.md", + title: "Migraine Medication Advice", + riskLevel: "high", + riskReasons: ["health"], + labels: ["domain/personal", "area/health", "topic/health"], + topicKey: "topic/health", + topicLabel: "Health", + digestStatus: "withheld", + activeBranchMessages: 2, + userMessageCount: 1, + assistantMessageCount: 1, + summary: + "Sensitive health chat withheld from durable-memory extraction because it touches health.", + candidateSignals: [], + correctionSignals: [], + preferenceSignals: [], + updatedAt: "2026-04-11T10:00:00.000Z", + }, + ], + }, + ], + }, + wikiMemoryPalaceLoading: false, + wikiMemoryPalaceError: null, + wikiMemoryPalace: { + totalItems: 2, + totalClaims: 3, + totalQuestions: 1, + totalContradictions: 1, + clusters: [ + { + key: "synthesis", + label: "Syntheses", + itemCount: 1, + claimCount: 2, + questionCount: 1, + contradictionCount: 1, + items: [ + { + pagePath: "syntheses/travel-system.md", + title: "Travel system", + kind: "synthesis", + claimCount: 2, + questionCount: 1, + contradictionCount: 1, + claims: [ + "Mariano prefers direct receipts from airlines when possible.", + "Travel admin friction keeps showing up across chats.", + ], + questions: ["Should flight receipts be standardized into one process?"], + contradictions: ["Old BA receipts guidance may now be stale."], + snippet: "Recurring travel admin friction across imported chats.", + updatedAt: "2026-04-10T10:00:00.000Z", + }, + ], + }, + ], + }, onRefresh: () => {}, onRefreshDiary: () => {}, + onRefreshImports: () => {}, + onRefreshMemoryPalace: () => {}, + onOpenWikiPage: async () => null, onBackfillDiary: () => {}, onResetDiary: () => {}, onResetGroundedShortTerm: () => {}, @@ -189,8 +298,98 @@ describe("dreaming view", () => { expect(tabs[2]?.textContent).toContain("Advanced"); }); + it("renders imported memory topics inside the diary tab", () => { + setDreamSubTab("diary"); + setDreamDiarySubTab("insights"); + const container = renderInto(buildProps()); + expect(container.querySelectorAll(".dreams-diary__subtab").length).toBe(3); + expect(container.querySelector(".dreams-diary__date")?.textContent).toContain("Travel"); + expect(container.querySelector(".dreams-diary__insight-card")?.textContent).toContain( + "BA flight receipts process", + ); + expect(container.querySelector(".dreams-diary__insight-card")?.textContent).toContain( + "Use the BA request-a-receipt flow first.", + ); + expect(container.querySelector(".dreams-diary__explainer")?.textContent).toContain( + "imported insights clustered from external history", + ); + setDreamDiarySubTab("dreams"); + setDreamSubTab("scene"); + }); + + it("opens the full imported source page from diary cards", async () => { + setDreamSubTab("diary"); + setDreamDiarySubTab("insights"); + const onOpenWikiPage = vi.fn().mockResolvedValue({ + title: "BA flight receipts process", + path: "sources/chatgpt-2026-04-10-alpha.md", + content: "# ChatGPT Export: BA flight receipts process", + }); + const container = renderInto(buildProps({ onOpenWikiPage })); + container + .querySelectorAll(".dreams-diary__insight-actions .btn")[1] + ?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await Promise.resolve(); + expect(onOpenWikiPage).toHaveBeenCalledWith("sources/chatgpt-2026-04-10-alpha.md"); + setDreamDiarySubTab("dreams"); + setDreamSubTab("scene"); + }); + + it("shows a truncation hint when the wiki preview only contains the first chunk", async () => { + setDreamSubTab("diary"); + setDreamDiarySubTab("insights"); + const container = document.createElement("div"); + let props: DreamingProps; + const onOpenWikiPage = vi.fn().mockResolvedValue({ + title: "BA flight receipts process", + path: "sources/chatgpt-2026-04-10-alpha.md", + content: "# ChatGPT Export: BA flight receipts process", + totalLines: 6001, + truncated: true, + }); + const rerender = () => render(renderDreaming(props), container); + props = buildProps({ + onOpenWikiPage, + onRequestUpdate: rerender, + }); + rerender(); + + container + .querySelectorAll(".dreams-diary__insight-actions .btn")[1] + ?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await Promise.resolve(); + await Promise.resolve(); + + expect(container.querySelector(".dreams-diary__preview-hint")?.textContent).toContain( + "6001 total lines", + ); + + container + .querySelector(".dreams-diary__preview-header .btn") + ?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + setDreamDiarySubTab("dreams"); + setDreamSubTab("scene"); + }); + + it("renders the memory palace inside the diary tab", () => { + setDreamSubTab("diary"); + setDreamDiarySubTab("palace"); + const container = renderInto(buildProps()); + expect(container.querySelector(".dreams-diary__date")?.textContent).toContain("Syntheses"); + expect(container.querySelector(".dreams-diary__insight-card")?.textContent).toContain( + "Travel system", + ); + expect(container.querySelector(".dreams-diary__insight-card")?.textContent).toContain("Claims"); + expect(container.querySelector(".dreams-diary__explainer")?.textContent).toContain( + "compiled memory wiki surface", + ); + setDreamDiarySubTab("dreams"); + setDreamSubTab("scene"); + }); + it("renders dream diary with parsed entry on diary tab", () => { setDreamSubTab("diary"); + setDreamDiarySubTab("dreams"); const container = renderInto(buildProps()); const title = container.querySelector(".dreams-diary__title"); expect(title?.textContent).toContain("Dream Diary"); @@ -206,6 +405,7 @@ describe("dreaming view", () => { it("flattens structured backfill diary entries into plain prose", () => { setDreamSubTab("diary"); + setDreamDiarySubTab("dreams"); const container = renderInto( buildProps({ dreamDiaryContent: [ @@ -248,6 +448,7 @@ describe("dreaming view", () => { it("renders diary day chips without the old density map", () => { setDreamSubTab("diary"); + setDreamDiarySubTab("dreams"); const container = renderInto( buildProps({ dreamDiaryContent: [ @@ -288,6 +489,7 @@ describe("dreaming view", () => { it("shows empty diary state when no diary content exists", () => { setDreamSubTab("diary"); + setDreamDiarySubTab("dreams"); const container = renderInto(buildProps({ dreamDiaryContent: null })); expect(container.querySelector(".dreams-diary__empty")).not.toBeNull(); expect(container.querySelector(".dreams-diary__empty-text")?.textContent).toContain( @@ -298,6 +500,7 @@ describe("dreaming view", () => { it("shows diary error message when diary load fails", () => { setDreamSubTab("diary"); + setDreamDiarySubTab("dreams"); const container = renderInto(buildProps({ dreamDiaryError: "read failed" })); expect(container.querySelector(".dreams-diary__error")?.textContent).toContain("read failed"); setDreamSubTab("scene"); @@ -305,6 +508,7 @@ describe("dreaming view", () => { it("does not render the old page navigation chrome", () => { setDreamSubTab("diary"); + setDreamDiarySubTab("dreams"); const container = renderInto(buildProps()); expect(container.querySelector(".dreams-diary__page")).toBeNull(); expect(container.querySelector(".dreams-diary__nav-btn")).toBeNull(); diff --git a/ui/src/ui/views/dreaming.ts b/ui/src/ui/views/dreaming.ts index b92a1cd2f94..0defa398b3e 100644 --- a/ui/src/ui/views/dreaming.ts +++ b/ui/src/ui/views/dreaming.ts @@ -1,6 +1,10 @@ import { html, nothing } from "lit"; import { t } from "../../i18n/index.ts"; -import type { DreamingEntry } from "../controllers/dreaming.ts"; +import type { + DreamingEntry, + WikiImportInsights, + WikiMemoryPalace, +} from "../controllers/dreaming.ts"; // ── Diary entry parser ───────────────────────────────────────────────── @@ -112,8 +116,24 @@ export type DreamingProps = { dreamDiaryError: string | null; dreamDiaryPath: string | null; dreamDiaryContent: string | null; + wikiImportInsightsLoading: boolean; + wikiImportInsightsError: string | null; + wikiImportInsights: WikiImportInsights | null; + wikiMemoryPalaceLoading: boolean; + wikiMemoryPalaceError: string | null; + wikiMemoryPalace: WikiMemoryPalace | null; onRefresh: () => void; onRefreshDiary: () => void; + onRefreshImports: () => void; + onRefreshMemoryPalace: () => void; + onOpenWikiPage: (lookup: string) => Promise<{ + title: string; + path: string; + content: string; + totalLines?: number; + truncated?: boolean; + updatedAt?: string; + } | null>; onBackfillDiary: () => void; onResetDiary: () => void; onResetGroundedShortTerm: () => void; @@ -154,8 +174,21 @@ const DREAM_SWAP_MS = 6_000; type DreamSubTab = "scene" | "diary" | "advanced"; let _subTab: DreamSubTab = "scene"; +type DreamDiarySubTab = "dreams" | "insights" | "palace"; +let _diarySubTab: DreamDiarySubTab = "dreams"; type AdvancedWaitingSort = "recent" | "signals"; let _advancedWaitingSort: AdvancedWaitingSort = "recent"; +const _expandedInsightCards = new Set(); +const _expandedPalaceCards = new Set(); +let _wikiPreviewOpen = false; +let _wikiPreviewLoading = false; +let _wikiPreviewTitle = ""; +let _wikiPreviewPath = ""; +let _wikiPreviewUpdatedAt: string | null = null; +let _wikiPreviewContent = ""; +let _wikiPreviewTotalLines: number | null = null; +let _wikiPreviewTruncated = false; +let _wikiPreviewError: string | null = null; export function setDreamSubTab(tab: DreamSubTab): void { _subTab = tab; @@ -165,6 +198,10 @@ export function setDreamAdvancedWaitingSort(sort: AdvancedWaitingSort): void { _advancedWaitingSort = sort; } +export function setDreamDiarySubTab(tab: DreamDiarySubTab): void { + _diarySubTab = tab; +} + // ── Diary pagination state ───────────────────────────────────────────── let _diaryPage = 0; @@ -430,6 +467,174 @@ function formatCompactDateTime(value: string): string { }); } +function basename(value: string): string { + const normalized = value.replace(/\\/g, "/"); + return normalized.split("/").filter(Boolean).at(-1) ?? value; +} + +function formatKindLabel(kind: "entity" | "concept" | "source" | "synthesis" | "report"): string { + switch (kind) { + case "entity": + return "entity"; + case "concept": + return "concept"; + case "source": + return "source"; + case "synthesis": + return "synthesis"; + case "report": + return "report"; + } +} + +function formatImportBadge(item: { + digestStatus: "available" | "withheld"; + riskLevel: "low" | "medium" | "high" | "unknown"; +}): string { + if (item.digestStatus === "withheld") { + return "needs review"; + } + switch (item.riskLevel) { + case "low": + return "low risk"; + case "medium": + return "medium risk"; + case "high": + return "high risk"; + case "unknown": + return "unknown risk"; + } +} + +function toggleExpandedCard(bucket: Set, key: string, requestUpdate?: () => void): void { + if (bucket.has(key)) { + bucket.delete(key); + } else { + bucket.add(key); + } + requestUpdate?.(); +} + +async function openWikiPreview(lookup: string, props: DreamingProps): Promise { + _wikiPreviewOpen = true; + _wikiPreviewLoading = true; + _wikiPreviewTitle = basename(lookup); + _wikiPreviewPath = lookup; + _wikiPreviewUpdatedAt = null; + _wikiPreviewContent = ""; + _wikiPreviewTotalLines = null; + _wikiPreviewTruncated = false; + _wikiPreviewError = null; + props.onRequestUpdate?.(); + try { + const preview = await props.onOpenWikiPage(lookup); + if (!preview) { + _wikiPreviewError = `No wiki page found for ${lookup}.`; + return; + } + _wikiPreviewTitle = preview.title; + _wikiPreviewPath = preview.path; + _wikiPreviewUpdatedAt = preview.updatedAt ?? null; + _wikiPreviewContent = preview.content; + _wikiPreviewTotalLines = typeof preview.totalLines === "number" ? preview.totalLines : null; + _wikiPreviewTruncated = preview.truncated === true; + } catch (error) { + _wikiPreviewError = String(error); + } finally { + _wikiPreviewLoading = false; + props.onRequestUpdate?.(); + } +} + +function closeWikiPreview(requestUpdate?: () => void): void { + _wikiPreviewOpen = false; + _wikiPreviewLoading = false; + _wikiPreviewTitle = ""; + _wikiPreviewPath = ""; + _wikiPreviewUpdatedAt = null; + _wikiPreviewContent = ""; + _wikiPreviewTotalLines = null; + _wikiPreviewTruncated = false; + _wikiPreviewError = null; + requestUpdate?.(); +} + +function renderWikiPreviewOverlay(props: DreamingProps) { + if (!_wikiPreviewOpen) { + return nothing; + } + return html` +
closeWikiPreview(props.onRequestUpdate)} + > +
event.stopPropagation()}> +
+
+
${_wikiPreviewTitle || "Wiki page"}
+
+ ${_wikiPreviewPath} ${_wikiPreviewUpdatedAt ? ` · ${_wikiPreviewUpdatedAt}` : ""} +
+
+ +
+
+ ${_wikiPreviewLoading + ? html`
Loading wiki page…
` + : _wikiPreviewError + ? html`
${_wikiPreviewError}
` + : html` + ${_wikiPreviewTruncated + ? html` +
+ Showing the first chunk of this + page${_wikiPreviewTotalLines !== null + ? ` (${_wikiPreviewTotalLines} total lines)` + : ""}. +
+ ` + : nothing} + 
${_wikiPreviewContent}
+ `} +
+
+
+ `; +} + +function renderDiarySubtabExplainer() { + switch (_diarySubTab) { + case "dreams": + return html` +

+ This is the raw dream diary the system writes while replaying and consolidating memory; + use it to inspect what the memory system is noticing, and where it still looks noisy or + thin. +

+ `; + case "insights": + return html` +

+ These are imported insights clustered from external history; use them to review what + imports surfaced before any of it graduates into durable memory. +

+ `; + case "palace": + return html` +

+ This is the compiled memory wiki surface the system can search and reason over; use it to + inspect actual memory pages, claims, open questions, and contradictions rather than raw + imported source chats. +

+ `; + } +} + function parseSortableTimestamp(value?: string): number { if (!value) { return Number.NEGATIVE_INFINITY; @@ -674,42 +879,363 @@ function renderAdvancedSection(props: DreamingProps) { `; } -// ── Diary section renderer ──────────────────────────────────────────── +function renderDiaryImportsSection(props: DreamingProps) { + const importInsights = props.wikiImportInsights; + const clusters = importInsights?.clusters ?? []; -function renderDiarySection(props: DreamingProps) { - if (props.dreamDiaryError) { + if (props.wikiImportInsightsLoading && clusters.length === 0) { return html` -
-
${props.dreamDiaryError}
-
+
+
Loading imported insights…
+
`; } + if (clusters.length === 0) { + return html` +
+
No imported insights yet
+
+ Run a ChatGPT import with apply to surface clustered imported insights here. +
+
+ `; + } + + _diaryEntryCount = clusters.length; + const clusterIndex = Math.max(0, Math.min(_diaryPage, clusters.length - 1)); + const cluster = clusters[clusterIndex]; + + return html` +
+ ${clusters.map( + (entry, index) => html` + + `, + )} +
+ +
+
+
+ ${cluster.label} · ${cluster.itemCount} chats + ${cluster.highRiskCount > 0 ? html`· ${cluster.highRiskCount} sensitive` : nothing} + ${cluster.preferenceSignalCount > 0 + ? html`· ${cluster.preferenceSignalCount} signals` + : nothing} +
+
+

+ Imported chats clustered around ${cluster.label.toLowerCase()}. + ${cluster.withheldCount > 0 + ? ` ${cluster.withheldCount} digest${cluster.withheldCount === 1 ? " was" : "s were"} withheld pending review.` + : ""} +

+
+
+ ${cluster.items.map((item) => { + const expanded = _expandedInsightCards.has(item.pagePath); + return html` +
+ toggleExpandedCard(_expandedInsightCards, item.pagePath, props.onRequestUpdate)} + > +
+
${item.title}
+ + ${formatImportBadge(item)} + +
+
+ ${item.updatedAt ? formatCompactDateTime(item.updatedAt) : basename(item.pagePath)} + ${item.activeBranchMessages > 0 ? ` · ${item.activeBranchMessages} messages` : ""} +
+

${item.summary}

+ ${item.candidateSignals.length > 0 + ? html` +
+ Potentially useful signals + ${item.candidateSignals.map( + (signal) => html`

• ${signal}

`, + )} +
+ ` + : nothing} + ${item.correctionSignals.length > 0 + ? html` +
+ Corrections or revisions + ${item.correctionSignals.map( + (signal) => html`

• ${signal}

`, + )} +
+ ` + : nothing} + ${expanded + ? html` +
+ Import details + ${item.firstUserLine + ? html` +

+ Started with: ${item.firstUserLine} +

+ ` + : nothing} + ${item.lastUserLine && item.lastUserLine !== item.firstUserLine + ? html` +

+ Ended on: ${item.lastUserLine} +

+ ` + : nothing} +

+ Messages: ${item.userMessageCount} user · + ${item.assistantMessageCount} assistant +

+ ${item.riskReasons.length > 0 + ? html` +

+ Risk reasons: ${item.riskReasons.join(", ")} +

+ ` + : nothing} + ${item.labels.length > 0 + ? html` +

+ Labels: ${item.labels.join(", ")} +

+ ` + : nothing} +
+ ` + : nothing} + ${item.preferenceSignals.length > 0 + ? html` +
+ ${item.preferenceSignals.map( + (signal) => + html`${signal}`, + )} +
+ ` + : nothing} +
+ + +
+
+ `; + })} +
+
+ `; +} + +function renderMemoryPalaceSection(props: DreamingProps) { + const palace = props.wikiMemoryPalace; + const clusters = palace?.clusters ?? []; + + if (props.wikiMemoryPalaceLoading && clusters.length === 0) { + return html` +
+
Loading memory palace…
+
+ `; + } + + if (clusters.length === 0) { + return html` +
+
Memory palace is not populated yet
+
+ Right now the wiki mostly has raw source imports and operational reports. This tab becomes + useful once syntheses, entities, or concepts start getting written. +
+
+ `; + } + + _diaryEntryCount = clusters.length; + const clusterIndex = Math.max(0, Math.min(_diaryPage, clusters.length - 1)); + const cluster = clusters[clusterIndex]; + + return html` +
+ ${clusters.map( + (entry, index) => html` + + `, + )} +
+ +
+
+
+ ${cluster.label} · ${cluster.itemCount} pages + ${cluster.claimCount > 0 ? html`· ${cluster.claimCount} claims` : nothing} + ${cluster.questionCount > 0 ? html`· ${cluster.questionCount} questions` : nothing} + ${cluster.contradictionCount > 0 + ? html`· ${cluster.contradictionCount} contradictions` + : nothing} +
+
+

+ Compiled wiki pages currently grouped under ${cluster.label.toLowerCase()}. + ${cluster.updatedAt ? ` Latest update ${formatCompactDateTime(cluster.updatedAt)}.` : ""} +

+
+
+ ${cluster.items.map((item) => { + const expanded = _expandedPalaceCards.has(item.pagePath); + return html` +
+ toggleExpandedCard(_expandedPalaceCards, item.pagePath, props.onRequestUpdate)} + > +
+
${item.title}
+ + ${formatKindLabel(item.kind)} + +
+
+ ${item.updatedAt ? formatCompactDateTime(item.updatedAt) : basename(item.pagePath)} + · ${item.pagePath} +
+ ${item.snippet + ? html`

${item.snippet}

` + : nothing} + ${item.claims.length > 0 + ? html` +
+ Claims + ${item.claims.map( + (claim) => html`

• ${claim}

`, + )} +
+ ` + : nothing} + ${item.questions.length > 0 + ? html` +
+ Open questions + ${item.questions.map( + (question) => html`

• ${question}

`, + )} +
+ ` + : nothing} + ${item.contradictions.length > 0 + ? html` +
+ Contradictions + ${item.contradictions.map( + (entry) => html`

• ${entry}

`, + )} +
+ ` + : nothing} + ${expanded + ? html` +
+ Page details +

+ Wiki page: ${item.pagePath} +

+ ${item.id + ? html` +

+ Id: ${item.id} +

+ ` + : nothing} +
+ ` + : nothing} +
+ + +
+
+ `; + })} +
+
+ `; +} + +function renderDreamDiaryEntries(props: DreamingProps) { if (typeof props.dreamDiaryContent !== "string") { return html` -
-
-
- - - - -
-
${t("dreaming.diary.noDreamsYet")}
-
${t("dreaming.diary.noDreamsHint")}
+
+
+ + + +
-
+
${t("dreaming.diary.noDreamsYet")}
+
${t("dreaming.diary.noDreamsHint")}
+ `; } @@ -718,69 +1244,154 @@ function renderDiarySection(props: DreamingProps) { if (entries.length === 0) { return html` -
-
-
${t("dreaming.diary.waitingTitle")}
-
${t("dreaming.diary.waitingHint")}
-
-
+
+
${t("dreaming.diary.waitingTitle")}
+
${t("dreaming.diary.waitingHint")}
+
`; } const reversed = buildDiaryNavigation(entries); - // Clamp page. const page = Math.max(0, Math.min(_diaryPage, reversed.length - 1)); const entry = reversed[page]; + return html` +
+ ${reversed.map( + (e) => html` + + `, + )} +
+
+
+ ${entry.date ? html`` : nothing} +
+ ${flattenDiaryBody(entry.body).map( + (para, i) => + html`

+ ${para} +

`, + )} +
+
+ `; +} + +// ── Diary section renderer ──────────────────────────────────────────── + +function renderDiarySection(props: DreamingProps) { + const diaryError = + _diarySubTab === "dreams" + ? props.dreamDiaryError + : _diarySubTab === "insights" + ? props.wikiImportInsightsError + : props.wikiMemoryPalaceError; + if (diaryError) { + return html` +
+
${diaryError}
+
+ `; + } + return html`
${t("dreaming.diary.title")} +
+ + + +
- - -
- ${reversed.map( - (e) => html` - - `, - )} -
+ ${renderDiarySubtabExplainer()}
-
-
- ${entry.date ? html`` : nothing} -
- ${flattenDiaryBody(entry.body).map( - (para, i) => - html`

- ${para} -

`, - )} -
-
+ ${_diarySubTab === "dreams" + ? renderDreamDiaryEntries(props) + : _diarySubTab === "insights" + ? renderDiaryImportsSection(props) + : renderMemoryPalaceSection(props)} + ${renderWikiPreviewOverlay(props)}
`; }