diff --git a/CHANGELOG.md b/CHANGELOG.md index 54f358cf66d..6cb1d9c71b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Docs: https://docs.openclaw.ai - iOS: pin release versioning to an explicit CalVer in `apps/ios/version.json`, keep TestFlight iteration on the same short version until maintainers intentionally promote the next gateway version, and add the documented `pnpm ios:version:pin -- --from-gateway` workflow for release trains. (#63001) Thanks @ngutman. - Plugins/provider-auth: let provider manifests declare `providerAuthAliases` so provider variants can share env vars, auth profiles, config-backed auth, and API-key onboarding choices without core-specific wiring. +- Memory/dreaming: add a grounded REM backfill lane with historical `rem-harness --path`, diary commit, and reset flows so old daily notes can be replayed safely into `DREAMS.md`. Thanks @mbelinky. ### Fixes diff --git a/extensions/memory-core/api.ts b/extensions/memory-core/api.ts index addc8e50543..f487e1dbc59 100644 --- a/extensions/memory-core/api.ts +++ b/extensions/memory-core/api.ts @@ -4,3 +4,8 @@ export type { MemoryProviderStatus, MemorySyncProgressUpdate, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +export { + removeBackfillDiaryEntries, + writeBackfillDiaryEntries, +} from "./src/dreaming-narrative.js"; +export { previewGroundedRemMarkdown } from "./src/rem-evidence.js"; diff --git a/extensions/memory-core/src/cli.runtime.ts b/extensions/memory-core/src/cli.runtime.ts index be45c1984d0..94465f352ff 100644 --- a/extensions/memory-core/src/cli.runtime.ts +++ b/extensions/memory-core/src/cli.runtime.ts @@ -4,7 +4,6 @@ import os from "node:os"; import path from "node:path"; import { resolveMemoryRemDreamingConfig } from "openclaw/plugin-sdk/memory-core-host-status"; import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing"; -import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; import { colorize, defaultRuntime, @@ -31,10 +30,13 @@ import type { MemoryCommandOptions, MemoryPromoteCommandOptions, MemoryPromoteExplainOptions, + MemoryRemBackfillOptions, MemoryRemHarnessOptions, MemorySearchCommandOptions, } from "./cli.types.js"; -import { previewRemDreaming } from "./dreaming-phases.js"; +import { previewRemDreaming, seedHistoricalDailyMemorySignals } from "./dreaming-phases.js"; +import { removeBackfillDiaryEntries, writeBackfillDiaryEntries } from "./dreaming-narrative.js"; +import { previewGroundedRemMarkdown } from "./rem-evidence.js"; import { asRecord } from "./dreaming-shared.js"; import { resolveShortTermPromotionDreamingConfig } from "./dreaming.js"; import { @@ -114,6 +116,78 @@ function resolveMemoryPluginConfig(cfg: OpenClawConfig): Record return asRecord(entry?.config) ?? {}; } +const DAILY_MEMORY_FILE_NAME_RE = /^(\d{4}-\d{2}-\d{2})\.md$/; + +async function listHistoricalDailyFiles(inputPath: string): Promise { + const resolvedPath = path.resolve(inputPath); + const stat = await fs.stat(resolvedPath); + if (stat.isFile()) { + return DAILY_MEMORY_FILE_NAME_RE.test(path.basename(resolvedPath)) ? [resolvedPath] : []; + } + if (!stat.isDirectory()) { + return []; + } + const entries = await fs.readdir(resolvedPath, { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && DAILY_MEMORY_FILE_NAME_RE.test(entry.name)) + .map((entry) => path.join(resolvedPath, entry.name)) + .toSorted((a, b) => path.basename(a).localeCompare(path.basename(b))); +} + +async function createHistoricalRemHarnessWorkspace(params: { + inputPath: string; + remLimit: number; + nowMs: number; + timezone?: string; +}): Promise<{ + workspaceDir: string; + sourceFiles: string[]; + workspaceSourceFiles: string[]; + importedFileCount: number; + importedSignalCount: number; + skippedPaths: string[]; +}> { + const sourceFiles = await listHistoricalDailyFiles(params.inputPath); + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rem-harness-")); + const memoryDir = path.join(workspaceDir, "memory"); + await fs.mkdir(memoryDir, { recursive: true }); + for (const filePath of sourceFiles) { + await fs.copyFile(filePath, path.join(memoryDir, path.basename(filePath))); + } + const workspaceSourceFiles = sourceFiles.map((entry) => path.join(memoryDir, path.basename(entry))); + const seeded = await seedHistoricalDailyMemorySignals({ + workspaceDir, + filePaths: workspaceSourceFiles, + limit: params.remLimit, + nowMs: params.nowMs, + timezone: params.timezone, + }); + return { + workspaceDir, + sourceFiles, + workspaceSourceFiles, + importedFileCount: seeded.importedFileCount, + importedSignalCount: seeded.importedSignalCount, + skippedPaths: seeded.skippedPaths, + }; +} + +async function listWorkspaceDailyFiles(workspaceDir: string, limit: number): Promise { + const memoryDir = path.join(workspaceDir, "memory"); + try { + const files = await listHistoricalDailyFiles(memoryDir); + if (!Number.isFinite(limit) || limit <= 0 || files.length <= limit) { + return files; + } + return files.slice(-limit); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + return []; + } + throw err; + } +} + function formatDreamingSummary(cfg: OpenClawConfig): string { const pluginConfig = resolveMemoryPluginConfig(cfg); const dreaming = resolveShortTermPromotionDreamingConfig({ pluginConfig, cfg }); @@ -208,6 +282,18 @@ function formatExtraPaths(workspaceDir: string, extraPaths: string[]): string[] return normalizeExtraMemoryPaths(workspaceDir, extraPaths).map((entry) => shortenHomePath(entry)); } +function extractIsoDayFromPath(filePath: string): string | null { + const match = path.basename(filePath).match(DAILY_MEMORY_FILE_NAME_RE); + return match?.[1] ?? null; +} + +function groundedMarkdownToDiaryLines(markdown: string): string[] { + return markdown + .split(/\r?\n/) + .map((line) => line.replace(/^##\s+/, "").trimEnd()) + .filter((line, index, lines) => !(line.length === 0 && lines[index - 1]?.length === 0)); +} + function matchesPromotionSelector( candidate: { key: string; @@ -216,15 +302,15 @@ function matchesPromotionSelector( }, selector: string, ): boolean { - const trimmed = normalizeLowercaseStringOrEmpty(selector); + const trimmed = selector.trim().toLowerCase(); if (!trimmed) { return false; } return ( - normalizeLowercaseStringOrEmpty(candidate.key) === trimmed || - normalizeLowercaseStringOrEmpty(candidate.key).includes(trimmed) || - normalizeLowercaseStringOrEmpty(candidate.path).includes(trimmed) || - normalizeLowercaseStringOrEmpty(candidate.snippet).includes(trimmed) + candidate.key.toLowerCase() === trimmed || + candidate.key.toLowerCase().includes(trimmed) || + candidate.path.toLowerCase().includes(trimmed) || + candidate.snippet.toLowerCase().includes(trimmed) ); } @@ -1250,13 +1336,13 @@ export async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) { purpose: "status", run: async (manager) => { const status = manager.status(); - const workspaceDir = status.workspaceDir?.trim(); + const managerWorkspaceDir = status.workspaceDir?.trim(); const pluginConfig = resolveMemoryPluginConfig(cfg); const deep = resolveShortTermPromotionDreamingConfig({ pluginConfig, cfg, }); - if (!workspaceDir) { + if (!managerWorkspaceDir && !opts.path) { defaultRuntime.error("Memory rem-harness requires a resolvable workspace directory."); process.exitCode = 1; return; @@ -1266,69 +1352,297 @@ export async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) { cfg, }); const nowMs = Date.now(); - const cutoffMs = nowMs - Math.max(0, remConfig.lookbackDays) * 24 * 60 * 60 * 1000; - const recallEntries = (await readShortTermRecallEntries({ workspaceDir, nowMs })).filter( - (entry) => Date.parse(entry.lastRecalledAt) >= cutoffMs, - ); - const remPreview = previewRemDreaming({ - entries: recallEntries, - limit: remConfig.limit, - minPatternStrength: remConfig.minPatternStrength, - }); - const deepCandidates = await rankShortTermPromotionCandidates({ - workspaceDir, - minScore: 0, - minRecallCount: 0, - minUniqueQueries: 0, - includePromoted: Boolean(opts.includePromoted), - recencyHalfLifeDays: deep.recencyHalfLifeDays, - maxAgeDays: deep.maxAgeDays, - }); - - if (opts.json) { - defaultRuntime.writeJson({ - workspaceDir, - remConfig, - deepConfig: { - minScore: deep.minScore, - minRecallCount: deep.minRecallCount, - minUniqueQueries: deep.minUniqueQueries, - recencyHalfLifeDays: deep.recencyHalfLifeDays, - maxAgeDays: deep.maxAgeDays ?? null, - }, - rem: remPreview, - deep: { - candidateCount: deepCandidates.length, - candidates: deepCandidates, - }, + let workspaceDir = managerWorkspaceDir ?? ""; + let sourceFiles: string[] = []; + let groundedInputPaths: string[] = []; + let importedFileCount = 0; + let importedSignalCount = 0; + let skippedPaths: string[] = []; + let cleanupWorkspaceDir: string | null = null; + if (opts.path) { + const historical = await createHistoricalRemHarnessWorkspace({ + inputPath: opts.path, + remLimit: remConfig.limit, + nowMs, + timezone: remConfig.timezone, }); + workspaceDir = historical.workspaceDir; + cleanupWorkspaceDir = historical.workspaceDir; + sourceFiles = historical.sourceFiles; + groundedInputPaths = historical.workspaceSourceFiles; + importedFileCount = historical.importedFileCount; + importedSignalCount = historical.importedSignalCount; + skippedPaths = historical.skippedPaths; + if (sourceFiles.length === 0) { + await fs.rm(historical.workspaceDir, { recursive: true, force: true }); + defaultRuntime.error( + `Memory rem-harness found no YYYY-MM-DD.md files at ${shortenHomePath(path.resolve(opts.path))}.`, + ); + process.exitCode = 1; + return; + } + } + if (!workspaceDir) { + defaultRuntime.error("Memory rem-harness requires a resolvable workspace directory."); + process.exitCode = 1; return; } + try { + if (groundedInputPaths.length === 0 && opts.grounded) { + groundedInputPaths = await listWorkspaceDailyFiles(workspaceDir, remConfig.limit); + } + const cutoffMs = nowMs - Math.max(0, remConfig.lookbackDays) * 24 * 60 * 60 * 1000; + const recallEntries = (await readShortTermRecallEntries({ workspaceDir, nowMs })).filter( + (entry) => Date.parse(entry.lastRecalledAt) >= cutoffMs, + ); + const remPreview = previewRemDreaming({ + entries: recallEntries, + limit: remConfig.limit, + minPatternStrength: remConfig.minPatternStrength, + }); + const groundedPreview = + opts.grounded && groundedInputPaths.length > 0 + ? await previewGroundedRemMarkdown({ + workspaceDir, + inputPaths: groundedInputPaths, + }) + : null; + const deepCandidates = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + includePromoted: Boolean(opts.includePromoted), + recencyHalfLifeDays: deep.recencyHalfLifeDays, + maxAgeDays: deep.maxAgeDays, + }); - const rich = isRich(); - const lines = [ - `${colorize(rich, theme.heading, "REM Harness")} ${colorize(rich, theme.muted, `(${agentId})`)}`, - colorize(rich, theme.muted, `workspace=${shortenHomePath(workspaceDir)}`), - colorize( - rich, - theme.muted, - `recentRecallEntries=${recallEntries.length} deepCandidates=${deepCandidates.length}`, - ), - "", - colorize(rich, theme.heading, "REM Preview"), - ...remPreview.bodyLines, - "", - colorize(rich, theme.heading, "Deep Candidates"), - ...(deepCandidates.length > 0 - ? deepCandidates - .slice(0, 10) - .map( - (candidate) => - `${candidate.score.toFixed(3)} ${candidate.snippet} [${shortenHomePath(candidate.path)}:${candidate.startLine}-${candidate.endLine}]`, - ) - : ["- No deep candidates."]), - ]; - defaultRuntime.log(lines.join("\n")); + if (opts.json) { + defaultRuntime.writeJson({ + workspaceDir, + sourcePath: opts.path ? path.resolve(opts.path) : null, + sourceFiles, + historicalImport: + opts.path + ? { + importedFileCount, + importedSignalCount, + skippedPaths, + } + : null, + remConfig, + deepConfig: { + minScore: deep.minScore, + minRecallCount: deep.minRecallCount, + minUniqueQueries: deep.minUniqueQueries, + recencyHalfLifeDays: deep.recencyHalfLifeDays, + maxAgeDays: deep.maxAgeDays ?? null, + }, + rem: remPreview, + grounded: groundedPreview, + deep: { + candidateCount: deepCandidates.length, + candidates: deepCandidates, + }, + }); + return; + } + + const rich = isRich(); + const lines = [ + `${colorize(rich, theme.heading, "REM Harness")} ${colorize(rich, theme.muted, `(${agentId})`)}`, + colorize(rich, theme.muted, `workspace=${shortenHomePath(workspaceDir)}`), + ...(opts.path + ? [ + colorize( + rich, + theme.muted, + `sourcePath=${shortenHomePath(path.resolve(opts.path))}`, + ), + colorize( + rich, + theme.muted, + `historicalFiles=${sourceFiles.length} importedFiles=${importedFileCount} importedSignals=${importedSignalCount}`, + ), + ...(skippedPaths.length > 0 + ? [ + colorize( + rich, + theme.warn, + `skipped=${skippedPaths.map((entry) => shortenHomePath(entry)).join(", ")}`, + ), + ] + : []), + ] + : []), + ...(opts.grounded + ? [ + colorize( + rich, + theme.muted, + `groundedInputs=${groundedInputPaths.length > 0 ? groundedInputPaths.map((entry) => shortenHomePath(entry)).join(", ") : "none"}`, + ), + ] + : []), + colorize( + rich, + theme.muted, + `recentRecallEntries=${recallEntries.length} deepCandidates=${deepCandidates.length}`, + ), + "", + colorize(rich, theme.heading, "REM Preview"), + ...remPreview.bodyLines, + ...(groundedPreview + ? [ + "", + colorize(rich, theme.heading, "Grounded REM"), + ...groundedPreview.files.flatMap((file) => [ + colorize(rich, theme.label, file.path), + file.renderedMarkdown, + "", + ]), + ] + : []), + "", + colorize(rich, theme.heading, "Deep Candidates"), + ...(deepCandidates.length > 0 + ? deepCandidates + .slice(0, 10) + .map( + (candidate) => + `${candidate.score.toFixed(3)} ${candidate.snippet} [${shortenHomePath(candidate.path)}:${candidate.startLine}-${candidate.endLine}]`, + ) + : ["- No deep candidates."]), + ]; + defaultRuntime.log(lines.join("\n")); + } finally { + if (cleanupWorkspaceDir) { + await fs.rm(cleanupWorkspaceDir, { recursive: true, force: true }); + } + } + }, + }); +} + +export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) { + const { config: cfg, diagnostics } = await loadMemoryCommandConfig("memory rem-backfill"); + emitMemorySecretResolveDiagnostics(diagnostics, { json: Boolean(opts.json) }); + const agentId = resolveAgent(cfg, opts.agent); + + await withMemoryManagerForAgent({ + cfg, + agentId, + purpose: "status", + run: async (manager) => { + const status = manager.status(); + const workspaceDir = status.workspaceDir?.trim(); + const pluginConfig = resolveMemoryPluginConfig(cfg); + const remConfig = resolveMemoryRemDreamingConfig({ + pluginConfig, + cfg, + }); + if (!workspaceDir) { + defaultRuntime.error("Memory rem-backfill requires a resolvable workspace directory."); + process.exitCode = 1; + return; + } + + if (opts.rollback) { + const removed = await removeBackfillDiaryEntries({ workspaceDir }); + if (opts.json) { + defaultRuntime.writeJson({ + workspaceDir, + rollback: true, + dreamsPath: removed.dreamsPath, + removedEntries: removed.removed, + }); + return; + } + defaultRuntime.log( + [ + `${colorize(isRich(), theme.heading, "REM Backfill")} ${colorize(isRich(), theme.muted, "(rollback)")}`, + colorize(isRich(), theme.muted, `workspace=${shortenHomePath(workspaceDir)}`), + colorize(isRich(), theme.muted, `dreamsPath=${shortenHomePath(removed.dreamsPath)}`), + colorize(isRich(), theme.muted, `removedEntries=${removed.removed}`), + ].join("\n"), + ); + return; + } + + if (!opts.path) { + defaultRuntime.error("Memory rem-backfill requires --path unless using --rollback."); + process.exitCode = 1; + return; + } + + const scratchDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rem-backfill-")); + try { + const sourceFiles = await listHistoricalDailyFiles(opts.path); + if (sourceFiles.length === 0) { + defaultRuntime.error( + `Memory rem-backfill found no YYYY-MM-DD.md files at ${shortenHomePath(path.resolve(opts.path))}.`, + ); + process.exitCode = 1; + return; + } + const scratchMemoryDir = path.join(scratchDir, "memory"); + await fs.mkdir(scratchMemoryDir, { recursive: true }); + const workspaceSourceFiles: string[] = []; + for (const filePath of sourceFiles) { + const dst = path.join(scratchMemoryDir, path.basename(filePath)); + await fs.copyFile(filePath, dst); + workspaceSourceFiles.push(dst); + } + const grounded = await previewGroundedRemMarkdown({ + workspaceDir: scratchDir, + inputPaths: workspaceSourceFiles, + }); + const entries = grounded.files + .map((file) => { + const isoDay = extractIsoDayFromPath(file.path); + if (!isoDay) { + return null; + } + return { + isoDay, + sourcePath: file.path, + bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown), + }; + }) + .filter((entry): entry is NonNullable => entry !== null); + + const written = await writeBackfillDiaryEntries({ + workspaceDir, + entries, + timezone: remConfig.timezone, + }); + + if (opts.json) { + defaultRuntime.writeJson({ + workspaceDir, + sourcePath: path.resolve(opts.path), + sourceFiles, + groundedFiles: grounded.scannedFiles, + writtenEntries: written.written, + replacedEntries: written.replaced, + dreamsPath: written.dreamsPath, + }); + return; + } + + const rich = isRich(); + defaultRuntime.log( + [ + `${colorize(rich, theme.heading, "REM Backfill")} ${colorize(rich, theme.muted, `(${agentId})`)}`, + colorize(rich, theme.muted, `workspace=${shortenHomePath(workspaceDir)}`), + colorize(rich, theme.muted, `sourcePath=${shortenHomePath(path.resolve(opts.path))}`), + colorize(rich, theme.muted, `historicalFiles=${sourceFiles.length} writtenEntries=${written.written} replacedEntries=${written.replaced}`), + colorize(rich, theme.muted, `dreamsPath=${shortenHomePath(written.dreamsPath)}`), + ].join("\n"), + ); + } finally { + await fs.rm(scratchDir, { recursive: true, force: true }); + } }, }); } diff --git a/extensions/memory-core/src/cli.test.ts b/extensions/memory-core/src/cli.test.ts index 9241b24a177..657449499a2 100644 --- a/extensions/memory-core/src/cli.test.ts +++ b/extensions/memory-core/src/cli.test.ts @@ -948,6 +948,177 @@ describe("memory cli", () => { }); }); + it("previews rem harness output from a historical daily file path", async () => { + await withTempWorkspace(async (workspaceDir) => { + const historyDir = path.join(workspaceDir, "history"); + await fs.mkdir(historyDir, { recursive: true }); + const historyPath = path.join(historyDir, "2025-01-01.md"); + await fs.writeFile( + historyPath, + [ + "# Preferences Learned", + '- Always use "Happy Together" calendar for flights and reservations.', + "- Calendar ID: udolnrooml2f2ha8jaio24v1r8@group.calendar.google.com", + ].join("\n") + "\n", + "utf-8", + ); + + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + + const writeJson = spyRuntimeJson(defaultRuntime); + await runMemoryCli(["rem-harness", "--json", "--path", historyPath]); + + const payload = firstWrittenJsonArg<{ + sourcePath?: string | null; + sourceFiles?: string[]; + historicalImport?: { importedFileCount?: number; importedSignalCount?: number } | null; + rem?: { candidateTruths?: Array<{ snippet?: string }> }; + deep?: { candidates?: Array<{ snippet?: string; path?: string }> }; + }>(writeJson); + expect(payload?.sourcePath).toBe(historyPath); + expect(payload?.sourceFiles).toEqual([historyPath]); + expect(payload?.historicalImport?.importedFileCount).toBe(1); + expect(payload?.historicalImport?.importedSignalCount).toBeGreaterThan(0); + expect(Array.isArray(payload?.rem?.candidateTruths)).toBe(true); + expect(payload?.deep?.candidates?.[0]?.snippet).toContain("Happy Together"); + expect(payload?.deep?.candidates?.[0]?.path).toBe("memory/2025-01-01.md"); + expect(close).toHaveBeenCalled(); + }); + }); + + it("previews grounded rem output from a historical daily file path", async () => { + await withTempWorkspace(async (workspaceDir) => { + const historyDir = path.join(workspaceDir, "history"); + await fs.mkdir(historyDir, { recursive: true }); + const historyPath = path.join(historyDir, "2025-01-01.md"); + await fs.writeFile( + historyPath, + [ + "## Preferences Learned", + '- Always use "Happy Together" calendar for flights and reservations.', + "- Calendar ID: udolnrooml2f2ha8jaio24v1r8@group.calendar.google.com", + "", + "## Setup", + "- Set up Gmail access via gog.", + ].join("\n") + "\n", + "utf-8", + ); + + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + + const writeJson = spyRuntimeJson(defaultRuntime); + await runMemoryCli(["rem-harness", "--json", "--grounded", "--path", historyPath]); + + const payload = firstWrittenJsonArg<{ + grounded?: { + scannedFiles?: number; + files?: Array<{ + path?: string; + renderedMarkdown?: string; + memoryImplications?: Array<{ text?: string }>; + }>; + } | null; + }>(writeJson); + expect(payload?.grounded?.scannedFiles).toBe(1); + expect(payload?.grounded?.files?.[0]?.path).toBe("memory/2025-01-01.md"); + expect(payload?.grounded?.files?.[0]?.renderedMarkdown).toContain("## What Happened"); + expect(payload?.grounded?.files?.[0]?.renderedMarkdown).toContain("## Reflections"); + expect(payload?.grounded?.files?.[0]?.renderedMarkdown).toContain( + "## Possible Lasting Updates", + ); + expect(payload?.grounded?.files?.[0]?.memoryImplications?.[0]?.text).toContain( + 'Always use "Happy Together" calendar for flights and reservations', + ); + expect(close).toHaveBeenCalled(); + }); + }); + + it("writes grounded rem backfill entries into DREAMS.md", async () => { + await withTempWorkspace(async (workspaceDir) => { + const historyDir = path.join(workspaceDir, "history"); + await fs.mkdir(historyDir, { recursive: true }); + const historyPath = path.join(historyDir, "2025-01-01.md"); + await fs.writeFile( + historyPath, + [ + "## Preferences Learned", + '- Always use "Happy Together" calendar for flights and reservations.', + "- Calendar ID: udolnrooml2f2ha8jaio24v1r8@group.calendar.google.com", + ].join("\n") + "\n", + "utf-8", + ); + + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + + await runMemoryCli(["rem-backfill", "--path", historyPath]); + + const dreams = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8"); + expect(dreams).toContain("openclaw:dreaming:backfill-entry"); + expect(dreams).toContain("January 1, 2025"); + expect(dreams).toContain("What Happened"); + expect(dreams).toContain("Possible Lasting Updates"); + expect(dreams).toContain("Happy Together"); + expect(close).toHaveBeenCalled(); + }); + }); + + it("rolls back grounded rem backfill entries from DREAMS.md", async () => { + await withTempWorkspace(async (workspaceDir) => { + const dreamsPath = path.join(workspaceDir, "DREAMS.md"); + await fs.writeFile( + dreamsPath, + [ + "# Dream Diary", + "", + "", + "---", + "", + "*April 5, 2026, 3:00 AM*", + "", + "Keep this normal dream.", + "", + "---", + "", + "*January 1, 2025*", + "", + "", + "", + "What Happened", + "1. Remove this entry.", + "", + "", + "", + ].join("\n"), + "utf-8", + ); + + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + + await runMemoryCli(["rem-backfill", "--rollback"]); + + const dreams = await fs.readFile(dreamsPath, "utf-8"); + expect(dreams).toContain("Keep this normal dream."); + expect(dreams).not.toContain("Remove this entry."); + expect(close).toHaveBeenCalled(); + }); + }); + it("applies top promote candidates into MEMORY.md", async () => { await withTempWorkspace(async (workspaceDir) => { await writeDailyMemoryNote(workspaceDir, "2026-04-01", [ diff --git a/extensions/memory-core/src/cli.ts b/extensions/memory-core/src/cli.ts index f0d5f8ac5d0..8c836cc052f 100644 --- a/extensions/memory-core/src/cli.ts +++ b/extensions/memory-core/src/cli.ts @@ -8,6 +8,7 @@ import type { MemoryCommandOptions, MemoryPromoteCommandOptions, MemoryPromoteExplainOptions, + MemoryRemBackfillOptions, MemoryRemHarnessOptions, MemorySearchCommandOptions, } from "./cli.types.js"; @@ -59,6 +60,11 @@ async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) { await runtime.runMemoryRemHarness(opts); } +async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) { + const runtime = await loadMemoryCliRuntime(); + await runtime.runMemoryRemBackfill(opts); +} + export function registerMemoryCli(program: Command) { const memory = program .command("memory") @@ -95,6 +101,10 @@ export function registerMemoryCli(program: Command) { "openclaw memory rem-harness --json", "Preview REM reflections, candidate truths, and deep promotion output.", ], + [ + "openclaw memory rem-backfill --path ./memory", + "Write grounded historical REM entries into DREAMS.md for UI review.", + ], ["openclaw memory status --json", "Output machine-readable JSON (good for scripts)."], ])}\n\n${theme.muted("Docs:")} ${formatDocsLink("/cli/memory", "docs.openclaw.ai/cli/memory")}\n`, ); @@ -177,9 +187,22 @@ export function registerMemoryCli(program: Command) { .command("rem-harness") .description("Preview REM reflections, candidate truths, and deep promotions without writing") .option("--agent ", "Agent id (default: default agent)") + .option("--path ", "Seed the harness from historical daily memory file(s)") + .option("--grounded", "Also render a grounded day-level REM preview") .option("--include-promoted", "Include already promoted deep candidates", false) .option("--json", "Print JSON") .action(async (opts: MemoryRemHarnessOptions) => { await runMemoryRemHarness(opts); }); + + memory + .command("rem-backfill") + .description("Write grounded historical REM summaries into DREAMS.md for UI review") + .option("--agent ", "Agent id (default: default agent)") + .option("--path ", "Historical daily memory file(s) or directory") + .option("--rollback", "Remove previously written grounded REM backfill entries", false) + .option("--json", "Print JSON") + .action(async (opts: MemoryRemBackfillOptions) => { + await runMemoryRemBackfill(opts); + }); } diff --git a/extensions/memory-core/src/cli.types.ts b/extensions/memory-core/src/cli.types.ts index 5e5f8262367..842bbcb29ce 100644 --- a/extensions/memory-core/src/cli.types.ts +++ b/extensions/memory-core/src/cli.types.ts @@ -29,4 +29,11 @@ export type MemoryPromoteExplainOptions = MemoryCommandOptions & { export type MemoryRemHarnessOptions = MemoryCommandOptions & { includePromoted?: boolean; + path?: string; + grounded?: boolean; +}; + +export type MemoryRemBackfillOptions = MemoryCommandOptions & { + path?: string; + rollback?: boolean; }; diff --git a/extensions/memory-core/src/dreaming-narrative.test.ts b/extensions/memory-core/src/dreaming-narrative.test.ts index 877ca383851..74ed10c234c 100644 --- a/extensions/memory-core/src/dreaming-narrative.test.ts +++ b/extensions/memory-core/src/dreaming-narrative.test.ts @@ -3,12 +3,16 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { appendNarrativeEntry, + buildBackfillDiaryEntry, buildDiaryEntry, buildNarrativePrompt, extractNarrativeText, formatNarrativeDate, + formatBackfillDiaryDate, generateAndAppendDreamNarrative, + removeBackfillDiaryEntries, type NarrativePhaseData, + writeBackfillDiaryEntries, } from "./dreaming-narrative.js"; import { createMemoryCoreTestHarness } from "./test-helpers.js"; @@ -117,6 +121,88 @@ describe("buildDiaryEntry", () => { }); }); +describe("backfill diary entries", () => { + it("formats a backfill date without time", () => { + expect(formatBackfillDiaryDate("2026-01-01", "UTC")).toBe("January 1, 2026"); + }); + + it("builds a marked backfill diary entry", () => { + const entry = buildBackfillDiaryEntry({ + isoDay: "2026-01-01", + sourcePath: "memory/2026-01-01.md", + bodyLines: ["What Happened", "1. A durable preference appeared."], + timezone: "UTC", + }); + expect(entry).toContain("*January 1, 2026*"); + expect(entry).toContain("openclaw:dreaming:backfill-entry"); + expect(entry).toContain("What Happened"); + }); + + it("writes and replaces backfill diary entries", async () => { + const workspaceDir = await createTempWorkspace("openclaw-dreaming-backfill-"); + const first = await writeBackfillDiaryEntries({ + workspaceDir, + timezone: "UTC", + entries: [ + { + isoDay: "2026-01-01", + sourcePath: "memory/2026-01-01.md", + bodyLines: ["What Happened", "1. First pass."], + }, + ], + }); + expect(first.written).toBe(1); + expect(first.replaced).toBe(0); + + const second = await writeBackfillDiaryEntries({ + workspaceDir, + timezone: "UTC", + entries: [ + { + isoDay: "2026-01-02", + sourcePath: "memory/2026-01-02.md", + bodyLines: ["Reflections", "1. Second pass."], + }, + ], + }); + expect(second.written).toBe(1); + expect(second.replaced).toBe(1); + + const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8"); + expect(content).not.toContain("First pass."); + expect(content).toContain("Second pass."); + expect(content.match(/openclaw:dreaming:backfill-entry/g)?.length).toBe(1); + }); + + it("removes only backfill diary entries", async () => { + const workspaceDir = await createTempWorkspace("openclaw-dreaming-backfill-"); + await appendNarrativeEntry({ + workspaceDir, + narrative: "Keep this real dream.", + nowMs: Date.parse("2026-04-05T03:00:00Z"), + timezone: "UTC", + }); + await writeBackfillDiaryEntries({ + workspaceDir, + timezone: "UTC", + entries: [ + { + isoDay: "2026-01-01", + sourcePath: "memory/2026-01-01.md", + bodyLines: ["What Happened", "1. Remove this backfill."], + }, + ], + }); + + const removed = await removeBackfillDiaryEntries({ workspaceDir }); + expect(removed.removed).toBe(1); + + const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8"); + expect(content).toContain("Keep this real dream."); + expect(content).not.toContain("Remove this backfill."); + }); +}); + describe("appendNarrativeEntry", () => { it("creates DREAMS.md with diary header on fresh workspace", async () => { const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); diff --git a/extensions/memory-core/src/dreaming-narrative.ts b/extensions/memory-core/src/dreaming-narrative.ts index 5d3d1f30e24..82c237295a6 100644 --- a/extensions/memory-core/src/dreaming-narrative.ts +++ b/extensions/memory-core/src/dreaming-narrative.ts @@ -70,6 +70,7 @@ const NARRATIVE_TIMEOUT_MS = 60_000; const DREAMS_FILENAMES = ["DREAMS.md", "dreams.md"] as const; const DIARY_START_MARKER = ""; const DIARY_END_MARKER = ""; +const BACKFILL_ENTRY_MARKER = "openclaw:dreaming:backfill-entry"; // ── Prompt building ──────────────────────────────────────────────────── @@ -167,6 +168,157 @@ async function resolveDreamsPath(workspaceDir: string): Promise { return path.join(workspaceDir, DREAMS_FILENAMES[0]); } +async function readDreamsFile(dreamsPath: string): Promise { + try { + return await fs.readFile(dreamsPath, "utf-8"); + } catch (err) { + if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { + return ""; + } + throw err; + } +} + +function ensureDiarySection(existing: string): string { + if (existing.includes(DIARY_START_MARKER) && existing.includes(DIARY_END_MARKER)) { + return existing; + } + const diarySection = `# Dream Diary\n\n${DIARY_START_MARKER}\n${DIARY_END_MARKER}\n`; + if (existing.trim().length === 0) { + return diarySection; + } + return diarySection + "\n" + existing; +} + +function replaceDiaryContent(existing: string, diaryContent: string): string { + const ensured = ensureDiarySection(existing); + const startIdx = ensured.indexOf(DIARY_START_MARKER); + const endIdx = ensured.indexOf(DIARY_END_MARKER); + if (startIdx < 0 || endIdx < 0 || endIdx < startIdx) { + return ensured; + } + const before = ensured.slice(0, startIdx + DIARY_START_MARKER.length); + const after = ensured.slice(endIdx); + const normalized = diaryContent.trim().length > 0 ? `\n${diaryContent.trim()}\n` : "\n"; + return before + normalized + after; +} + +function splitDiaryBlocks(diaryContent: string): string[] { + return diaryContent + .split(/\n---\n/) + .map((block) => block.trim()) + .filter((block) => block.length > 0); +} + +function joinDiaryBlocks(blocks: string[]): string { + if (blocks.length === 0) { + return ""; + } + return blocks.map((block) => `---\n\n${block.trim()}\n`).join("\n"); +} + +function stripBackfillDiaryBlocks(existing: string): { updated: string; removed: number } { + const ensured = ensureDiarySection(existing); + const startIdx = ensured.indexOf(DIARY_START_MARKER); + const endIdx = ensured.indexOf(DIARY_END_MARKER); + if (startIdx < 0 || endIdx < 0 || endIdx < startIdx) { + return { updated: ensured, removed: 0 }; + } + const inner = ensured.slice(startIdx + DIARY_START_MARKER.length, endIdx); + const kept: string[] = []; + let removed = 0; + for (const block of splitDiaryBlocks(inner)) { + if (block.includes(BACKFILL_ENTRY_MARKER)) { + removed += 1; + continue; + } + kept.push(block); + } + return { + updated: replaceDiaryContent(ensured, joinDiaryBlocks(kept)), + removed, + }; +} + +export function formatBackfillDiaryDate(isoDay: string, timezone?: string): string { + const opts: Intl.DateTimeFormatOptions = { + timeZone: timezone ?? "UTC", + year: "numeric", + month: "long", + day: "numeric", + }; + const epochMs = Date.parse(`${isoDay}T12:00:00Z`); + return new Intl.DateTimeFormat("en-US", opts).format(new Date(epochMs)); +} + +export function buildBackfillDiaryEntry(params: { + isoDay: string; + bodyLines: string[]; + sourcePath?: string; + timezone?: string; +}): string { + const dateStr = formatBackfillDiaryDate(params.isoDay, params.timezone); + const marker = ``; + const body = params.bodyLines.map((line) => line.trimEnd()).join("\n").trim(); + return [`*${dateStr}*`, marker, body].filter((part) => part.length > 0).join("\n\n"); +} + +export async function writeBackfillDiaryEntries(params: { + workspaceDir: string; + entries: Array<{ + isoDay: string; + bodyLines: string[]; + sourcePath?: string; + }>; + timezone?: string; +}): Promise<{ dreamsPath: string; written: number; replaced: number }> { + const dreamsPath = await resolveDreamsPath(params.workspaceDir); + await fs.mkdir(path.dirname(dreamsPath), { recursive: true }); + const existing = await readDreamsFile(dreamsPath); + const stripped = stripBackfillDiaryBlocks(existing); + const startIdx = stripped.updated.indexOf(DIARY_START_MARKER); + const endIdx = stripped.updated.indexOf(DIARY_END_MARKER); + const inner = + startIdx >= 0 && endIdx > startIdx + ? stripped.updated.slice(startIdx + DIARY_START_MARKER.length, endIdx) + : ""; + const preservedBlocks = splitDiaryBlocks(inner); + const nextBlocks = [ + ...preservedBlocks, + ...params.entries.map((entry) => + buildBackfillDiaryEntry({ + isoDay: entry.isoDay, + bodyLines: entry.bodyLines, + sourcePath: entry.sourcePath, + timezone: params.timezone, + }), + ), + ]; + const updated = replaceDiaryContent(stripped.updated, joinDiaryBlocks(nextBlocks)); + await fs.writeFile(dreamsPath, updated, "utf-8"); + return { + dreamsPath, + written: params.entries.length, + replaced: stripped.removed, + }; +} + +export async function removeBackfillDiaryEntries(params: { + workspaceDir: string; +}): Promise<{ dreamsPath: string; removed: number }> { + const dreamsPath = await resolveDreamsPath(params.workspaceDir); + const existing = await readDreamsFile(dreamsPath); + const stripped = stripBackfillDiaryBlocks(existing); + if (stripped.removed > 0 || existing.length > 0) { + await fs.mkdir(path.dirname(dreamsPath), { recursive: true }); + await fs.writeFile(dreamsPath, stripped.updated, "utf-8"); + } + return { + dreamsPath, + removed: stripped.removed, + }; +} + export function buildDiaryEntry(narrative: string, dateStr: string): string { return `\n---\n\n*${dateStr}*\n\n${narrative}\n`; } diff --git a/extensions/memory-core/src/dreaming-phases.ts b/extensions/memory-core/src/dreaming-phases.ts index 1e56d0efb14..38af9820ba7 100644 --- a/extensions/memory-core/src/dreaming-phases.ts +++ b/extensions/memory-core/src/dreaming-phases.ts @@ -18,18 +18,9 @@ import { type MemoryLightDreamingConfig, type MemoryRemDreamingConfig, } from "openclaw/plugin-sdk/memory-core-host-status"; -import { - lowercasePreservingWhitespace, - normalizeLowercaseStringOrEmpty, -} from "openclaw/plugin-sdk/text-runtime"; import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js"; import { generateAndAppendDreamNarrative, type NarrativePhaseData } from "./dreaming-narrative.js"; -import { - asRecord, - formatErrorMessage, - includesSystemEventToken, - normalizeTrimmedString, -} from "./dreaming-shared.js"; +import { asRecord, formatErrorMessage, normalizeTrimmedString } from "./dreaming-shared.js"; import { readShortTermRecallEntries, recordDreamingPhaseSignals, @@ -131,7 +122,7 @@ function isGenericDailyHeading(heading: string): boolean { if (!normalized) { return true; } - const lower = normalizeLowercaseStringOrEmpty(normalized); + const lower = normalized.toLowerCase(); if (lower === "today" || lower === "yesterday" || lower === "tomorrow") { return true; } @@ -428,7 +419,7 @@ type SessionIngestionCollectionResult = { function normalizeWorkspaceKey(workspaceDir: string): string { const resolved = path.resolve(workspaceDir).replace(/\\/g, "/"); - return process.platform === "win32" ? lowercasePreservingWhitespace(resolved) : resolved; + return process.platform === "win32" ? resolved.toLowerCase() : resolved; } function resolveSessionIngestionStatePath(workspaceDir: string): string { @@ -1100,13 +1091,117 @@ async function ingestDailyMemorySignals(params: { } } +export async function seedHistoricalDailyMemorySignals(params: { + workspaceDir: string; + filePaths: string[]; + limit: number; + nowMs: number; + timezone?: string; +}): Promise<{ + importedFileCount: number; + importedSignalCount: number; + skippedPaths: string[]; +}> { + const normalizedPaths = [...new Set(params.filePaths.map((entry) => entry.trim()).filter(Boolean))]; + if (normalizedPaths.length === 0) { + return { + importedFileCount: 0, + importedSignalCount: 0, + skippedPaths: [], + }; + } + + const resolved = normalizedPaths + .map((filePath) => { + const fileName = path.basename(filePath); + const match = fileName.match(DAILY_MEMORY_FILENAME_RE); + if (!match) { + return { filePath, day: null as string | null }; + } + return { filePath, day: match[1] ?? null }; + }) + .toSorted((a, b) => { + if (a.day && b.day) { + return b.day.localeCompare(a.day); + } + if (a.day) { + return -1; + } + if (b.day) { + return 1; + } + return a.filePath.localeCompare(b.filePath); + }); + + const valid = resolved.filter((entry): entry is { filePath: string; day: string } => Boolean(entry.day)); + const skippedPaths = resolved.filter((entry) => !entry.day).map((entry) => entry.filePath); + const totalCap = Math.max(20, params.limit * 4); + const perFileCap = Math.max(6, Math.ceil(totalCap / Math.max(1, valid.length))); + let importedSignalCount = 0; + let importedFileCount = 0; + + for (const entry of valid) { + if (importedSignalCount >= totalCap) { + break; + } + const raw = await fs.readFile(entry.filePath, "utf-8").catch((err: unknown) => { + if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { + skippedPaths.push(entry.filePath); + return ""; + } + throw err; + }); + if (!raw) { + continue; + } + const lines = stripManagedDailyDreamingLines(raw.split(/\r?\n/)); + const chunks = buildDailySnippetChunks(lines, perFileCap); + const results: MemorySearchResult[] = []; + for (const chunk of chunks) { + results.push({ + path: `memory/${entry.day}.md`, + startLine: chunk.startLine, + endLine: chunk.endLine, + score: DAILY_INGESTION_SCORE, + snippet: chunk.snippet, + source: "memory", + }); + if (results.length >= perFileCap || importedSignalCount + results.length >= totalCap) { + break; + } + } + if (results.length === 0) { + continue; + } + await recordShortTermRecalls({ + workspaceDir: params.workspaceDir, + query: `__dreaming_daily__:${entry.day}`, + results, + signalType: "daily", + dedupeByQueryPerDay: true, + dayBucket: entry.day, + nowMs: params.nowMs, + timezone: params.timezone, + }); + importedSignalCount += results.length; + importedFileCount += 1; + } + + return { + importedFileCount, + importedSignalCount, + skippedPaths, + }; +} + function entryAverageScore(entry: ShortTermRecallEntry): number { return entry.recallCount > 0 ? Math.max(0, Math.min(1, entry.totalScore / entry.recallCount)) : 0; } function tokenizeSnippet(snippet: string): Set { return new Set( - normalizeLowercaseStringOrEmpty(snippet) + snippet + .toLowerCase() .split(/[^a-z0-9]+/i) .map((token) => token.trim()) .filter(Boolean), @@ -1117,7 +1212,7 @@ function jaccardSimilarity(left: string, right: string): number { const leftTokens = tokenizeSnippet(left); const rightTokens = tokenizeSnippet(right); if (leftTokens.size === 0 || rightTokens.size === 0) { - return normalizeLowercaseStringOrEmpty(left) === normalizeLowercaseStringOrEmpty(right) ? 1 : 0; + return left.trim().toLowerCase() === right.trim().toLowerCase() ? 1 : 0; } let intersection = 0; for (const token of leftTokens) { @@ -1529,10 +1624,7 @@ async function runPhaseIfTriggered(params: { storage: { mode: "inline" | "separate" | "both"; separateReports: boolean }; }); }): Promise<{ handled: true; reason: string } | undefined> { - if ( - params.trigger !== "heartbeat" || - !includesSystemEventToken(params.cleanedBody, params.eventText) - ) { + if (params.trigger !== "heartbeat" || params.cleanedBody.trim() !== params.eventText) { return undefined; } if (!params.config.enabled) { @@ -1558,10 +1650,7 @@ async function runPhaseIfTriggered(params: { await runLightDreaming({ workspaceDir, cfg: params.cfg, - config: params.config as MemoryLightDreamingConfig & { - timezone?: string; - storage: { mode: "inline" | "separate" | "both"; separateReports: boolean }; - }, + config: params.config, logger: params.logger, subagent: params.subagent, }); @@ -1569,10 +1658,7 @@ async function runPhaseIfTriggered(params: { await runRemDreaming({ workspaceDir, cfg: params.cfg, - config: params.config as MemoryRemDreamingConfig & { - timezone?: string; - storage: { mode: "inline" | "separate" | "both"; separateReports: boolean }; - }, + config: params.config, logger: params.logger, subagent: params.subagent, }); diff --git a/extensions/memory-core/src/rem-evidence.ts b/extensions/memory-core/src/rem-evidence.ts new file mode 100644 index 00000000000..b62b20ba7d9 --- /dev/null +++ b/extensions/memory-core/src/rem-evidence.ts @@ -0,0 +1,903 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +const REM_BLOCKED_SECTION_RE = + /\b(morning reminders|tasks? for today|to-?do|pickups?|action items?|next steps?|open questions?|stats|setup tasks?|priority contacts|visitors?|top priority candidates|timeline coverage|action items for morning review|test .* skill|heartbeat checks?|date semantics guardrail|still broken|last message (?:&|and) status|plugin \/ service warning|email triage cron)\b/i; +const REM_GENERIC_SECTION_RE = + /^(setup|session notes?|notes|summary|major accomplishments?|infrastructure|process improvements?)$/i; +const REM_MEMORY_SIGNAL_RE = + /\b(always use|prefers?|preference|preferences|standing rule|rule:|use .* calendar|durable|remember)\b/i; +const REM_BUILD_SIGNAL_RE = + /\b(set up|setup|created|built|rewrite|rewrote|implemented|installed|configured|added|updated|exported|documented)\b/i; +const REM_INCIDENT_SIGNAL_RE = + /\b(fail(?:ed|ing)?|error|issue|problem|auth|expired|broken|unable|missing|required|root cause|consecutive failures?)\b/i; +const REM_LOGISTICS_SIGNAL_RE = + /\b(visitor|arriv(?:e|al|ing)|flight|calendar|reservation|schedule|coordinate|travel|pickup)\b/i; +const REM_TASK_SIGNAL_RE = + /\b(reminder|task|to-?do|action item|next step|need to|follow up|respond to|call\b|check\b)\b/i; +const REM_ROUTING_SIGNAL_RE = + /\b(categor(?:ize|ized|ization)|route|routing|workflow|processor|read later|auto-implement|codex|razor)\b/i; +const REM_OPERATOR_RULE_SIGNAL_RE = /\b(learned:|rule:|always [a-z])\b/i; +const REM_EXTERNALIZATION_SIGNAL_RE = + /\b(obsidian|memory|tracker|notes captured|committed to memory|updated .*md|documented|file comparison table)\b/i; +const REM_RETRY_SIGNAL_RE = + /\b(repeat(?:ed|edly)?|again|retry|root cause|third attempt|fourth|fifth|consecutive failures?)\b/i; +const REM_PERSON_PATTERN_SIGNAL_RE = + /\b(relationship|who:|patterns?:|failure modes?:|best stance:|space|boundaries|timing|family quick reference)\b/i; +const REM_SITUATIONAL_SIGNAL_RE = + /\b(hotel|address|phone|reservation|check-?in|check-?out|flight|arrival|departure|terminal|price shown|invoice|pending items|screenshot|butler)\b/i; +const REM_PERSISTENCE_SIGNAL_RE = + /\b(always|preference|prefers?|standing rule|best stance|failure modes?|key patterns?|relationship|who:|important .* keep track|people in .* life|partner|wife|husband|boyfriend|girlfriend)\b/i; +const REM_TRANSIENT_SIGNAL_RE = + /\b(today|this session|in progress|installed|booked|confirmed|pending|status:|action pending|open items?|next steps?|issue:|diagnostics|screenshot|source file|insight files|thread\b|ticket|price shown|calendar fix|cron fixes|security audit|updates? this session|bought:|order\b)\b/i; +const REM_SECTION_PERSISTENCE_TITLE_RE = + /\b(preferences? learned|preference|people update|relationship|standing|patterns?|identity|memory)\b/i; +const REM_SECTION_TRANSIENT_TITLE_RE = + /\b(setup|fix|fixes|audit|booked|call|today|session|updates?|file paths|open items?|next steps?|research pipeline|info gathered|calendar|tickets?)\b/i; +const REM_METADATA_HEAVY_SIGNAL_RE = + /\b(address|phone|email|website|google maps|source file|insight files|conversation id|thread has|order\b|reservation\b|price\b|cost\b|ticket|uuid|url:|model:|workspace:|bindings:|accountid|config change|path:)\b/i; +const REM_PROJECT_META_SIGNAL_RE = + /\b(strategy|audit|discussion|research|topic|candidate|north star|pipeline|data dump|export|draft|insights? draft|weekly|analysis|findings)\b/i; +const REM_PROCESS_FRAME_SIGNAL_RE = + /\b(dossier|registry|cadence|framework|facts,\s*timeline|open loops|next actions|auto preference rollups?|insights? draft created)\b/i; +const REM_TOOLING_META_SIGNAL_RE = + /\b(cli|tool|tools\.md|agents\.md|sessionssend|subagents?|spawn|tmux|xurl|bird|codex exec|interactive codex)\b/i; +const REM_TRAVEL_DECISION_SIGNAL_RE = + /\b(routing|cabin|business class|trip brief|departure|arrival|hotel|reservation|tickets?|show tonight|cheaper alternatives?|venue timing)\b/i; +const REM_STABLE_PERSON_SIGNAL_RE = + /\b(partner|wife|husband|boyfriend|girlfriend|relationship interest|lives in)\b/i; +const REM_EXPLICIT_PREFERENCE_SIGNAL_RE = + /\b(explicitly|wants?|does not want|don't want|default .* should|should default to|likes?|dislikes?|treat .* as|prefers?)\b/i; +const REM_SPECIFICITY_BURDEN_RE = + /\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\b|€|\$\d|→|\b\d{1,2}:\d{2}\b|\+\d{6,}/i; +const REM_TIME_PREFIX_RE = /^\d{1,2}:\d{2}\s*-\s*/; +const REM_CODE_FENCE_RE = /^\s*```/; +const REM_TABLE_RE = /^\s*\|.*\|\s*$/; +const REM_TABLE_DIVIDER_RE = /^\s*\|?[\s:-]+\|[\s|:-]*$/; +const REM_SUMMARY_FACT_LIMIT = 4; +const REM_SUMMARY_REFLECTION_LIMIT = 4; +const REM_SUMMARY_MEMORY_LIMIT = 3; + +export type GroundedRemPreviewItem = { + text: string; + refs: string[]; +}; + +export type GroundedRemCandidate = GroundedRemPreviewItem & { + lean: "likely_durable" | "unclear" | "likely_situational"; +}; + +export type GroundedRemFilePreview = { + path: string; + facts: GroundedRemPreviewItem[]; + reflections: GroundedRemPreviewItem[]; + memoryImplications: GroundedRemPreviewItem[]; + candidates: GroundedRemCandidate[]; + renderedMarkdown: string; +}; + +export type GroundedRemPreviewResult = { + workspaceDir: string; + scannedFiles: number; + files: GroundedRemFilePreview[]; +}; + +type CandidateSnippetSummary = GroundedRemCandidate & { + score: number; +}; + +type ParsedSectionLine = { + line: number; + text: string; +}; + +type ParsedMarkdownSection = { + title: string; + startLine: number; + endLine: number; + lines: ParsedSectionLine[]; +}; + +type SectionSnippet = { + text: string; + line: number; +}; + +type SectionSummary = { + title: string; + text: string; + refs: string[]; + scores: { + preference: number; + build: number; + incident: number; + logistics: number; + tasks: number; + routing: number; + externalization: number; + retries: number; + overall: number; + }; +}; + +function normalizeWhitespace(value: string): string { + return value.trim().replace(/\s+/g, " "); +} + +function normalizePath(rawPath: string): string { + return rawPath.replaceAll("\\", "/").replace(/^\.\//, ""); +} + +function stripMarkdown(text: string): string { + return normalizeWhitespace( + text + .replace(/!\[[^\]]*]\([^)]*\)/g, "") + .replace(/\[([^\]]+)]\([^)]*\)/g, "$1") + .replace(/[`*_~>#]/g, "") + .replace(/\s+/g, " "), + ); +} + +function sanitizeSectionTitle(title: string): string { + return normalizeWhitespace(stripMarkdown(title).replace(REM_TIME_PREFIX_RE, "")); +} + +function makeRef(pathValue: string, startLine: number, endLine = startLine): string { + return startLine === endLine + ? `${pathValue}:${startLine}` + : `${pathValue}:${startLine}-${endLine}`; +} + +function parseMarkdownSections(content: string): ParsedMarkdownSection[] { + const sections: ParsedMarkdownSection[] = []; + const lines = content.split(/\r?\n/); + let current: ParsedMarkdownSection | null = null; + let inCodeFence = false; + + const flush = () => { + if (!current) { + return; + } + const meaningfulLines = current.lines.filter( + (entry) => normalizeWhitespace(entry.text).length > 0, + ); + if (meaningfulLines.length > 0) { + const endLine = meaningfulLines[meaningfulLines.length - 1]?.line ?? current.endLine; + sections.push({ ...current, endLine, lines: meaningfulLines }); + } + current = null; + }; + + for (let index = 0; index < lines.length; index += 1) { + const rawLine = lines[index] ?? ""; + const lineNumber = index + 1; + if (REM_CODE_FENCE_RE.test(rawLine)) { + inCodeFence = !inCodeFence; + continue; + } + if (inCodeFence) { + continue; + } + const headingMatch = rawLine.match(/^\s{0,3}(#{2,6})\s+(.+)$/); + if (headingMatch?.[2]) { + flush(); + current = { + title: sanitizeSectionTitle(headingMatch[2]), + startLine: lineNumber, + endLine: lineNumber, + lines: [], + }; + continue; + } + if (!current) { + continue; + } + current.endLine = lineNumber; + const trimmed = rawLine.trim(); + if ( + !trimmed || + /^---+$/.test(trimmed) || + REM_TABLE_RE.test(trimmed) || + REM_TABLE_DIVIDER_RE.test(trimmed) + ) { + continue; + } + current.lines.push({ line: lineNumber, text: rawLine }); + } + + flush(); + return sections; +} + +function sectionToSnippets(section: ParsedMarkdownSection): SectionSnippet[] { + const snippets: SectionSnippet[] = []; + const seen = new Set(); + for (const entry of section.lines) { + const trimmed = entry.text.trim(); + if (!trimmed) { + continue; + } + const bulletMatch = trimmed.match(/^(?:[-*+]|\d+\.)\s+(?:\[[ xX]\]\s*)?(.*)$/); + const candidateText = bulletMatch?.[1] ?? trimmed; + const text = normalizeWhitespace(stripMarkdown(candidateText)); + if (text.length < 10) { + continue; + } + const dedupeKey = text.toLowerCase(); + if (seen.has(dedupeKey)) { + continue; + } + seen.add(dedupeKey); + snippets.push({ text, line: entry.line }); + } + return snippets; +} + +function countMatchingSnippets(snippets: SectionSnippet[], pattern: RegExp): number { + let count = 0; + for (const snippet of snippets) { + if (pattern.test(snippet.text)) { + count += 1; + } + } + return count; +} + +function scoreSection(section: ParsedMarkdownSection, snippets: SectionSnippet[]) { + const title = section.title; + const titleBonus = (pattern: RegExp) => (pattern.test(title) ? 1 : 0); + const preference = + countMatchingSnippets(snippets, REM_MEMORY_SIGNAL_RE) + titleBonus(REM_MEMORY_SIGNAL_RE); + const build = + countMatchingSnippets(snippets, REM_BUILD_SIGNAL_RE) + titleBonus(REM_BUILD_SIGNAL_RE); + const incident = + countMatchingSnippets(snippets, REM_INCIDENT_SIGNAL_RE) + titleBonus(REM_INCIDENT_SIGNAL_RE); + const logistics = + countMatchingSnippets(snippets, REM_LOGISTICS_SIGNAL_RE) + titleBonus(REM_LOGISTICS_SIGNAL_RE); + const tasks = + countMatchingSnippets(snippets, REM_TASK_SIGNAL_RE) + titleBonus(REM_TASK_SIGNAL_RE); + const routing = + countMatchingSnippets(snippets, REM_ROUTING_SIGNAL_RE) + titleBonus(REM_ROUTING_SIGNAL_RE); + const externalization = + countMatchingSnippets(snippets, REM_EXTERNALIZATION_SIGNAL_RE) + + titleBonus(REM_EXTERNALIZATION_SIGNAL_RE); + const retries = + countMatchingSnippets(snippets, REM_RETRY_SIGNAL_RE) + titleBonus(REM_RETRY_SIGNAL_RE); + const overall = + preference * 2 + + build * 1.6 + + incident * 1.6 + + logistics * 1.2 + + routing * 1.8 + + externalization * 1.4 + + Math.min(snippets.length, 3) * 0.3 - + (REM_GENERIC_SECTION_RE.test(title) ? 0.8 : 0); + return { + preference, + build, + incident, + logistics, + tasks, + routing, + externalization, + retries, + overall, + }; +} + +function scoreSnippet(text: string, title: string): number { + let score = 1; + if (REM_MEMORY_SIGNAL_RE.test(text)) { + score += 2.2; + } + if (REM_BUILD_SIGNAL_RE.test(text)) { + score += 1.2; + } + if (REM_INCIDENT_SIGNAL_RE.test(text)) { + score += 1.2; + } + if (REM_LOGISTICS_SIGNAL_RE.test(text)) { + score += 0.9; + } + if (REM_ROUTING_SIGNAL_RE.test(text)) { + score += 1.4; + } + if (REM_EXTERNALIZATION_SIGNAL_RE.test(text)) { + score += 1.1; + } + if (REM_RETRY_SIGNAL_RE.test(text)) { + score += 0.9; + } + if (REM_TASK_SIGNAL_RE.test(text) && !REM_BUILD_SIGNAL_RE.test(text)) { + score -= 0.8; + } + if (title && !REM_GENERIC_SECTION_RE.test(title)) { + score += 0.25; + } + return score; +} + +function chooseSummarySnippets( + section: ParsedMarkdownSection, + snippets: SectionSnippet[], +): SectionSnippet[] { + const selectionLimit = REM_GENERIC_SECTION_RE.test(section.title) ? 2 : 3; + return [...snippets] + .toSorted((left, right) => { + const scoreDelta = + scoreSnippet(right.text, section.title) - scoreSnippet(left.text, section.title); + if (scoreDelta !== 0) { + return scoreDelta; + } + return left.line - right.line; + }) + .slice(0, selectionLimit) + .toSorted((left, right) => left.line - right.line); +} + +function joinSummaryParts(parts: string[]): string { + if (parts.length <= 1) { + return parts[0] ?? ""; + } + if (parts.length === 2) { + return `${parts[0]} and ${parts[1]}`; + } + return `${parts.slice(0, -1).join("; ")}; and ${parts[parts.length - 1]}`; +} + +function summarizeSection( + pathValue: string, + section: ParsedMarkdownSection, +): SectionSummary | null { + if (REM_BLOCKED_SECTION_RE.test(section.title)) { + return null; + } + const snippets = sectionToSnippets(section); + if (snippets.length === 0) { + return null; + } + const selected = chooseSummarySnippets(section, snippets); + if (selected.length === 0) { + return null; + } + const title = sanitizeSectionTitle(section.title); + const body = joinSummaryParts(selected.map((snippet) => snippet.text)); + const text = !title || REM_GENERIC_SECTION_RE.test(title) ? body : `${title}: ${body}`; + return { + title, + text, + refs: selected.map((snippet) => makeRef(pathValue, snippet.line)), + scores: scoreSection(section, snippets), + }; +} + +function compactCandidateTitle(title: string): string { + let compact = sanitizeSectionTitle(title) + .replace(/\s*\((?:via:|from qmd \+ memory|this session)[^)]+\)\s*/gi, " ") + .replace( + /\s*[—-]\s*(?:research results.*|in progress.*|working.*|installed.*|booked.*|proposed.*|clarified.*|candidate.*|fixes.*|updates?.*)$/i, + "", + ) + .trim(); + if (/^(?:preferences? learned|candidate facts?)$/i.test(compact)) { + return ""; + } + compact = compact.replace(/^preference:\s*/i, ""); + return compact; +} + +function compactCandidateSnippetText(text: string, title: string): string { + const normalized = normalizeWhitespace(text); + if (REM_STABLE_PERSON_SIGNAL_RE.test(`${title} ${normalized}`)) { + return (normalized.split(/(?<=[.?!])\s+/)[0] ?? normalized).trim(); + } + return normalized; +} + +function scoreCandidateSnippet(text: string, title: string): number { + let score = 0; + if (REM_PERSISTENCE_SIGNAL_RE.test(text)) { + score += 3.2; + } + if (REM_MEMORY_SIGNAL_RE.test(text)) { + score += 2.4; + } + if (REM_EXPLICIT_PREFERENCE_SIGNAL_RE.test(text)) { + score += 1.8; + } + if (REM_PERSON_PATTERN_SIGNAL_RE.test(text)) { + score += 2.3; + } + if (REM_OPERATOR_RULE_SIGNAL_RE.test(text)) { + score += 1.6; + } + if (REM_SECTION_PERSISTENCE_TITLE_RE.test(title)) { + score += 1.2; + } + if (REM_STABLE_PERSON_SIGNAL_RE.test(text)) { + score += 1.5; + } + if (REM_METADATA_HEAVY_SIGNAL_RE.test(text)) { + score -= 2.4; + } + if (REM_PROJECT_META_SIGNAL_RE.test(`${title} ${text}`)) { + score -= 2.2; + } + if (REM_PROCESS_FRAME_SIGNAL_RE.test(text)) { + score -= 2.4; + } + if (REM_TOOLING_META_SIGNAL_RE.test(text) && !REM_STABLE_PERSON_SIGNAL_RE.test(text)) { + score -= 2.1; + } + if (REM_TRAVEL_DECISION_SIGNAL_RE.test(text)) { + score -= 2.6; + } + if (REM_SPECIFICITY_BURDEN_RE.test(text) && !REM_STABLE_PERSON_SIGNAL_RE.test(text)) { + score -= 1.2; + } + if (REM_SITUATIONAL_SIGNAL_RE.test(text)) { + score -= 2.8; + } + if (REM_TRANSIENT_SIGNAL_RE.test(text)) { + score -= 2; + } + if (REM_INCIDENT_SIGNAL_RE.test(text)) { + score -= 1.6; + } + if (REM_TASK_SIGNAL_RE.test(text)) { + score -= 1.2; + } + if (REM_LOGISTICS_SIGNAL_RE.test(text) && !REM_MEMORY_SIGNAL_RE.test(text)) { + score -= 1.4; + } + if (REM_BUILD_SIGNAL_RE.test(text) && !REM_MEMORY_SIGNAL_RE.test(text)) { + score -= 0.8; + } + if (REM_SECTION_TRANSIENT_TITLE_RE.test(title) && !REM_SECTION_PERSISTENCE_TITLE_RE.test(title)) { + score -= 1.2; + } + if (/[`/]/.test(text) || /https?:\/\//i.test(text)) { + score -= 0.8; + } + return score; +} + +function chooseFactSnippets( + section: ParsedMarkdownSection, + snippets: SectionSnippet[], +): SectionSnippet[] { + return [...snippets] + .map((snippet) => { + const text = compactCandidateSnippetText(snippet.text, section.title); + const score = + scoreCandidateSnippet(text, section.title) + (REM_MEMORY_SIGNAL_RE.test(text) ? 0.6 : 0); + return { snippet: { ...snippet, text }, score }; + }) + .filter((entry) => entry.snippet.text.length >= 18 && entry.score >= 1.4) + .toSorted((left, right) => { + if (right.score !== left.score) { + return right.score - left.score; + } + return left.snippet.line - right.snippet.line; + }) + .slice(0, 2) + .map((entry) => entry.snippet) + .toSorted((left, right) => left.line - right.line); +} + +type FactSnippetSummary = GroundedRemPreviewItem & { + score: number; +}; + +function buildFactText(title: string, text: string): string { + const compactTitle = compactCandidateTitle(title); + if (!compactTitle) { + return text; + } + if ( + REM_SECTION_PERSISTENCE_TITLE_RE.test(compactTitle) || + REM_STABLE_PERSON_SIGNAL_RE.test(compactTitle) || + /\b(relationship|people mentioned|people update|identity)\b/i.test(compactTitle) + ) { + return `${compactTitle}: ${text}`; + } + return text; +} + +function chooseCandidateSnippets( + section: ParsedMarkdownSection, + snippets: SectionSnippet[], +): SectionSnippet[] { + return [...snippets] + .map((snippet) => { + const text = compactCandidateSnippetText(snippet.text, section.title); + const score = scoreCandidateSnippet(text, section.title); + return { snippet: { ...snippet, text }, score }; + }) + .filter((entry) => entry.snippet.text.length >= 18 && entry.score >= 1.8) + .toSorted((left, right) => { + if (right.score !== left.score) { + return right.score - left.score; + } + return left.snippet.line - right.snippet.line; + }) + .slice(0, 2) + .map((entry) => entry.snippet) + .toSorted((left, right) => left.line - right.line); +} + +function buildCandidateSnippetText(title: string, text: string): string { + return buildFactText(title, text); +} + +function classifyCandidateLeanFromText(text: string, title: string): GroundedRemCandidate["lean"] { + const score = scoreCandidateSnippet(text, title); + if (score >= 4) { + return "likely_durable"; + } + if (score <= 0.25 || REM_SITUATIONAL_SIGNAL_RE.test(text) || REM_TRANSIENT_SIGNAL_RE.test(text)) { + return "likely_situational"; + } + return "unclear"; +} + +function addReflection( + reflections: GroundedRemPreviewItem[], + seen: Set, + text: string, + refs: string[], +) { + const normalized = normalizeWhitespace(text); + const key = normalized.toLowerCase(); + if (!normalized || seen.has(key)) { + return; + } + seen.add(key); + reflections.push({ text: normalized, refs }); +} + +function isOperatorRuleSummary(summary: SectionSummary): boolean { + return ( + /process improvements?/i.test(summary.title) || REM_OPERATOR_RULE_SIGNAL_RE.test(summary.text) + ); +} + +function isRoutingSummary(summary: SectionSummary): boolean { + return summary.scores.routing > 0 || REM_ROUTING_SIGNAL_RE.test(summary.text); +} + +function previewGroundedRemForFile(params: { + relPath: string; + content: string; +}): GroundedRemFilePreview { + const sections = parseMarkdownSections(params.content); + const sectionScores = sections.map((section) => ({ + section, + snippets: sectionToSnippets(section), + })); + const summaries = sectionScores + .map(({ section }) => summarizeSection(params.relPath, section)) + .filter((summary): summary is SectionSummary => summary !== null); + const factSummaries: FactSnippetSummary[] = sections.flatMap((section) => { + if (REM_BLOCKED_SECTION_RE.test(section.title)) { + return []; + } + const snippets = sectionToSnippets(section); + if (snippets.length === 0) { + return []; + } + return chooseFactSnippets(section, snippets).map((snippet) => ({ + text: buildFactText(section.title, snippet.text), + refs: [makeRef(params.relPath, snippet.line)], + score: scoreCandidateSnippet(snippet.text, section.title), + })); + }); + + const memoryImplications = summaries + .filter((summary) => summary.scores.preference > 0 || isOperatorRuleSummary(summary)) + .map((summary) => ({ + text: summary.text.replace(/^[^:]+:\s*/, ""), + refs: summary.refs, + })) + .filter((item, index, items) => items.findIndex((entry) => entry.text === item.text) === index) + .slice(0, REM_SUMMARY_MEMORY_LIMIT); + + const candidateSnippets: CandidateSnippetSummary[] = sections.flatMap((section) => { + if (REM_BLOCKED_SECTION_RE.test(section.title)) { + return []; + } + const snippets = sectionToSnippets(section); + if (snippets.length === 0) { + return []; + } + return chooseCandidateSnippets(section, snippets) + .map((snippet) => { + const score = scoreCandidateSnippet(snippet.text, section.title); + const text = buildCandidateSnippetText(section.title, snippet.text); + return { + text, + refs: [makeRef(params.relPath, snippet.line)], + lean: classifyCandidateLeanFromText(snippet.text, section.title), + score, + }; + }) + .filter((candidate) => candidate.text.length >= 12 && candidate.score >= 1.8); + }); + + const candidates = candidateSnippets + .toSorted((left, right) => { + const leanRank = { likely_durable: 0, unclear: 1, likely_situational: 2 }; + const leanDelta = leanRank[left.lean] - leanRank[right.lean]; + if (leanDelta !== 0) { + return leanDelta; + } + return right.score - left.score; + }) + .filter( + (candidate, index, items) => + items.findIndex((entry) => entry.text === candidate.text) === index, + ) + .slice(0, 4); + + const durableImplications = candidateSnippets + .filter((candidate) => candidate.lean === "likely_durable" || candidate.score >= 4) + .filter( + (candidate, index, items) => + items.findIndex((entry) => entry.text === candidate.text) === index, + ) + .toSorted((left, right) => right.score - left.score) + .slice(0, REM_SUMMARY_MEMORY_LIMIT) + .map((candidate) => ({ text: candidate.text, refs: candidate.refs })); + + const candidateDrivenImplications = candidateSnippets + .filter((candidate) => candidate.lean !== "likely_situational" && candidate.score >= 2.2) + .filter( + (candidate, index, items) => + items.findIndex((entry) => entry.text === candidate.text) === index, + ) + .toSorted((left, right) => right.score - left.score) + .slice(0, REM_SUMMARY_MEMORY_LIMIT) + .map((candidate) => ({ text: candidate.text, refs: candidate.refs })); + + const effectiveMemoryImplications = + durableImplications.length > 0 + ? durableImplications + : candidateDrivenImplications.length > 0 + ? candidateDrivenImplications + : memoryImplications; + + const facts: GroundedRemPreviewItem[] = []; + const usedFactTexts = new Set(); + for (const summary of factSummaries.toSorted((left, right) => right.score - left.score)) { + const key = summary.text.toLowerCase(); + if (usedFactTexts.has(key)) { + continue; + } + usedFactTexts.add(key); + facts.push({ text: summary.text, refs: summary.refs }); + if (facts.length >= REM_SUMMARY_FACT_LIMIT) { + break; + } + } + if (facts.length === 0) { + const bestFor = (metric: keyof SectionSummary["scores"]) => + summaries + .filter((summary) => summary.scores[metric] > 0) + .toSorted((left, right) => { + if (right.scores[metric] !== left.scores[metric]) { + return right.scores[metric] - left.scores[metric]; + } + return right.scores.overall - left.scores.overall; + })[0]; + for (const summary of [ + bestFor("preference"), + bestFor("routing"), + bestFor("externalization"), + ...summaries.toSorted((left, right) => right.scores.overall - left.scores.overall), + ]) { + if (!summary) { + continue; + } + const key = summary.text.toLowerCase(); + if (usedFactTexts.has(key)) { + continue; + } + usedFactTexts.add(key); + facts.push({ text: summary.text, refs: summary.refs }); + if (facts.length >= REM_SUMMARY_FACT_LIMIT) { + break; + } + } + } + + const reflections: GroundedRemPreviewItem[] = []; + const seenReflections = new Set(); + const buildSignal = summaries.reduce((sum, item) => sum + item.scores.build, 0); + const incidentSignal = summaries.reduce((sum, item) => sum + item.scores.incident, 0); + const logisticsSignal = summaries.reduce((sum, item) => sum + item.scores.logistics, 0); + const routingSignal = summaries.reduce((sum, item) => sum + item.scores.routing, 0); + const externalizationSignal = summaries.reduce( + (sum, item) => sum + item.scores.externalization, + 0, + ); + const retrySignal = summaries.reduce((sum, item) => sum + item.scores.retries, 0); + const taskSignal = sectionScores.reduce( + (sum, { section, snippets }) => sum + scoreSection(section, snippets).tasks, + 0, + ); + const strongestRoutingSummary = summaries + .filter((summary) => isRoutingSummary(summary)) + .toSorted((left, right) => right.scores.overall - left.scores.overall)[0]; + const strongestIncidentSummary = summaries + .filter((summary) => summary.scores.incident > 0) + .toSorted((left, right) => right.scores.overall - left.scores.overall)[0]; + const strongestExternalizationSummary = summaries + .filter((summary) => summary.scores.externalization > 0) + .toSorted((left, right) => right.scores.overall - left.scores.overall)[0]; + + if (effectiveMemoryImplications.length > 0) { + addReflection( + reflections, + seenReflections, + "A stable rule or preference was stated explicitly, which suggests operating choices are being made legible instead of left implicit.", + effectiveMemoryImplications.flatMap((item) => item.refs).slice(0, 3), + ); + } + if (facts.length > 0 && routingSignal >= 2 && strongestRoutingSummary && buildSignal >= incidentSignal) { + addReflection( + reflections, + seenReflections, + "The strongest pattern here is a preference for converting messy inbound information into routed workflows with different downstream actions, instead of handling each case manually.", + strongestRoutingSummary.refs, + ); + } + if (facts.length > 0 && externalizationSignal >= 2 && strongestExternalizationSummary) { + addReflection( + reflections, + seenReflections, + "Important context tends to get externalized quickly into notes, trackers, or memory surfaces, which suggests a preference for explicit systems over holding context informally.", + strongestExternalizationSummary.refs, + ); + } + if (facts.length > 0 && buildSignal >= 2) { + const buildRefs = facts + .filter((item) => REM_BUILD_SIGNAL_RE.test(item.text)) + .flatMap((item) => item.refs) + .slice(0, 3); + if (buildRefs.length > 0) { + addReflection( + reflections, + seenReflections, + "The day leaned toward building operator infrastructure, which suggests the interaction is often used to reshape the system around recurring needs rather than just complete isolated tasks.", + buildRefs, + ); + } + } + if (facts.length > 0 && incidentSignal >= 2 && strongestIncidentSummary) { + addReflection( + reflections, + seenReflections, + retrySignal >= 2 + ? "When something breaks repeatedly, the response is systematic: retries, root-cause narrowing, and preserving enough state to resume once the blocker is fixed." + : "A meaningful share of the day went into friction, and the interaction pattern looks pragmatic rather than emotional: diagnose the blocker, preserve state, and move on.", + strongestIncidentSummary.refs, + ); + } + if (facts.length > 0 && logisticsSignal >= 2) { + const logisticsRefs = facts + .filter((item) => REM_LOGISTICS_SIGNAL_RE.test(item.text)) + .flatMap((item) => item.refs) + .slice(0, 3); + if (logisticsRefs.length > 0) { + addReflection( + reflections, + seenReflections, + "Personal logistics and operating-system work are being managed in the same surface, which suggests a preference for one integrated control plane rather than separate personal and technical loops.", + logisticsRefs, + ); + } + } + if (taskSignal >= 3 && reflections.length === 0) { + addReflection( + reflections, + seenReflections, + "The raw note is mostly task and current-state material, so it should not be over-read as memory.", + [ + makeRef( + params.relPath, + sections[0]?.startLine ?? 1, + sections[sections.length - 1]?.endLine ?? 1, + ), + ], + ); + } + + const visibleReflections = reflections.slice(0, REM_SUMMARY_REFLECTION_LIMIT); + + const renderedLines: string[] = []; + renderedLines.push("## What Happened"); + if (facts.length === 0) { + renderedLines.push("1. No grounded facts were extracted."); + } else { + for (const [index, fact] of facts.entries()) { + renderedLines.push(`${index + 1}. ${fact.text} [${fact.refs.join(", ")}]`); + } + } + renderedLines.push(""); + renderedLines.push("## Reflections"); + if (visibleReflections.length === 0) { + renderedLines.push("1. No grounded reflections emerged from this note yet."); + } else { + for (const [index, reflection] of visibleReflections.entries()) { + renderedLines.push(`${index + 1}. ${reflection.text} [${reflection.refs.join(", ")}]`); + } + } + if (candidates.length > 0) { + renderedLines.push(""); + renderedLines.push("## Candidates"); + for (const candidate of candidates) { + renderedLines.push(`- [${candidate.lean}] ${candidate.text} [${candidate.refs.join(", ")}]`); + } + } + if (effectiveMemoryImplications.length > 0) { + renderedLines.push(""); + renderedLines.push("## Possible Lasting Updates"); + for (const implication of effectiveMemoryImplications) { + renderedLines.push(`- ${implication.text} [${implication.refs.join(", ")}]`); + } + } + + return { + path: params.relPath, + facts, + reflections: visibleReflections, + memoryImplications: effectiveMemoryImplications, + candidates, + renderedMarkdown: renderedLines.join("\n"), + }; +} + +async function collectMarkdownFiles(inputPaths: string[]): Promise { + const found = new Set(); + async function walk(targetPath: string): Promise { + const resolved = path.resolve(targetPath); + const stat = await fs.stat(resolved); + if (stat.isDirectory()) { + const entries = await fs.readdir(resolved, { withFileTypes: true }); + for (const entry of entries) { + await walk(path.join(resolved, entry.name)); + } + return; + } + if (stat.isFile() && resolved.toLowerCase().endsWith(".md")) { + found.add(resolved); + } + } + for (const inputPath of inputPaths) { + const trimmed = inputPath.trim(); + if (!trimmed) { + continue; + } + await walk(trimmed); + } + return Array.from(found).toSorted((left, right) => left.localeCompare(right)); +} + +export async function previewGroundedRemMarkdown(params: { + workspaceDir: string; + inputPaths: string[]; +}): Promise { + const workspaceDir = params.workspaceDir.trim(); + const files = await collectMarkdownFiles(params.inputPaths); + const previews: GroundedRemFilePreview[] = []; + for (const filePath of files) { + const content = await fs.readFile(filePath, "utf-8"); + const relPath = normalizePath(path.relative(workspaceDir, filePath)); + previews.push(previewGroundedRemForFile({ relPath, content })); + } + return { + workspaceDir, + scannedFiles: files.length, + files: previews, + }; +}