diff --git a/CHANGELOG.md b/CHANGELOG.md index 224721a6fe5..17bf6013d43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Docs: https://docs.openclaw.ai - Plugins/provider-auth: let provider manifests declare `providerAuthAliases` so provider variants can share env vars, auth profiles, config-backed auth, and API-key onboarding choices without core-specific wiring. - Memory/dreaming: add a grounded REM backfill lane with historical `rem-harness --path`, diary commit, and reset flows so old daily notes can be replayed safely into `DREAMS.md`. Thanks @mbelinky. - Memory/dreaming: harden grounded diary extraction so `What Happened`, `Reflections`, and durable candidates suppress operational noise and preserve more atomic lasting facts. Thanks @mbelinky. +- Memory/dreaming: feed grounded durable backfill candidates into the live short-term promotion store so historical diary replays can flow through the normal deep promotion path without a second memory stack. Thanks @mbelinky. - Control UI/dreaming: add a structured diary view with timeline navigation, backfill/reset controls, and traceable dreaming summaries. Thanks @mbelinky. ### Fixes diff --git a/extensions/memory-core/src/cli.runtime.ts b/extensions/memory-core/src/cli.runtime.ts index 8401411917d..dc79746aad2 100644 --- a/extensions/memory-core/src/cli.runtime.ts +++ b/extensions/memory-core/src/cli.runtime.ts @@ -42,8 +42,10 @@ import { previewGroundedRemMarkdown } from "./rem-evidence.js"; import { applyShortTermPromotions, auditShortTermPromotionArtifacts, + removeGroundedShortTermCandidates, repairShortTermPromotionArtifacts, readShortTermRecallEntries, + recordGroundedShortTermCandidates, recordShortTermRecalls, rankShortTermPromotionCandidates, resolveShortTermRecallLockPath, @@ -296,6 +298,100 @@ function groundedMarkdownToDiaryLines(markdown: string): string[] { .filter((line, index, lines) => !(line.length === 0 && lines[index - 1]?.length === 0)); } +function parseGroundedRef( + fallbackPath: string, + ref: string, +): { path: string; startLine: number; endLine: number } | null { + const trimmed = ref.trim(); + if (!trimmed) { + return null; + } + const match = trimmed.match(/^(.*?):(\d+)(?:-(\d+))?$/); + if (!match) { + return null; + } + return { + path: (match[1] ?? fallbackPath).replaceAll("\\", "/").replace(/^\.\//, ""), + startLine: Math.max(1, Number(match[2])), + endLine: Math.max(1, Number(match[3] ?? match[2])), + }; +} + +function collectGroundedShortTermSeedItems( + previews: Awaited>["files"], +): Array<{ + path: string; + startLine: number; + endLine: number; + snippet: string; + score: number; + query: string; + signalCount: number; + dayBucket?: string; +}> { + const items: Array<{ + path: string; + startLine: number; + endLine: number; + snippet: string; + score: number; + query: string; + signalCount: number; + dayBucket?: string; + }> = []; + const seen = new Set(); + + for (const file of previews) { + const dayBucket = extractIsoDayFromPath(file.path) ?? undefined; + const signals = [ + ...file.memoryImplications.map((item) => ({ + text: item.text, + refs: item.refs, + score: 0.92, + query: "__dreaming_grounded_backfill__:lasting-update", + signalCount: 2, + })), + ...file.candidates + .filter((candidate) => candidate.lean === "likely_durable") + .map((candidate) => ({ + text: candidate.text, + refs: candidate.refs, + score: 0.82, + query: "__dreaming_grounded_backfill__:candidate", + signalCount: 1, + })), + ]; + + for (const signal of signals) { + if (!signal.text.trim()) { + continue; + } + const firstRef = signal.refs.find((ref) => ref.trim().length > 0); + const parsedRef = firstRef ? parseGroundedRef(file.path, firstRef) : null; + if (!parsedRef) { + continue; + } + const key = `${parsedRef.path}:${parsedRef.startLine}:${parsedRef.endLine}:${signal.query}:${signal.text.toLowerCase()}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + items.push({ + path: parsedRef.path, + startLine: parsedRef.startLine, + endLine: parsedRef.endLine, + snippet: signal.text, + score: signal.score, + query: signal.query, + signalCount: signal.signalCount, + ...(dayBucket ? { dayBucket } : {}), + }); + } + } + + return items; +} + function matchesPromotionSelector( candidate: { key: string; @@ -551,9 +647,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { purpose: managerPurpose, run: async (manager) => { const deep = Boolean(opts.deep || opts.index); - let embeddingProbe: - | Awaited> - | undefined; + let embeddingProbe: { ok?: boolean; error?: string } | undefined; let indexError: string | undefined; const syncFn = manager.sync ? manager.sync.bind(manager) : undefined; if (deep) { @@ -1548,14 +1642,30 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) { return; } - if (opts.rollback) { - const removed = await removeBackfillDiaryEntries({ workspaceDir }); + if (opts.rollback || opts.rollbackShortTerm) { + const diaryRollback = opts.rollback + ? await removeBackfillDiaryEntries({ workspaceDir }) + : null; + const shortTermRollback = opts.rollbackShortTerm + ? await removeGroundedShortTermCandidates({ workspaceDir }) + : null; if (opts.json) { defaultRuntime.writeJson({ workspaceDir, - rollback: true, - dreamsPath: removed.dreamsPath, - removedEntries: removed.removed, + rollback: Boolean(opts.rollback), + rollbackShortTerm: Boolean(opts.rollbackShortTerm), + ...(diaryRollback + ? { + dreamsPath: diaryRollback.dreamsPath, + removedEntries: diaryRollback.removed, + } + : {}), + ...(shortTermRollback + ? { + shortTermStorePath: shortTermRollback.storePath, + removedShortTermEntries: shortTermRollback.removed, + } + : {}), }); return; } @@ -1563,8 +1673,30 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) { [ `${colorize(isRich(), theme.heading, "REM Backfill")} ${colorize(isRich(), theme.muted, "(rollback)")}`, colorize(isRich(), theme.muted, `workspace=${shortenHomePath(workspaceDir)}`), - colorize(isRich(), theme.muted, `dreamsPath=${shortenHomePath(removed.dreamsPath)}`), - colorize(isRich(), theme.muted, `removedEntries=${removed.removed}`), + ...(diaryRollback + ? [ + colorize( + isRich(), + theme.muted, + `dreamsPath=${shortenHomePath(diaryRollback.dreamsPath)}`, + ), + colorize(isRich(), theme.muted, `removedEntries=${diaryRollback.removed}`), + ] + : []), + ...(shortTermRollback + ? [ + colorize( + isRich(), + theme.muted, + `shortTermStorePath=${shortenHomePath(shortTermRollback.storePath)}`, + ), + colorize( + isRich(), + theme.muted, + `removedShortTermEntries=${shortTermRollback.removed}`, + ), + ] + : []), ].join("\n"), ); return; @@ -1619,6 +1751,24 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) { entries, timezone: remConfig.timezone, }); + let stagedShortTermEntries = 0; + let replacedShortTermEntries = 0; + if (opts.stageShortTerm) { + const cleared = await removeGroundedShortTermCandidates({ workspaceDir }); + replacedShortTermEntries = cleared.removed; + const shortTermSeedItems = collectGroundedShortTermSeedItems(grounded.files); + if (shortTermSeedItems.length > 0) { + await recordGroundedShortTermCandidates({ + workspaceDir, + query: "__dreaming_grounded_backfill__", + items: shortTermSeedItems, + dedupeByQueryPerDay: true, + nowMs: Date.now(), + timezone: remConfig.timezone, + }); + } + stagedShortTermEntries = shortTermSeedItems.length; + } if (opts.json) { defaultRuntime.writeJson({ @@ -1629,6 +1779,12 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) { writtenEntries: written.written, replacedEntries: written.replaced, dreamsPath: written.dreamsPath, + ...(opts.stageShortTerm + ? { + stagedShortTermEntries, + replacedShortTermEntries, + } + : {}), }); return; } @@ -1644,6 +1800,15 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) { theme.muted, `historicalFiles=${sourceFiles.length} writtenEntries=${written.written} replacedEntries=${written.replaced}`, ), + ...(opts.stageShortTerm + ? [ + colorize( + rich, + theme.muted, + `stagedShortTermEntries=${stagedShortTermEntries} replacedShortTermEntries=${replacedShortTermEntries}`, + ), + ] + : []), colorize(rich, theme.muted, `dreamsPath=${shortenHomePath(written.dreamsPath)}`), ].join("\n"), ); diff --git a/extensions/memory-core/src/cli.test.ts b/extensions/memory-core/src/cli.test.ts index 7333c6730ba..14f654d28ef 100644 --- a/extensions/memory-core/src/cli.test.ts +++ b/extensions/memory-core/src/cli.test.ts @@ -9,7 +9,7 @@ import { spyRuntimeJson, spyRuntimeLogs, } from "../../../src/cli/test-runtime-capture.js"; -import { recordShortTermRecalls } from "./short-term-promotion.js"; +import { readShortTermRecallEntries, recordShortTermRecalls } from "./short-term-promotion.js"; const getMemorySearchManager = vi.hoisted(() => vi.fn()); const loadConfig = vi.hoisted(() => vi.fn(() => ({}))); @@ -1074,6 +1074,71 @@ describe("memory cli", () => { }); }); + it("stages grounded durable candidates into the live short-term store", async () => { + await withTempWorkspace(async (workspaceDir) => { + const historyDir = path.join(workspaceDir, "history"); + await fs.mkdir(historyDir, { recursive: true }); + const historyPath = path.join(historyDir, "2025-01-01.md"); + await fs.writeFile( + historyPath, + [ + "## Preferences Learned", + '- Always use "Happy Together" calendar for flights and reservations.', + ].join("\n") + "\n", + "utf-8", + ); + + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + + await runMemoryCli(["rem-backfill", "--path", historyPath, "--stage-short-term"]); + + const entries = await readShortTermRecallEntries({ workspaceDir }); + expect(entries).toHaveLength(1); + expect(entries[0]?.snippet).toContain("Happy Together"); + expect(entries[0]?.groundedCount).toBe(3); + expect(entries[0]?.queryHashes).toHaveLength(2); + expect(entries[0]?.recallCount).toBe(0); + expect(close).toHaveBeenCalled(); + }); + }); + + it("rolls back grounded staged short-term entries without touching diary rollback", async () => { + await withTempWorkspace(async (workspaceDir) => { + const historyDir = path.join(workspaceDir, "history"); + await fs.mkdir(historyDir, { recursive: true }); + const historyPath = path.join(historyDir, "2025-01-01.md"); + await fs.writeFile( + historyPath, + [ + "## Preferences Learned", + '- Always use "Happy Together" calendar for flights and reservations.', + ].join("\n") + "\n", + "utf-8", + ); + + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + + await runMemoryCli(["rem-backfill", "--path", historyPath, "--stage-short-term"]); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + await runMemoryCli(["rem-backfill", "--rollback-short-term"]); + + const entries = await readShortTermRecallEntries({ workspaceDir }); + expect(entries).toHaveLength(0); + expect(close).toHaveBeenCalled(); + }); + }); + it("prefers persistence-relevant evidence over narrated operational logs in grounded what happened", async () => { await withTempWorkspace(async (workspaceDir) => { const historyDir = path.join(workspaceDir, "history"); diff --git a/extensions/memory-core/src/cli.ts b/extensions/memory-core/src/cli.ts index 8c836cc052f..38bc691f2a5 100644 --- a/extensions/memory-core/src/cli.ts +++ b/extensions/memory-core/src/cli.ts @@ -105,6 +105,10 @@ export function registerMemoryCli(program: Command) { "openclaw memory rem-backfill --path ./memory", "Write grounded historical REM entries into DREAMS.md for UI review.", ], + [ + "openclaw memory rem-backfill --path ./memory --stage-short-term", + "Also seed durable grounded candidates into the live short-term promotion store.", + ], ["openclaw memory status --json", "Output machine-readable JSON (good for scripts)."], ])}\n\n${theme.muted("Docs:")} ${formatDocsLink("/cli/memory", "docs.openclaw.ai/cli/memory")}\n`, ); @@ -201,6 +205,16 @@ export function registerMemoryCli(program: Command) { .option("--agent ", "Agent id (default: default agent)") .option("--path ", "Historical daily memory file(s) or directory") .option("--rollback", "Remove previously written grounded REM backfill entries", false) + .option( + "--stage-short-term", + "Also seed grounded durable candidates into the short-term promotion store", + false, + ) + .option( + "--rollback-short-term", + "Remove previously seeded grounded short-term candidates", + false, + ) .option("--json", "Print JSON") .action(async (opts: MemoryRemBackfillOptions) => { await runMemoryRemBackfill(opts); diff --git a/extensions/memory-core/src/cli.types.ts b/extensions/memory-core/src/cli.types.ts index 842bbcb29ce..315644a1b48 100644 --- a/extensions/memory-core/src/cli.types.ts +++ b/extensions/memory-core/src/cli.types.ts @@ -36,4 +36,6 @@ export type MemoryRemHarnessOptions = MemoryCommandOptions & { export type MemoryRemBackfillOptions = MemoryCommandOptions & { path?: string; rollback?: boolean; + stageShortTerm?: boolean; + rollbackShortTerm?: boolean; }; diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index 50244307313..8a320fc89e7 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -11,9 +11,11 @@ import { applyShortTermPromotions, auditShortTermPromotionArtifacts, isShortTermMemoryPath, + recordGroundedShortTermCandidates, rankShortTermPromotionCandidates, recordDreamingPhaseSignals, recordShortTermRecalls, + removeGroundedShortTermCandidates, repairShortTermPromotionArtifacts, resolveShortTermRecallLockPath, resolveShortTermPhaseSignalStorePath, @@ -177,6 +179,128 @@ describe("short-term promotion", () => { }); }); + it("lets grounded durable evidence satisfy default deep thresholds", async () => { + await withTempWorkspace(async (workspaceDir) => { + await writeDailyMemoryNote(workspaceDir, "2026-04-03", [ + 'Always use "Happy Together" calendar for flights and reservations.', + ]); + + await recordGroundedShortTermCandidates({ + workspaceDir, + query: "__dreaming_grounded_backfill__", + items: [ + { + path: "memory/2026-04-03.md", + startLine: 1, + endLine: 1, + snippet: 'Always use "Happy Together" calendar for flights and reservations.', + score: 0.92, + query: "__dreaming_grounded_backfill__:lasting-update", + signalCount: 2, + dayBucket: "2026-04-03", + }, + { + path: "memory/2026-04-03.md", + startLine: 1, + endLine: 1, + snippet: 'Always use "Happy Together" calendar for flights and reservations.', + score: 0.82, + query: "__dreaming_grounded_backfill__:candidate", + signalCount: 1, + dayBucket: "2026-04-03", + }, + ], + dedupeByQueryPerDay: true, + nowMs: Date.parse("2026-04-03T10:00:00.000Z"), + }); + + const ranked = await rankShortTermPromotionCandidates({ + workspaceDir, + nowMs: Date.parse("2026-04-03T10:00:00.000Z"), + }); + + expect(ranked).toHaveLength(1); + expect(ranked[0]?.groundedCount).toBe(3); + expect(ranked[0]?.uniqueQueries).toBe(2); + expect(ranked[0]?.avgScore).toBeGreaterThan(0.85); + + const applied = await applyShortTermPromotions({ + workspaceDir, + candidates: ranked, + nowMs: Date.parse("2026-04-03T10:00:00.000Z"), + }); + + expect(applied.applied).toBe(1); + const memory = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"); + expect(memory).toContain('Always use "Happy Together" calendar'); + }); + }); + + it("removes grounded-only staged entries without deleting mixed live entries", async () => { + await withTempWorkspace(async (workspaceDir) => { + await writeDailyMemoryNote(workspaceDir, "2026-04-03", [ + "Grounded only rule.", + "Live recall-backed rule.", + ]); + + await recordGroundedShortTermCandidates({ + workspaceDir, + query: "__dreaming_grounded_backfill__", + items: [ + { + path: "memory/2026-04-03.md", + startLine: 1, + endLine: 1, + snippet: "Grounded only rule.", + score: 0.92, + query: "__dreaming_grounded_backfill__:lasting-update", + signalCount: 2, + dayBucket: "2026-04-03", + }, + { + path: "memory/2026-04-03.md", + startLine: 2, + endLine: 2, + snippet: "Live recall-backed rule.", + score: 0.92, + query: "__dreaming_grounded_backfill__:lasting-update", + signalCount: 2, + dayBucket: "2026-04-03", + }, + ], + dedupeByQueryPerDay: true, + }); + await recordShortTermRecalls({ + workspaceDir, + query: "live recall", + results: [ + { + path: "memory/2026-04-03.md", + startLine: 2, + endLine: 2, + score: 0.87, + snippet: "Live recall-backed rule.", + source: "memory", + }, + ], + }); + + const result = await removeGroundedShortTermCandidates({ workspaceDir }); + expect(result.removed).toBe(1); + + const ranked = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + expect(ranked).toHaveLength(1); + expect(ranked[0]?.snippet).toContain("Live recall-backed rule"); + expect(ranked[0]?.groundedCount).toBe(2); + expect(ranked[0]?.recallCount).toBe(1); + }); + }); + it("rewards spaced recalls as consolidation instead of only raw count", async () => { await withTempWorkspace(async (workspaceDir) => { await recordShortTermRecalls({ @@ -1100,6 +1224,7 @@ describe("short-term promotion", () => { snippet, recallCount: 2, dailyCount: 0, + groundedCount: 0, totalScore: 1.8, maxScore: 0.95, firstRecalledAt: "2026-04-01T00:00:00.000Z", diff --git a/extensions/memory-core/src/short-term-promotion.ts b/extensions/memory-core/src/short-term-promotion.ts index 9e05a247fb7..416a6909e8f 100644 --- a/extensions/memory-core/src/short-term-promotion.ts +++ b/extensions/memory-core/src/short-term-promotion.ts @@ -64,6 +64,7 @@ export type ShortTermRecallEntry = { snippet: string; recallCount: number; dailyCount: number; + groundedCount: number; totalScore: number; maxScore: number; firstRecalledAt: string; @@ -71,6 +72,7 @@ export type ShortTermRecallEntry = { queryHashes: string[]; recallDays: string[]; conceptTags: string[]; + claimHash?: string; promotedAt?: string; }; @@ -112,10 +114,12 @@ export type PromotionCandidate = { snippet: string; recallCount: number; dailyCount?: number; + groundedCount?: number; signalCount?: number; avgScore: number; maxScore: number; uniqueQueries: number; + claimHash?: string; promotedAt?: string; firstRecalledAt: string; lastRecalledAt: string; @@ -232,13 +236,19 @@ function normalizeMemoryPath(rawPath: string): string { return rawPath.replaceAll("\\", "/").replace(/^\.\//, ""); } +function buildClaimHash(snippet: string): string { + return createHash("sha1").update(normalizeSnippet(snippet)).digest("hex").slice(0, 12); +} + function buildEntryKey(result: { path: string; startLine: number; endLine: number; source: string; + claimHash?: string; }): string { - return `${result.source}:${normalizeMemoryPath(result.path)}:${result.startLine}:${result.endLine}`; + const base = `${result.source}:${normalizeMemoryPath(result.path)}:${result.startLine}:${result.endLine}`; + return result.claimHash ? `${base}:${result.claimHash}` : base; } function hashQuery(query: string): string { @@ -315,6 +325,18 @@ function normalizeDistinctStrings(values: unknown[], limit: number): string[] { return normalized; } +function totalSignalCountForEntry(entry: { + recallCount?: number; + dailyCount?: number; + groundedCount?: number; +}): number { + return ( + Math.max(0, Math.floor(entry.recallCount ?? 0)) + + Math.max(0, Math.floor(entry.dailyCount ?? 0)) + + Math.max(0, Math.floor(entry.groundedCount ?? 0)) + ); +} + function calculateConsolidationComponent(recallDays: string[]): number { if (recallDays.length === 0) { return 0; @@ -371,6 +393,7 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore { const recallCount = Math.max(0, Math.floor(Number(entry.recallCount) || 0)); const dailyCount = Math.max(0, Math.floor(Number(entry.dailyCount) || 0)); + const groundedCount = Math.max(0, Math.floor(Number(entry.groundedCount) || 0)); const totalScore = Math.max(0, Number(entry.totalScore) || 0); const maxScore = clampScore(Number(entry.maxScore) || 0); const firstRecalledAt = @@ -378,6 +401,10 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore { const lastRecalledAt = typeof entry.lastRecalledAt === "string" ? entry.lastRecalledAt : nowIso; const promotedAt = typeof entry.promotedAt === "string" ? entry.promotedAt : undefined; + const claimHash = + typeof entry.claimHash === "string" && entry.claimHash.trim().length > 0 + ? entry.claimHash.trim() + : undefined; const snippet = typeof entry.snippet === "string" ? normalizeSnippet(entry.snippet) : ""; const queryHashes = Array.isArray(entry.queryHashes) ? normalizeDistinctStrings(entry.queryHashes, MAX_QUERY_HASHES) @@ -396,7 +423,8 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore { ) : deriveConceptTags({ path: entryPath, snippet }); - const normalizedKey = key || buildEntryKey({ path: entryPath, startLine, endLine, source }); + const normalizedKey = + key || buildEntryKey({ path: entryPath, startLine, endLine, source, claimHash }); entries[normalizedKey] = { key: normalizedKey, path: entryPath, @@ -406,6 +434,7 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore { snippet, recallCount, dailyCount, + groundedCount, totalScore, maxScore, firstRecalledAt, @@ -413,6 +442,7 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore { queryHashes, recallDays: recallDays.slice(-MAX_RECALL_DAYS), conceptTags, + ...(claimHash ? { claimHash } : {}), ...(promotedAt ? { promotedAt } : {}), }; } @@ -568,7 +598,7 @@ function isProcessLikelyAlive(pid: number): boolean { process.kill(pid, 0); return true; } catch (err) { - const code = (err as NodeJS.ErrnoException | undefined)?.code; + const code = (err as NodeJS.ErrnoException).code; if (code === "ESRCH") { return false; } @@ -621,9 +651,8 @@ async function withShortTermLock(workspaceDir: string, task: () => Promise const startedAt = Date.now(); while (true) { - let lockHandle: Awaited> | undefined; try { - lockHandle = await fs.open(lockPath, "wx"); + const lockHandle = await fs.open(lockPath, "wx"); await lockHandle .writeFile(`${process.pid}:${Date.now()}\n`, "utf-8") .catch(() => undefined); @@ -812,10 +841,21 @@ export async function recordShortTermRecalls(params: { const store = await readStore(workspaceDir, nowIso); for (const result of relevant) { - const key = buildEntryKey(result); const normalizedPath = normalizeMemoryPath(result.path); - const existing = store.entries[key]; const snippet = normalizeSnippet(result.snippet); + const claimHash = snippet ? buildClaimHash(snippet) : undefined; + const groundedKey = claimHash + ? buildEntryKey({ + path: normalizedPath, + startLine: Math.max(1, Math.floor(result.startLine)), + endLine: Math.max(1, Math.floor(result.endLine)), + source: "memory", + claimHash, + }) + : null; + const baseKey = buildEntryKey(result); + const key = groundedKey && store.entries[groundedKey] ? groundedKey : baseKey; + const existing = store.entries[key]; const score = clampScore(result.score); const recallDaysBase = existing?.recallDays ?? []; const queryHashesBase = existing?.queryHashes ?? []; @@ -846,6 +886,7 @@ export async function recordShortTermRecalls(params: { snippet: snippet || existing?.snippet || "", recallCount, dailyCount, + groundedCount: Math.max(0, Math.floor(existing?.groundedCount ?? 0)), totalScore, maxScore, firstRecalledAt: existing?.firstRecalledAt ?? nowIso, @@ -853,6 +894,7 @@ export async function recordShortTermRecalls(params: { queryHashes, recallDays, conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []), + ...(existing?.claimHash ? { claimHash: existing.claimHash } : {}), ...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}), }; } @@ -874,6 +916,129 @@ export async function recordShortTermRecalls(params: { }); } +export async function recordGroundedShortTermCandidates(params: { + workspaceDir?: string; + query: string; + items: Array<{ + path: string; + startLine: number; + endLine: number; + snippet: string; + score: number; + query?: string; + signalCount?: number; + dayBucket?: string; + }>; + dedupeByQueryPerDay?: boolean; + dayBucket?: string; + nowMs?: number; + timezone?: string; +}): Promise { + const workspaceDir = params.workspaceDir?.trim(); + if (!workspaceDir) { + return; + } + const query = params.query.trim(); + if (!query) { + return; + } + const relevant = params.items + .map((item) => { + const snippet = normalizeSnippet(item.snippet); + const normalizedPath = normalizeMemoryPath(item.path); + if ( + !snippet || + !normalizedPath || + !isShortTermMemoryPath(normalizedPath) || + !Number.isFinite(item.startLine) || + !Number.isFinite(item.endLine) + ) { + return null; + } + return { + path: normalizedPath, + startLine: Math.max(1, Math.floor(item.startLine)), + endLine: Math.max(1, Math.floor(item.endLine)), + snippet, + score: clampScore(item.score), + query: normalizeSnippet(item.query ?? query), + signalCount: Math.max(1, Math.floor(item.signalCount ?? 1)), + dayBucket: normalizeIsoDay(item.dayBucket ?? params.dayBucket ?? ""), + }; + }) + .filter((item): item is NonNullable => item !== null); + if (relevant.length === 0) { + return; + } + + const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now(); + const nowIso = new Date(nowMs).toISOString(); + const fallbackDayBucket = formatMemoryDreamingDay(nowMs, params.timezone); + await withShortTermLock(workspaceDir, async () => { + const store = await readStore(workspaceDir, nowIso); + + for (const item of relevant) { + const dayBucket = item.dayBucket ?? fallbackDayBucket; + const effectiveQuery = item.query || query; + if (!effectiveQuery) { + continue; + } + const queryHash = hashQuery(effectiveQuery); + const claimHash = buildClaimHash(item.snippet); + const key = buildEntryKey({ + path: item.path, + startLine: item.startLine, + endLine: item.endLine, + source: "memory", + claimHash, + }); + const existing = store.entries[key]; + const recallDaysBase = existing?.recallDays ?? []; + const queryHashesBase = existing?.queryHashes ?? []; + const dedupeSignal = + Boolean(params.dedupeByQueryPerDay) && + queryHashesBase.includes(queryHash) && + recallDaysBase.includes(dayBucket); + const groundedCount = Math.max( + 0, + Math.floor(existing?.groundedCount ?? 0) + (dedupeSignal ? 0 : item.signalCount), + ); + const totalScore = Math.max( + 0, + (existing?.totalScore ?? 0) + (dedupeSignal ? 0 : item.score * item.signalCount), + ); + const maxScore = Math.max(existing?.maxScore ?? 0, dedupeSignal ? 0 : item.score); + const queryHashes = mergeQueryHashes(existing?.queryHashes ?? [], queryHash); + const recallDays = mergeRecentDistinct(recallDaysBase, dayBucket, MAX_RECALL_DAYS); + const conceptTags = deriveConceptTags({ path: item.path, snippet: item.snippet }); + + store.entries[key] = { + key, + path: item.path, + startLine: item.startLine, + endLine: item.endLine, + source: "memory", + snippet: item.snippet, + recallCount: Math.max(0, Math.floor(existing?.recallCount ?? 0)), + dailyCount: Math.max(0, Math.floor(existing?.dailyCount ?? 0)), + groundedCount, + totalScore, + maxScore, + firstRecalledAt: existing?.firstRecalledAt ?? nowIso, + lastRecalledAt: nowIso, + queryHashes, + recallDays, + conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []), + claimHash, + ...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}), + }; + } + + store.updatedAt = nowIso; + await writeStore(workspaceDir, store); + }); +} + export async function recordDreamingPhaseSignals(params: { workspaceDir?: string; phase: "light" | "rem"; @@ -970,7 +1135,8 @@ export async function rankShortTermPromotionCandidates( } const recallCount = Math.max(0, Math.floor(entry.recallCount ?? 0)); const dailyCount = Math.max(0, Math.floor(entry.dailyCount ?? 0)); - const signalCount = recallCount + dailyCount; + const groundedCount = Math.max(0, Math.floor(entry.groundedCount ?? 0)); + const signalCount = totalSignalCountForEntry(entry); if (signalCount <= 0) { continue; } @@ -996,7 +1162,10 @@ export async function rankShortTermPromotionCandidates( const recency = clampScore(calculateRecencyComponent(ageDays, halfLifeDays)); const recallDays = entry.recallDays ?? []; const conceptTags = entry.conceptTags ?? []; - const consolidation = calculateConsolidationComponent(recallDays); + const consolidation = Math.max( + calculateConsolidationComponent(recallDays), + clampScore(groundedCount / 3), + ); const conceptual = calculateConceptualComponent(conceptTags); const phaseBoost = calculatePhaseSignalBoost(phaseSignals.entries[entry.key], nowMs); @@ -1022,10 +1191,12 @@ export async function rankShortTermPromotionCandidates( snippet: entry.snippet, recallCount, dailyCount, + groundedCount, signalCount, avgScore, maxScore: clampScore(entry.maxScore), uniqueQueries, + ...(entry.claimHash ? { claimHash: entry.claimHash } : {}), promotedAt: entry.promotedAt, firstRecalledAt: entry.firstRecalledAt, lastRecalledAt: entry.lastRecalledAt, @@ -1300,9 +1471,15 @@ export async function applyShortTermPromotions( if (candidate.score < minScore) { return false; } - const candidateSignalCount = + const candidateSignalCount = Math.max( + 0, candidate.signalCount ?? - Math.max(0, candidate.recallCount) + Math.max(0, candidate.dailyCount ?? 0); + totalSignalCountForEntry({ + recallCount: candidate.recallCount, + dailyCount: candidate.dailyCount, + groundedCount: candidate.groundedCount, + }), + ); if (candidateSignalCount < minRecallCount) { return false; } @@ -1606,6 +1783,10 @@ export async function repairShortTermPromotionArtifacts(params: { 0, Math.floor((entry as { dailyCount?: number }).dailyCount ?? 0), ), + groundedCount: Math.max( + 0, + Math.floor((entry as { groundedCount?: number }).groundedCount ?? 0), + ), queryHashes: (entry.queryHashes ?? []).slice(-MAX_QUERY_HASHES), recallDays: mergeRecentDistinct(entry.recallDays ?? [], fallbackDay, MAX_RECALL_DAYS), conceptTags: conceptTags.length > 0 ? conceptTags : (entry.conceptTags ?? []), @@ -1641,6 +1822,50 @@ export async function repairShortTermPromotionArtifacts(params: { }; } +export async function removeGroundedShortTermCandidates(params: { + workspaceDir: string; +}): Promise<{ removed: number; storePath: string }> { + const workspaceDir = params.workspaceDir.trim(); + const storePath = resolveStorePath(workspaceDir); + const nowIso = new Date().toISOString(); + let removed = 0; + + await withShortTermLock(workspaceDir, async () => { + const [store, phaseSignals] = await Promise.all([ + readStore(workspaceDir, nowIso), + readPhaseSignalStore(workspaceDir, nowIso), + ]); + + for (const [key, entry] of Object.entries(store.entries)) { + if ( + Math.max(0, Math.floor(entry.groundedCount ?? 0)) > 0 && + Math.max(0, Math.floor(entry.recallCount ?? 0)) === 0 && + Math.max(0, Math.floor(entry.dailyCount ?? 0)) === 0 + ) { + delete store.entries[key]; + removed += 1; + } + } + + for (const key of Object.keys(phaseSignals.entries)) { + if (!Object.hasOwn(store.entries, key)) { + delete phaseSignals.entries[key]; + } + } + + if (removed > 0) { + store.updatedAt = nowIso; + phaseSignals.updatedAt = nowIso; + await Promise.all([ + writeStore(workspaceDir, store), + writePhaseSignalStore(workspaceDir, phaseSignals), + ]); + } + }); + + return { removed, storePath }; +} + export const __testing = { parseLockOwnerPid, canStealStaleLock, @@ -1648,4 +1873,6 @@ export const __testing = { deriveConceptTags, calculateConsolidationComponent, calculatePhaseSignalBoost, + buildClaimHash, + totalSignalCountForEntry, }; diff --git a/src/gateway/server-methods/doctor.ts b/src/gateway/server-methods/doctor.ts index 5965bf353d4..d6b9d24d27a 100644 --- a/src/gateway/server-methods/doctor.ts +++ b/src/gateway/server-methods/doctor.ts @@ -64,6 +64,7 @@ type DoctorMemoryDreamingEntryPayload = { snippet: string; recallCount: number; dailyCount: number; + groundedCount: number; totalSignalCount: number; lightHits: number; remHits: number; @@ -81,6 +82,7 @@ type DoctorMemoryDreamingPayload = { shortTermCount: number; recallSignalCount: number; dailySignalCount: number; + groundedSignalCount: number; totalSignalCount: number; phaseSignalCount: number; lightPhaseHitCount: number; @@ -166,6 +168,7 @@ function resolveDreamingConfig( | "shortTermCount" | "recallSignalCount" | "dailySignalCount" + | "groundedSignalCount" | "totalSignalCount" | "phaseSignalCount" | "lightPhaseHitCount" @@ -258,6 +261,7 @@ type DreamingStoreStats = Pick< | "shortTermCount" | "recallSignalCount" | "dailySignalCount" + | "groundedSignalCount" | "totalSignalCount" | "phaseSignalCount" | "lightPhaseHitCount" @@ -370,6 +374,7 @@ async function loadDreamingStoreStats( let shortTermCount = 0; let recallSignalCount = 0; let dailySignalCount = 0; + let groundedSignalCount = 0; let totalSignalCount = 0; let phaseSignalCount = 0; let lightPhaseHitCount = 0; @@ -396,7 +401,8 @@ async function loadDreamingStoreStats( const range = parseEntryRangeFromKey(entryKey, entry.startLine, entry.endLine); const recallCount = toNonNegativeInt(entry.recallCount); const dailyCount = toNonNegativeInt(entry.dailyCount); - const totalEntrySignalCount = recallCount + dailyCount; + const groundedCount = toNonNegativeInt(entry.groundedCount); + const totalEntrySignalCount = recallCount + dailyCount + groundedCount; const snippet = normalizeTrimmedString(entry.snippet) ?? normalizeTrimmedString(entry.summary) ?? @@ -410,6 +416,7 @@ async function loadDreamingStoreStats( snippet, recallCount, dailyCount, + groundedCount, totalSignalCount: totalEntrySignalCount, lightHits: 0, remHits: 0, @@ -422,6 +429,7 @@ async function loadDreamingStoreStats( activeKeys.add(entryKey); recallSignalCount += recallCount; dailySignalCount += dailyCount; + groundedSignalCount += groundedCount; totalSignalCount += totalEntrySignalCount; shortTermEntries.push(detail); activeEntries.set(entryKey, detail); @@ -476,6 +484,7 @@ async function loadDreamingStoreStats( shortTermCount, recallSignalCount, dailySignalCount, + groundedSignalCount, totalSignalCount, phaseSignalCount, lightPhaseHitCount, diff --git a/ui/src/ui/controllers/dreaming.test.ts b/ui/src/ui/controllers/dreaming.test.ts index d28c0e997f8..fc59648e05f 100644 --- a/ui/src/ui/controllers/dreaming.test.ts +++ b/ui/src/ui/controllers/dreaming.test.ts @@ -52,6 +52,7 @@ describe("dreaming controller", () => { shortTermCount: 8, recallSignalCount: 14, dailySignalCount: 6, + groundedSignalCount: 5, totalSignalCount: 20, phaseSignalCount: 11, lightPhaseHitCount: 7, @@ -67,6 +68,7 @@ describe("dreaming controller", () => { snippet: "Emma prefers shorter, lower-pressure check-ins.", recallCount: 2, dailyCount: 1, + groundedCount: 1, totalSignalCount: 3, lightHits: 1, remHits: 2, @@ -83,6 +85,7 @@ describe("dreaming controller", () => { snippet: "Emma prefers shorter, lower-pressure check-ins.", recallCount: 2, dailyCount: 1, + groundedCount: 1, totalSignalCount: 3, lightHits: 1, remHits: 2, @@ -98,6 +101,7 @@ describe("dreaming controller", () => { snippet: "Use the Happy Together calendar for flights.", recallCount: 3, dailyCount: 2, + groundedCount: 0, totalSignalCount: 5, lightHits: 0, remHits: 0, @@ -146,6 +150,7 @@ describe("dreaming controller", () => { expect.objectContaining({ enabled: true, shortTermCount: 8, + groundedSignalCount: 5, totalSignalCount: 20, phaseSignalCount: 11, promotedToday: 2, @@ -153,6 +158,7 @@ describe("dreaming controller", () => { expect.objectContaining({ snippet: "Emma prefers shorter, lower-pressure check-ins.", totalSignalCount: 3, + groundedCount: 1, phaseHitCount: 3, }), ], diff --git a/ui/src/ui/controllers/dreaming.ts b/ui/src/ui/controllers/dreaming.ts index c40bfd3f834..f80aa3f48e8 100644 --- a/ui/src/ui/controllers/dreaming.ts +++ b/ui/src/ui/controllers/dreaming.ts @@ -40,6 +40,7 @@ export type DreamingEntry = { snippet: string; recallCount: number; dailyCount: number; + groundedCount: number; totalSignalCount: number; lightHits: number; remHits: number; @@ -57,6 +58,7 @@ export type DreamingStatus = { shortTermCount: number; recallSignalCount: number; dailySignalCount: number; + groundedSignalCount: number; totalSignalCount: number; phaseSignalCount: number; lightPhaseHitCount: number; @@ -211,6 +213,7 @@ function normalizeDreamingEntry(raw: unknown): DreamingEntry | null { snippet, recallCount: normalizeFiniteInt(record?.recallCount, 0), dailyCount: normalizeFiniteInt(record?.dailyCount, 0), + groundedCount: normalizeFiniteInt(record?.groundedCount, 0), totalSignalCount: normalizeFiniteInt(record?.totalSignalCount, 0), lightHits: normalizeFiniteInt(record?.lightHits, 0), remHits: normalizeFiniteInt(record?.remHits, 0), @@ -252,6 +255,7 @@ function normalizeDreamingStatus(raw: unknown): DreamingStatus | null { shortTermCount: normalizeFiniteInt(record.shortTermCount, 0), recallSignalCount: normalizeFiniteInt(record.recallSignalCount, 0), dailySignalCount: normalizeFiniteInt(record.dailySignalCount, 0), + groundedSignalCount: normalizeFiniteInt(record.groundedSignalCount, 0), totalSignalCount: normalizeFiniteInt(record.totalSignalCount, 0), phaseSignalCount: normalizeFiniteInt(record.phaseSignalCount, 0), lightPhaseHitCount: normalizeFiniteInt(record.lightPhaseHitCount, 0), diff --git a/ui/src/ui/views/dreaming.test.ts b/ui/src/ui/views/dreaming.test.ts index be29b727405..1afa5d179c2 100644 --- a/ui/src/ui/views/dreaming.test.ts +++ b/ui/src/ui/views/dreaming.test.ts @@ -20,6 +20,7 @@ function buildProps(overrides?: Partial): DreamingProps { snippet: "Emma prefers shorter, lower-pressure check-ins.", recallCount: 2, dailyCount: 1, + groundedCount: 1, totalSignalCount: 3, lightHits: 1, remHits: 1, @@ -35,6 +36,7 @@ function buildProps(overrides?: Partial): DreamingProps { snippet: "Emma prefers shorter, lower-pressure check-ins.", recallCount: 2, dailyCount: 1, + groundedCount: 1, totalSignalCount: 3, lightHits: 1, remHits: 1, @@ -50,6 +52,7 @@ function buildProps(overrides?: Partial): DreamingProps { snippet: "Use the Happy Together calendar for flights.", recallCount: 3, dailyCount: 2, + groundedCount: 0, totalSignalCount: 5, lightHits: 0, remHits: 0, diff --git a/ui/src/ui/views/dreaming.ts b/ui/src/ui/views/dreaming.ts index 726a36bc835..7bfadf4c1f5 100644 --- a/ui/src/ui/views/dreaming.ts +++ b/ui/src/ui/views/dreaming.ts @@ -269,6 +269,7 @@ export type DreamingProps = { snippet: string; recallCount: number; dailyCount: number; + groundedCount: number; totalSignalCount: number; lightHits: number; remHits: number; @@ -284,6 +285,7 @@ export type DreamingProps = { snippet: string; recallCount: number; dailyCount: number; + groundedCount: number; totalSignalCount: number; lightHits: number; remHits: number; @@ -299,6 +301,7 @@ export type DreamingProps = { snippet: string; recallCount: number; dailyCount: number; + groundedCount: number; totalSignalCount: number; lightHits: number; remHits: number; @@ -580,6 +583,7 @@ function renderScene(props: DreamingProps, idle: boolean, dreamText: string) { ? `${entry.recallCount} recall${entry.recallCount === 1 ? "" : "s"}` : null, entry.dailyCount > 0 ? `${entry.dailyCount} daily` : null, + entry.groundedCount > 0 ? `${entry.groundedCount} grounded` : null, entry.phaseHitCount > 0 ? `${entry.phaseHitCount} phase hit${entry.phaseHitCount === 1 ? "" : "s"}` : null,