From 8e83e5221371a5d0aaa4693955243b52343196a6 Mon Sep 17 00:00:00 2001 From: Ted Li Date: Sat, 25 Apr 2026 14:10:38 -0700 Subject: [PATCH] fix(memory-core): skip stale dreaming recall sources (#71695) * fix(memory-core): skip stale dreaming recall sources * fix(memory-core): parallelize live recall filtering --- .../memory-core/src/dreaming-phases.test.ts | 96 ++++++++++++++++++- extensions/memory-core/src/dreaming-phases.ts | 19 +++- .../memory-core/src/short-term-promotion.ts | 37 +++++++ 3 files changed, 145 insertions(+), 7 deletions(-) diff --git a/extensions/memory-core/src/dreaming-phases.test.ts b/extensions/memory-core/src/dreaming-phases.test.ts index 8e70e176035..e0f992ac5d0 100644 --- a/extensions/memory-core/src/dreaming-phases.test.ts +++ b/extensions/memory-core/src/dreaming-phases.test.ts @@ -2118,6 +2118,11 @@ describe("memory-core dreaming phases", () => { it("records light/rem signals that reinforce deep promotion ranking", async () => { const workspaceDir = await createDreamingWorkspace(); const nowMs = Date.parse("2026-04-05T10:00:00.000Z"); + await fs.writeFile( + path.join(workspaceDir, "memory", "2026-04-03.md"), + "Move backups to S3 Glacier.\n", + "utf-8", + ); await recordShortTermRecalls({ workspaceDir, query: "glacier backup", @@ -2126,7 +2131,7 @@ describe("memory-core dreaming phases", () => { { path: "memory/2026-04-03.md", startLine: 1, - endLine: 2, + endLine: 1, score: 0.92, snippet: "Move backups to S3 Glacier.", source: "memory", @@ -2141,7 +2146,7 @@ describe("memory-core dreaming phases", () => { { path: "memory/2026-04-03.md", startLine: 1, - endLine: 2, + endLine: 1, score: 0.9, snippet: "Move backups to S3 Glacier.", source: "memory", @@ -2221,6 +2226,93 @@ describe("memory-core dreaming phases", () => { }); }); + it("skips REM short-term candidates whose source file disappeared", async () => { + const workspaceDir = await createDreamingWorkspace(); + await fs.writeFile( + path.join(workspaceDir, "memory", "2026-04-03.md"), + "Move backups to S3 Glacier.\n", + "utf-8", + ); + const nowMs = DREAMING_TEST_BASE_TIME.getTime(); + await recordShortTermRecalls({ + workspaceDir, + query: "live backup", + nowMs, + results: [ + { + path: "memory/2026-04-03.md", + startLine: 1, + endLine: 1, + score: 0.91, + snippet: "Move backups to S3 Glacier.", + source: "memory", + }, + ], + }); + await recordShortTermRecalls({ + workspaceDir, + query: "stale provider setup", + nowMs, + results: [ + { + path: "memory/.dreams/session-corpus/2026-04-16.txt", + startLine: 2, + endLine: 2, + score: 0.88, + snippet: "Assistant: Documented Ollama provider setup.", + source: "memory", + }, + ], + }); + const baseline = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + nowMs, + }); + const liveKey = baseline.find((candidate) => candidate.path === "memory/2026-04-03.md")?.key; + const staleKey = baseline.find((candidate) => + candidate.path.includes("session-corpus/2026-04-16.txt"), + )?.key; + expect(liveKey).toBeDefined(); + expect(staleKey).toBeDefined(); + + await withDreamingTestClock(async () => { + setDreamingTestTime(); + await __testing.runPhaseIfTriggered({ + cleanedBody: __testing.constants.REM_SLEEP_EVENT_TEXT, + trigger: "heartbeat", + workspaceDir, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + phase: "rem", + eventText: __testing.constants.REM_SLEEP_EVENT_TEXT, + config: { + enabled: true, + lookbackDays: 7, + limit: 10, + minPatternStrength: 0, + timezone: "UTC", + storage: { mode: "inline", separateReports: false }, + }, + }); + }); + + const phaseSignalPath = resolveShortTermPhaseSignalStorePath(workspaceDir); + const phaseSignalStore = JSON.parse(await fs.readFile(phaseSignalPath, "utf-8")) as { + entries: Record; + }; + expect(phaseSignalStore.entries[liveKey!]).toMatchObject({ remHits: 1 }); + expect(phaseSignalStore.entries[staleKey!]).toBeUndefined(); + + const remOutput = await fs.readFile( + path.join(workspaceDir, "memory", `${DREAMING_TEST_DAY}.md`), + "utf-8", + ); + expect(remOutput).toContain("Move backups to S3 Glacier."); + expect(remOutput).not.toContain("Documented Ollama provider setup"); + }); + it("passes staged light-dreaming snippets into the narrative pipeline", async () => { const workspaceDir = await createDreamingWorkspace(); const subagent = createMockNarrativeSubagent("The backup plan glowed like cold storage."); diff --git a/extensions/memory-core/src/dreaming-phases.ts b/extensions/memory-core/src/dreaming-phases.ts index 099e7a12372..cbbe7d92414 100644 --- a/extensions/memory-core/src/dreaming-phases.ts +++ b/extensions/memory-core/src/dreaming-phases.ts @@ -26,6 +26,7 @@ import { } from "./dreaming-narrative.js"; import { asRecord, formatErrorMessage, normalizeTrimmedString } from "./dreaming-shared.js"; import { + filterLiveShortTermRecallEntries, readShortTermRecallEntries, recordDreamingPhaseSignals, recordShortTermRecalls, @@ -1520,9 +1521,14 @@ async function runLightDreaming(params: { nowMs, timezone: params.config.timezone, }); + const recentEntries = await filterLiveShortTermRecallEntries({ + workspaceDir: params.workspaceDir, + entries: ( + await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }) + ).filter((entry) => entryWithinLookback(entry, cutoffMs)), + }); const entries = dedupeEntries( - (await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs })) - .filter((entry) => entryWithinLookback(entry, cutoffMs)) + recentEntries .toSorted((a, b) => { const byTime = Date.parse(b.lastRecalledAt) - Date.parse(a.lastRecalledAt); if (byTime !== 0) { @@ -1611,9 +1617,12 @@ async function runRemDreaming(params: { nowMs, timezone: params.config.timezone, }); - const entries = ( - await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }) - ).filter((entry) => entryWithinLookback(entry, cutoffMs)); + const entries = await filterLiveShortTermRecallEntries({ + workspaceDir: params.workspaceDir, + entries: ( + await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }) + ).filter((entry) => entryWithinLookback(entry, cutoffMs)), + }); const preview = previewRemDreaming({ entries, limit: params.config.limit, diff --git a/extensions/memory-core/src/short-term-promotion.ts b/extensions/memory-core/src/short-term-promotion.ts index b20f21bab3d..e0d9c303598 100644 --- a/extensions/memory-core/src/short-term-promotion.ts +++ b/extensions/memory-core/src/short-term-promotion.ts @@ -874,6 +874,43 @@ export function isShortTermMemoryPath(filePath: string): boolean { return SHORT_TERM_BASENAME_RE.test(normalized); } +async function shortTermRecallSourceExists(params: { + workspaceDir: string; + entry: Pick; +}): Promise { + const workspaceDir = params.workspaceDir.trim(); + if (!workspaceDir) { + return false; + } + for (const sourcePath of resolveShortTermSourcePathCandidates(workspaceDir, params.entry.path)) { + try { + const stat = await fs.stat(sourcePath); + if (stat.isFile()) { + return true; + } + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + continue; + } + throw err; + } + } + return false; +} + +export async function filterLiveShortTermRecallEntries(params: { + workspaceDir: string; + entries: ShortTermRecallEntry[]; +}): Promise { + const results = await Promise.all( + params.entries.map(async (entry) => ({ + entry, + exists: await shortTermRecallSourceExists({ workspaceDir: params.workspaceDir, entry }), + })), + ); + return results.filter((result) => result.exists).map((result) => result.entry); +} + export async function recordShortTermRecalls(params: { workspaceDir?: string; query: string;