fix(memory-core): skip stale dreaming recall sources (#71695)

* fix(memory-core): skip stale dreaming recall sources

* fix(memory-core): parallelize live recall filtering

extensions/memory-core/src/dreaming-phases.test.ts

@@ -2118,6 +2118,11 @@ describe("memory-core dreaming phases", () => {
   it("records light/rem signals that reinforce deep promotion ranking", async () => {
     const workspaceDir = await createDreamingWorkspace();
     const nowMs = Date.parse("2026-04-05T10:00:00.000Z");
+    await fs.writeFile(
+      path.join(workspaceDir, "memory", "2026-04-03.md"),
+      "Move backups to S3 Glacier.\n",
+      "utf-8",
+    );
     await recordShortTermRecalls({
       workspaceDir,
       query: "glacier backup",
@@ -2126,7 +2131,7 @@ describe("memory-core dreaming phases", () => {
         {
           path: "memory/2026-04-03.md",
           startLine: 1,
-          endLine: 2,
+          endLine: 1,
           score: 0.92,
           snippet: "Move backups to S3 Glacier.",
           source: "memory",
@@ -2141,7 +2146,7 @@ describe("memory-core dreaming phases", () => {
         {
           path: "memory/2026-04-03.md",
           startLine: 1,
-          endLine: 2,
+          endLine: 1,
           score: 0.9,
           snippet: "Move backups to S3 Glacier.",
           source: "memory",
@@ -2221,6 +2226,93 @@ describe("memory-core dreaming phases", () => {
     });
   });
 
+  it("skips REM short-term candidates whose source file disappeared", async () => {
+    const workspaceDir = await createDreamingWorkspace();
+    await fs.writeFile(
+      path.join(workspaceDir, "memory", "2026-04-03.md"),
+      "Move backups to S3 Glacier.\n",
+      "utf-8",
+    );
+    const nowMs = DREAMING_TEST_BASE_TIME.getTime();
+    await recordShortTermRecalls({
+      workspaceDir,
+      query: "live backup",
+      nowMs,
+      results: [
+        {
+          path: "memory/2026-04-03.md",
+          startLine: 1,
+          endLine: 1,
+          score: 0.91,
+          snippet: "Move backups to S3 Glacier.",
+          source: "memory",
+        },
+      ],
+    });
+    await recordShortTermRecalls({
+      workspaceDir,
+      query: "stale provider setup",
+      nowMs,
+      results: [
+        {
+          path: "memory/.dreams/session-corpus/2026-04-16.txt",
+          startLine: 2,
+          endLine: 2,
+          score: 0.88,
+          snippet: "Assistant: Documented Ollama provider setup.",
+          source: "memory",
+        },
+      ],
+    });
+    const baseline = await rankShortTermPromotionCandidates({
+      workspaceDir,
+      minScore: 0,
+      minRecallCount: 0,
+      minUniqueQueries: 0,
+      nowMs,
+    });
+    const liveKey = baseline.find((candidate) => candidate.path === "memory/2026-04-03.md")?.key;
+    const staleKey = baseline.find((candidate) =>
+      candidate.path.includes("session-corpus/2026-04-16.txt"),
+    )?.key;
+    expect(liveKey).toBeDefined();
+    expect(staleKey).toBeDefined();
+
+    await withDreamingTestClock(async () => {
+      setDreamingTestTime();
+      await __testing.runPhaseIfTriggered({
+        cleanedBody: __testing.constants.REM_SLEEP_EVENT_TEXT,
+        trigger: "heartbeat",
+        workspaceDir,
+        logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+        phase: "rem",
+        eventText: __testing.constants.REM_SLEEP_EVENT_TEXT,
+        config: {
+          enabled: true,
+          lookbackDays: 7,
+          limit: 10,
+          minPatternStrength: 0,
+          timezone: "UTC",
+          storage: { mode: "inline", separateReports: false },
+        },
+      });
+    });
+
+    const phaseSignalPath = resolveShortTermPhaseSignalStorePath(workspaceDir);
+    const phaseSignalStore = JSON.parse(await fs.readFile(phaseSignalPath, "utf-8")) as {
+      entries: Record<string, { remHits: number }>;
+    };
+    expect(phaseSignalStore.entries[liveKey!]).toMatchObject({ remHits: 1 });
+    expect(phaseSignalStore.entries[staleKey!]).toBeUndefined();
+
+    const remOutput = await fs.readFile(
+      path.join(workspaceDir, "memory", `${DREAMING_TEST_DAY}.md`),
+      "utf-8",
+    );
+    expect(remOutput).toContain("Move backups to S3 Glacier.");
+    expect(remOutput).not.toContain("Documented Ollama provider setup");
+  });
+
   it("passes staged light-dreaming snippets into the narrative pipeline", async () => {
     const workspaceDir = await createDreamingWorkspace();
     const subagent = createMockNarrativeSubagent("The backup plan glowed like cold storage.");

extensions/memory-core/src/dreaming-phases.ts

@@ -26,6 +26,7 @@ import {
 } from "./dreaming-narrative.js";
 import { asRecord, formatErrorMessage, normalizeTrimmedString } from "./dreaming-shared.js";
 import {
+  filterLiveShortTermRecallEntries,
   readShortTermRecallEntries,
   recordDreamingPhaseSignals,
   recordShortTermRecalls,
@@ -1520,9 +1521,14 @@ async function runLightDreaming(params: {
     nowMs,
     timezone: params.config.timezone,
   });
+  const recentEntries = await filterLiveShortTermRecallEntries({
+    workspaceDir: params.workspaceDir,
+    entries: (
+      await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs })
+    ).filter((entry) => entryWithinLookback(entry, cutoffMs)),
+  });
   const entries = dedupeEntries(
-    (await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }))
-      .filter((entry) => entryWithinLookback(entry, cutoffMs))
+    recentEntries
       .toSorted((a, b) => {
         const byTime = Date.parse(b.lastRecalledAt) - Date.parse(a.lastRecalledAt);
         if (byTime !== 0) {
@@ -1611,9 +1617,12 @@ async function runRemDreaming(params: {
     nowMs,
     timezone: params.config.timezone,
   });
-  const entries = (
-    await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs })
-  ).filter((entry) => entryWithinLookback(entry, cutoffMs));
+  const entries = await filterLiveShortTermRecallEntries({
+    workspaceDir: params.workspaceDir,
+    entries: (
+      await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs })
+    ).filter((entry) => entryWithinLookback(entry, cutoffMs)),
+  });
   const preview = previewRemDreaming({
     entries,
     limit: params.config.limit,

extensions/memory-core/src/short-term-promotion.ts

@@ -874,6 +874,43 @@ export function isShortTermMemoryPath(filePath: string): boolean {
   return SHORT_TERM_BASENAME_RE.test(normalized);
 }
 
+async function shortTermRecallSourceExists(params: {
+  workspaceDir: string;
+  entry: Pick<ShortTermRecallEntry, "path">;
+}): Promise<boolean> {
+  const workspaceDir = params.workspaceDir.trim();
+  if (!workspaceDir) {
+    return false;
+  }
+  for (const sourcePath of resolveShortTermSourcePathCandidates(workspaceDir, params.entry.path)) {
+    try {
+      const stat = await fs.stat(sourcePath);
+      if (stat.isFile()) {
+        return true;
+      }
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+        continue;
+      }
+      throw err;
+    }
+  }
+  return false;
+}
+
+export async function filterLiveShortTermRecallEntries(params: {
+  workspaceDir: string;
+  entries: ShortTermRecallEntry[];
+}): Promise<ShortTermRecallEntry[]> {
+  const results = await Promise.all(
+    params.entries.map(async (entry) => ({
+      entry,
+      exists: await shortTermRecallSourceExists({ workspaceDir: params.workspaceDir, entry }),
+    })),
+  );
+  return results.filter((result) => result.exists).map((result) => result.entry);
+}
+
 export async function recordShortTermRecalls(params: {
   workspaceDir?: string;
   query: string;