memory: block dreaming self-ingestion

2026-05-06 23:50:42 +00:00 · 2026-04-14 18:43:29 -04:00
parent 5702ab695b
commit 02cbfeaa86
5 changed files with 244 additions and 3 deletions
--- a/docs/concepts/dreaming.md
+++ b/docs/concepts/dreaming.md
@@ -80,6 +80,9 @@ After each phase has enough material, `memory-core` runs a best-effort backgroun
 subagent turn (using the default runtime model) and appends a short diary entry.

 This diary is for human reading in the Dreams UI, not a promotion source.
+Dreaming-generated diary/report artifacts are excluded from short-term
+promotion. Only grounded memory snippets are eligible to promote into
+`MEMORY.md`.

 There is also a grounded historical backfill lane for review and recovery work:

--- a/extensions/memory-core/src/short-term-promotion.test.ts
+++ b/extensions/memory-core/src/short-term-promotion.test.ts
@@ -203,6 +203,33 @@ describe("short-term promotion", () => {
    });
  });

+  it("ignores contaminated dreaming snippets when recording short-term recalls", async () => {
+    await withTempWorkspace(async (workspaceDir) => {
+      await recordShortTermRecalls({
+        workspaceDir,
+        query: "action preference",
+        results: [
+          {
+            path: "memory/2026-04-03.md",
+            source: "memory",
+            startLine: 1,
+            endLine: 1,
+            score: 0.92,
+            snippet:
+              "Candidate: Default to action. confidence: 0.76 evidence: memory/.dreams/session-corpus/2026-04-08.txt:1-1 recalls: 3 status: staged",
+          },
+        ],
+      });
+
+      expect(
+        JSON.parse(await fs.readFile(resolveShortTermRecallStorePath(workspaceDir), "utf-8")),
+      ).toMatchObject({
+        version: 1,
+        entries: {},
+      });
+    });
+  });
+
  it("records recalls and ranks candidates with weighted scores", async () => {
    await withTempWorkspace(async (workspaceDir) => {
      await recordShortTermRecalls({
@@ -940,6 +967,54 @@ describe("short-term promotion", () => {
    });
  });

+  it("does not rank contaminated dreaming snippets from an existing short-term store", async () => {
+    await withTempWorkspace(async (workspaceDir) => {
+      const storePath = resolveShortTermRecallStorePath(workspaceDir);
+      await fs.writeFile(
+        storePath,
+        JSON.stringify(
+          {
+            version: 1,
+            updatedAt: "2026-04-04T00:00:00.000Z",
+            entries: {
+              contaminated: {
+                key: "contaminated",
+                path: "memory/2026-04-03.md",
+                startLine: 1,
+                endLine: 1,
+                source: "memory",
+                snippet:
+                  "Reflections: Theme: assistant. confidence: 1.00 evidence: memory/.dreams/session-corpus/2026-04-08.txt:2-2 recalls: 4 status: staged",
+                recallCount: 4,
+                dailyCount: 0,
+                groundedCount: 0,
+                totalScore: 3.6,
+                maxScore: 0.95,
+                firstRecalledAt: "2026-04-03T00:00:00.000Z",
+                lastRecalledAt: "2026-04-04T00:00:00.000Z",
+                queryHashes: ["a", "b"],
+                recallDays: ["2026-04-03", "2026-04-04"],
+                conceptTags: ["assistant"],
+              },
+            },
+          },
+          null,
+          2,
+        ),
+        "utf-8",
+      );
+
+      const ranked = await rankShortTermPromotionCandidates({
+        workspaceDir,
+        minScore: 0,
+        minRecallCount: 0,
+        minUniqueQueries: 0,
+      });
+
+      expect(ranked).toEqual([]);
+    });
+  });
+
  it("skips direct candidates that exceed maxAgeDays during apply", async () => {
    await withTempWorkspace(async (workspaceDir) => {
      const applied = await applyShortTermPromotions({
@@ -987,6 +1062,53 @@ describe("short-term promotion", () => {
    });
  });

+  it("does not append contaminated dreaming snippets during direct apply", async () => {
+    await withTempWorkspace(async (workspaceDir) => {
+      const applied = await applyShortTermPromotions({
+        workspaceDir,
+        minScore: 0,
+        minRecallCount: 0,
+        minUniqueQueries: 0,
+        candidates: [
+          {
+            key: "memory:memory/2026-04-03.md:1:1",
+            path: "memory/2026-04-03.md",
+            startLine: 1,
+            endLine: 1,
+            source: "memory",
+            snippet:
+              "Candidate: Default to action. confidence: 0.76 evidence: memory/.dreams/session-corpus/2026-04-08.txt:1-1 recalls: 3 status: staged",
+            recallCount: 4,
+            avgScore: 0.97,
+            maxScore: 0.97,
+            uniqueQueries: 2,
+            firstRecalledAt: "2026-04-03T00:00:00.000Z",
+            lastRecalledAt: "2026-04-04T00:00:00.000Z",
+            ageDays: 0,
+            score: 0.99,
+            recallDays: ["2026-04-03", "2026-04-04"],
+            conceptTags: ["assistant"],
+            components: {
+              frequency: 1,
+              relevance: 1,
+              diversity: 1,
+              recency: 1,
+              consolidation: 1,
+              conceptual: 1,
+            },
+          },
+        ],
+      });
+
+      expect(applied.applied).toBe(0);
+      await expect(
+        fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"),
+      ).rejects.toMatchObject({
+        code: "ENOENT",
+      });
+    });
+  });
+
  it("applies promotion candidates to MEMORY.md and marks them promoted", async () => {
    await withTempWorkspace(async (workspaceDir) => {
      await writeDailyMemoryNote(workspaceDir, "2026-04-01", [
--- a/extensions/memory-core/src/short-term-promotion.ts
+++ b/extensions/memory-core/src/short-term-promotion.ts
@@ -32,6 +32,8 @@ const SHORT_TERM_LOCK_RELATIVE_PATH = path.join("memory", ".dreams", "short-term
 const SHORT_TERM_LOCK_WAIT_TIMEOUT_MS = 10_000;
 const SHORT_TERM_LOCK_STALE_MS = 60_000;
 const SHORT_TERM_LOCK_RETRY_DELAY_MS = 40;
+const DREAMING_NARRATIVE_PROMPT_PREFIX = "Write a dream diary entry from these memory fragments";
+const DREAMING_PROMOTION_META_PREFIX = "openclaw-memory-promotion:";
 // Repeated dreaming revisits should be able to clear the default promotion gate
 // without requiring separate organic recall traffic for the same snippet.
 const PHASE_SIGNAL_LIGHT_BOOST_MAX = 0.06;
@@ -235,6 +237,37 @@ function normalizeSnippet(raw: string): string {
  return trimmed.replace(/\s+/g, " ");
 }

+function isContaminatedDreamingSnippet(raw: string): boolean {
+  const snippet = normalizeSnippet(raw);
+  if (!snippet) {
+    return false;
+  }
+  if (
+    snippet.includes(DREAMING_NARRATIVE_PROMPT_PREFIX) ||
+    snippet.includes(DREAMING_PROMOTION_META_PREFIX) ||
+    snippet.includes("dreaming-narrative-")
+  ) {
+    return true;
+  }
+
+  const hasCandidateLead = /^Candidate:/i.test(snippet) || /[([]\s*Candidate:/i.test(snippet);
+  const hasReflectionLead =
+    /^Reflections?:/i.test(snippet) || /[([]\s*Reflections?:/i.test(snippet);
+  const hasConfidence = /\bconfidence:\s*\d/i.test(snippet);
+  const hasEvidence = /\bevidence:\s*(?:memory\/\.dreams\/session-corpus\/|memory\/)/i.test(
+    snippet,
+  );
+  const hasStatus = /\bstatus:\s*staged\b/i.test(snippet);
+  const hasRecalls = /\brecalls:\s*\d+\b/i.test(snippet);
+  if (hasEvidence && (hasCandidateLead || hasReflectionLead)) {
+    return true;
+  }
+  const structuredMarkers = [hasConfidence, hasEvidence, hasStatus, hasRecalls].filter(
+    Boolean,
+  ).length;
+  return (hasCandidateLead || hasReflectionLead) && structuredMarkers >= 2;
+}
+
 function normalizeMemoryPath(rawPath: string): string {
  return rawPath.replaceAll("\\", "/").replace(/^\.\//, "");
 }
@@ -409,6 +442,9 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
          ? entry.claimHash.trim()
          : undefined;
      const snippet = typeof entry.snippet === "string" ? normalizeSnippet(entry.snippet) : "";
+      if (snippet && isContaminatedDreamingSnippet(snippet)) {
+        continue;
+      }
      const queryHashes = Array.isArray(entry.queryHashes)
        ? normalizeDistinctStrings(entry.queryHashes, MAX_QUERY_HASHES)
        : [];
@@ -849,6 +885,9 @@ export async function recordShortTermRecalls(params: {
    for (const result of relevant) {
      const normalizedPath = normalizeMemoryPath(result.path);
      const snippet = normalizeSnippet(result.snippet);
+      if (!snippet || isContaminatedDreamingSnippet(snippet)) {
+        continue;
+      }
      const claimHash = snippet ? buildClaimHash(snippet) : undefined;
      const groundedKey = claimHash
        ? buildEntryKey({
@@ -954,6 +993,7 @@ export async function recordGroundedShortTermCandidates(params: {
      const normalizedPath = normalizeMemoryPath(item.path);
      if (
        !snippet ||
+        isContaminatedDreamingSnippet(snippet) ||
        !normalizedPath ||
        !isShortTermMemoryPath(normalizedPath) ||
        !Number.isFinite(item.startLine) ||
@@ -1136,6 +1176,9 @@ export async function rankShortTermPromotionCandidates(
    if (!entry || entry.source !== "memory" || !isShortTermMemoryPath(entry.path)) {
      continue;
    }
+    if (isContaminatedDreamingSnippet(entry.snippet)) {
+      continue;
+    }
    if (!includePromoted && entry.promotedAt) {
      continue;
    }
@@ -1471,6 +1514,9 @@ export async function applyShortTermPromotions(
    const store = await readStore(workspaceDir, nowIso);
    const selected = options.candidates
      .filter((candidate) => {
+        if (isContaminatedDreamingSnippet(candidate.snippet)) {
+          return false;
+        }
        if (candidate.promotedAt) {
          return false;
        }
@@ -1506,7 +1552,7 @@ export async function applyShortTermPromotions(
    const rehydratedSelected: PromotionCandidate[] = [];
    for (const candidate of selected) {
      const rehydrated = await rehydratePromotionCandidate(workspaceDir, candidate);
-      if (rehydrated) {
+      if (rehydrated && !isContaminatedDreamingSnippet(rehydrated.snippet)) {
        rehydratedSelected.push(rehydrated);
      }
    }
@@ -1881,4 +1927,5 @@ export const __testing = {
  calculatePhaseSignalBoost,
  buildClaimHash,
  totalSignalCountForEntry,
+  isContaminatedDreamingSnippet,
 };
--- a/src/memory-host-sdk/host/session-files.test.ts
+++ b/src/memory-host-sdk/host/session-files.test.ts
@@ -174,4 +174,30 @@ describe("buildSessionEntry", () => {
    expect(entry).not.toBeNull();
    expect(entry?.generatedByDreamingNarrative).toBe(true);
  });
+
+  it("flags dreaming narrative transcripts from the dream-diary prompt body", async () => {
+    const jsonlLines = [
+      JSON.stringify({
+        type: "message",
+        message: {
+          role: "user",
+          content:
+            "Write a dream diary entry from these memory fragments:\n- Candidate: durable note",
+        },
+      }),
+      JSON.stringify({
+        type: "message",
+        message: { role: "assistant", content: "A drifting archive breathed in moonlight." },
+      }),
+    ];
+    const filePath = path.join(tmpDir, "dreaming-prompt-session.jsonl");
+    await fs.writeFile(filePath, jsonlLines.join("\n"));
+
+    const entry = await buildSessionEntry(filePath);
+
+    expect(entry).not.toBeNull();
+    expect(entry?.generatedByDreamingNarrative).toBe(true);
+    expect(entry?.content).toBe("");
+    expect(entry?.lineMap).toEqual([]);
+  });
 });
--- a/src/memory-host-sdk/host/session-files.ts
+++ b/src/memory-host-sdk/host/session-files.ts
@@ -7,6 +7,8 @@ import { createSubsystemLogger } from "../../logging/subsystem.js";
 import { hashText } from "./internal.js";

 const log = createSubsystemLogger("memory");
+const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-";
+const DREAMING_NARRATIVE_PROMPT_PREFIX = "Write a dream diary entry from these memory fragments";

 export type SessionFileEntry = {
  path: string;
@@ -42,7 +44,42 @@ function isDreamingNarrativeBootstrapRecord(record: unknown): boolean {
    return false;
  }
  const runId = (candidate.data as { runId?: unknown }).runId;
-  return typeof runId === "string" && runId.startsWith("dreaming-narrative-");
+  return typeof runId === "string" && runId.startsWith(DREAMING_NARRATIVE_RUN_PREFIX);
+}
+
+function hasDreamingNarrativeRunId(value: unknown): boolean {
+  return typeof value === "string" && value.includes(DREAMING_NARRATIVE_RUN_PREFIX);
+}
+
+function isDreamingNarrativeGeneratedRecord(record: unknown, messageText?: string | null): boolean {
+  if (isDreamingNarrativeBootstrapRecord(record)) {
+    return true;
+  }
+  if (messageText?.includes(DREAMING_NARRATIVE_PROMPT_PREFIX)) {
+    return true;
+  }
+  if (!record || typeof record !== "object" || Array.isArray(record)) {
+    return false;
+  }
+  const candidate = record as {
+    runId?: unknown;
+    sessionKey?: unknown;
+    data?: unknown;
+  };
+  if (
+    hasDreamingNarrativeRunId(candidate.runId) ||
+    hasDreamingNarrativeRunId(candidate.sessionKey)
+  ) {
+    return true;
+  }
+  if (!candidate.data || typeof candidate.data !== "object" || Array.isArray(candidate.data)) {
+    return false;
+  }
+  const nested = candidate.data as {
+    runId?: unknown;
+    sessionKey?: unknown;
+  };
+  return hasDreamingNarrativeRunId(nested.runId) || hasDreamingNarrativeRunId(nested.sessionKey);
 }

 export async function listSessionFilesForAgent(agentId: string): Promise<string[]> {
@@ -140,7 +177,7 @@ export async function buildSessionEntry(absPath: string): Promise<SessionFileEnt
      } catch {
        continue;
      }
-      if (!generatedByDreamingNarrative && isDreamingNarrativeBootstrapRecord(record)) {
+      if (!generatedByDreamingNarrative && isDreamingNarrativeGeneratedRecord(record)) {
        generatedByDreamingNarrative = true;
      }
      if (
@@ -163,6 +200,12 @@ export async function buildSessionEntry(absPath: string): Promise<SessionFileEnt
      if (!text) {
        continue;
      }
+      if (!generatedByDreamingNarrative && isDreamingNarrativeGeneratedRecord(record, text)) {
+        generatedByDreamingNarrative = true;
+      }
+      if (generatedByDreamingNarrative) {
+        continue;
+      }
      const safe = redactSensitiveText(text, { mode: "tools" });
      const label = message.role === "user" ? "User" : "Assistant";
      collected.push(`${label}: ${safe}`);