From 02cbfeaa86be015d916b7425b0ebd6068833b630 Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Tue, 14 Apr 2026 18:43:29 -0400 Subject: [PATCH] memory: block dreaming self-ingestion --- docs/concepts/dreaming.md | 3 + .../src/short-term-promotion.test.ts | 122 ++++++++++++++++++ .../memory-core/src/short-term-promotion.ts | 49 ++++++- .../host/session-files.test.ts | 26 ++++ src/memory-host-sdk/host/session-files.ts | 47 ++++++- 5 files changed, 244 insertions(+), 3 deletions(-) diff --git a/docs/concepts/dreaming.md b/docs/concepts/dreaming.md index 0a3795f4bb1..49f548e6af0 100644 --- a/docs/concepts/dreaming.md +++ b/docs/concepts/dreaming.md @@ -80,6 +80,9 @@ After each phase has enough material, `memory-core` runs a best-effort backgroun subagent turn (using the default runtime model) and appends a short diary entry. This diary is for human reading in the Dreams UI, not a promotion source. +Dreaming-generated diary/report artifacts are excluded from short-term +promotion. Only grounded memory snippets are eligible to promote into +`MEMORY.md`. There is also a grounded historical backfill lane for review and recovery work: diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index 85e384277be..9be0b976963 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -203,6 +203,33 @@ describe("short-term promotion", () => { }); }); + it("ignores contaminated dreaming snippets when recording short-term recalls", async () => { + await withTempWorkspace(async (workspaceDir) => { + await recordShortTermRecalls({ + workspaceDir, + query: "action preference", + results: [ + { + path: "memory/2026-04-03.md", + source: "memory", + startLine: 1, + endLine: 1, + score: 0.92, + snippet: + "Candidate: Default to action. confidence: 0.76 evidence: memory/.dreams/session-corpus/2026-04-08.txt:1-1 recalls: 3 status: staged", + }, + ], + }); + + expect( + JSON.parse(await fs.readFile(resolveShortTermRecallStorePath(workspaceDir), "utf-8")), + ).toMatchObject({ + version: 1, + entries: {}, + }); + }); + }); + it("records recalls and ranks candidates with weighted scores", async () => { await withTempWorkspace(async (workspaceDir) => { await recordShortTermRecalls({ @@ -940,6 +967,54 @@ describe("short-term promotion", () => { }); }); + it("does not rank contaminated dreaming snippets from an existing short-term store", async () => { + await withTempWorkspace(async (workspaceDir) => { + const storePath = resolveShortTermRecallStorePath(workspaceDir); + await fs.writeFile( + storePath, + JSON.stringify( + { + version: 1, + updatedAt: "2026-04-04T00:00:00.000Z", + entries: { + contaminated: { + key: "contaminated", + path: "memory/2026-04-03.md", + startLine: 1, + endLine: 1, + source: "memory", + snippet: + "Reflections: Theme: assistant. confidence: 1.00 evidence: memory/.dreams/session-corpus/2026-04-08.txt:2-2 recalls: 4 status: staged", + recallCount: 4, + dailyCount: 0, + groundedCount: 0, + totalScore: 3.6, + maxScore: 0.95, + firstRecalledAt: "2026-04-03T00:00:00.000Z", + lastRecalledAt: "2026-04-04T00:00:00.000Z", + queryHashes: ["a", "b"], + recallDays: ["2026-04-03", "2026-04-04"], + conceptTags: ["assistant"], + }, + }, + }, + null, + 2, + ), + "utf-8", + ); + + const ranked = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + + expect(ranked).toEqual([]); + }); + }); + it("skips direct candidates that exceed maxAgeDays during apply", async () => { await withTempWorkspace(async (workspaceDir) => { const applied = await applyShortTermPromotions({ @@ -987,6 +1062,53 @@ describe("short-term promotion", () => { }); }); + it("does not append contaminated dreaming snippets during direct apply", async () => { + await withTempWorkspace(async (workspaceDir) => { + const applied = await applyShortTermPromotions({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + candidates: [ + { + key: "memory:memory/2026-04-03.md:1:1", + path: "memory/2026-04-03.md", + startLine: 1, + endLine: 1, + source: "memory", + snippet: + "Candidate: Default to action. confidence: 0.76 evidence: memory/.dreams/session-corpus/2026-04-08.txt:1-1 recalls: 3 status: staged", + recallCount: 4, + avgScore: 0.97, + maxScore: 0.97, + uniqueQueries: 2, + firstRecalledAt: "2026-04-03T00:00:00.000Z", + lastRecalledAt: "2026-04-04T00:00:00.000Z", + ageDays: 0, + score: 0.99, + recallDays: ["2026-04-03", "2026-04-04"], + conceptTags: ["assistant"], + components: { + frequency: 1, + relevance: 1, + diversity: 1, + recency: 1, + consolidation: 1, + conceptual: 1, + }, + }, + ], + }); + + expect(applied.applied).toBe(0); + await expect( + fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + }); + it("applies promotion candidates to MEMORY.md and marks them promoted", async () => { await withTempWorkspace(async (workspaceDir) => { await writeDailyMemoryNote(workspaceDir, "2026-04-01", [ diff --git a/extensions/memory-core/src/short-term-promotion.ts b/extensions/memory-core/src/short-term-promotion.ts index e73a9b5816a..20bfd85122c 100644 --- a/extensions/memory-core/src/short-term-promotion.ts +++ b/extensions/memory-core/src/short-term-promotion.ts @@ -32,6 +32,8 @@ const SHORT_TERM_LOCK_RELATIVE_PATH = path.join("memory", ".dreams", "short-term const SHORT_TERM_LOCK_WAIT_TIMEOUT_MS = 10_000; const SHORT_TERM_LOCK_STALE_MS = 60_000; const SHORT_TERM_LOCK_RETRY_DELAY_MS = 40; +const DREAMING_NARRATIVE_PROMPT_PREFIX = "Write a dream diary entry from these memory fragments"; +const DREAMING_PROMOTION_META_PREFIX = "openclaw-memory-promotion:"; // Repeated dreaming revisits should be able to clear the default promotion gate // without requiring separate organic recall traffic for the same snippet. const PHASE_SIGNAL_LIGHT_BOOST_MAX = 0.06; @@ -235,6 +237,37 @@ function normalizeSnippet(raw: string): string { return trimmed.replace(/\s+/g, " "); } +function isContaminatedDreamingSnippet(raw: string): boolean { + const snippet = normalizeSnippet(raw); + if (!snippet) { + return false; + } + if ( + snippet.includes(DREAMING_NARRATIVE_PROMPT_PREFIX) || + snippet.includes(DREAMING_PROMOTION_META_PREFIX) || + snippet.includes("dreaming-narrative-") + ) { + return true; + } + + const hasCandidateLead = /^Candidate:/i.test(snippet) || /[([]\s*Candidate:/i.test(snippet); + const hasReflectionLead = + /^Reflections?:/i.test(snippet) || /[([]\s*Reflections?:/i.test(snippet); + const hasConfidence = /\bconfidence:\s*\d/i.test(snippet); + const hasEvidence = /\bevidence:\s*(?:memory\/\.dreams\/session-corpus\/|memory\/)/i.test( + snippet, + ); + const hasStatus = /\bstatus:\s*staged\b/i.test(snippet); + const hasRecalls = /\brecalls:\s*\d+\b/i.test(snippet); + if (hasEvidence && (hasCandidateLead || hasReflectionLead)) { + return true; + } + const structuredMarkers = [hasConfidence, hasEvidence, hasStatus, hasRecalls].filter( + Boolean, + ).length; + return (hasCandidateLead || hasReflectionLead) && structuredMarkers >= 2; +} + function normalizeMemoryPath(rawPath: string): string { return rawPath.replaceAll("\\", "/").replace(/^\.\//, ""); } @@ -409,6 +442,9 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore { ? entry.claimHash.trim() : undefined; const snippet = typeof entry.snippet === "string" ? normalizeSnippet(entry.snippet) : ""; + if (snippet && isContaminatedDreamingSnippet(snippet)) { + continue; + } const queryHashes = Array.isArray(entry.queryHashes) ? normalizeDistinctStrings(entry.queryHashes, MAX_QUERY_HASHES) : []; @@ -849,6 +885,9 @@ export async function recordShortTermRecalls(params: { for (const result of relevant) { const normalizedPath = normalizeMemoryPath(result.path); const snippet = normalizeSnippet(result.snippet); + if (!snippet || isContaminatedDreamingSnippet(snippet)) { + continue; + } const claimHash = snippet ? buildClaimHash(snippet) : undefined; const groundedKey = claimHash ? buildEntryKey({ @@ -954,6 +993,7 @@ export async function recordGroundedShortTermCandidates(params: { const normalizedPath = normalizeMemoryPath(item.path); if ( !snippet || + isContaminatedDreamingSnippet(snippet) || !normalizedPath || !isShortTermMemoryPath(normalizedPath) || !Number.isFinite(item.startLine) || @@ -1136,6 +1176,9 @@ export async function rankShortTermPromotionCandidates( if (!entry || entry.source !== "memory" || !isShortTermMemoryPath(entry.path)) { continue; } + if (isContaminatedDreamingSnippet(entry.snippet)) { + continue; + } if (!includePromoted && entry.promotedAt) { continue; } @@ -1471,6 +1514,9 @@ export async function applyShortTermPromotions( const store = await readStore(workspaceDir, nowIso); const selected = options.candidates .filter((candidate) => { + if (isContaminatedDreamingSnippet(candidate.snippet)) { + return false; + } if (candidate.promotedAt) { return false; } @@ -1506,7 +1552,7 @@ export async function applyShortTermPromotions( const rehydratedSelected: PromotionCandidate[] = []; for (const candidate of selected) { const rehydrated = await rehydratePromotionCandidate(workspaceDir, candidate); - if (rehydrated) { + if (rehydrated && !isContaminatedDreamingSnippet(rehydrated.snippet)) { rehydratedSelected.push(rehydrated); } } @@ -1881,4 +1927,5 @@ export const __testing = { calculatePhaseSignalBoost, buildClaimHash, totalSignalCountForEntry, + isContaminatedDreamingSnippet, }; diff --git a/src/memory-host-sdk/host/session-files.test.ts b/src/memory-host-sdk/host/session-files.test.ts index ddb105df222..5878b764240 100644 --- a/src/memory-host-sdk/host/session-files.test.ts +++ b/src/memory-host-sdk/host/session-files.test.ts @@ -174,4 +174,30 @@ describe("buildSessionEntry", () => { expect(entry).not.toBeNull(); expect(entry?.generatedByDreamingNarrative).toBe(true); }); + + it("flags dreaming narrative transcripts from the dream-diary prompt body", async () => { + const jsonlLines = [ + JSON.stringify({ + type: "message", + message: { + role: "user", + content: + "Write a dream diary entry from these memory fragments:\n- Candidate: durable note", + }, + }), + JSON.stringify({ + type: "message", + message: { role: "assistant", content: "A drifting archive breathed in moonlight." }, + }), + ]; + const filePath = path.join(tmpDir, "dreaming-prompt-session.jsonl"); + await fs.writeFile(filePath, jsonlLines.join("\n")); + + const entry = await buildSessionEntry(filePath); + + expect(entry).not.toBeNull(); + expect(entry?.generatedByDreamingNarrative).toBe(true); + expect(entry?.content).toBe(""); + expect(entry?.lineMap).toEqual([]); + }); }); diff --git a/src/memory-host-sdk/host/session-files.ts b/src/memory-host-sdk/host/session-files.ts index 954c6765a89..654d75c48dd 100644 --- a/src/memory-host-sdk/host/session-files.ts +++ b/src/memory-host-sdk/host/session-files.ts @@ -7,6 +7,8 @@ import { createSubsystemLogger } from "../../logging/subsystem.js"; import { hashText } from "./internal.js"; const log = createSubsystemLogger("memory"); +const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-"; +const DREAMING_NARRATIVE_PROMPT_PREFIX = "Write a dream diary entry from these memory fragments"; export type SessionFileEntry = { path: string; @@ -42,7 +44,42 @@ function isDreamingNarrativeBootstrapRecord(record: unknown): boolean { return false; } const runId = (candidate.data as { runId?: unknown }).runId; - return typeof runId === "string" && runId.startsWith("dreaming-narrative-"); + return typeof runId === "string" && runId.startsWith(DREAMING_NARRATIVE_RUN_PREFIX); +} + +function hasDreamingNarrativeRunId(value: unknown): boolean { + return typeof value === "string" && value.includes(DREAMING_NARRATIVE_RUN_PREFIX); +} + +function isDreamingNarrativeGeneratedRecord(record: unknown, messageText?: string | null): boolean { + if (isDreamingNarrativeBootstrapRecord(record)) { + return true; + } + if (messageText?.includes(DREAMING_NARRATIVE_PROMPT_PREFIX)) { + return true; + } + if (!record || typeof record !== "object" || Array.isArray(record)) { + return false; + } + const candidate = record as { + runId?: unknown; + sessionKey?: unknown; + data?: unknown; + }; + if ( + hasDreamingNarrativeRunId(candidate.runId) || + hasDreamingNarrativeRunId(candidate.sessionKey) + ) { + return true; + } + if (!candidate.data || typeof candidate.data !== "object" || Array.isArray(candidate.data)) { + return false; + } + const nested = candidate.data as { + runId?: unknown; + sessionKey?: unknown; + }; + return hasDreamingNarrativeRunId(nested.runId) || hasDreamingNarrativeRunId(nested.sessionKey); } export async function listSessionFilesForAgent(agentId: string): Promise { @@ -140,7 +177,7 @@ export async function buildSessionEntry(absPath: string): Promise