From bed53c77aa2aa749e117768cd15a11adde8a9f52 Mon Sep 17 00:00:00 2001 From: Mariano Date: Thu, 9 Apr 2026 23:31:10 +0200 Subject: [PATCH] fix(memory-core): add dreaming narrative idempotency (#63876) Merged via squash. Prepared head SHA: 34f317cbcf418b148cf1fdb75efb2dfb167d3f08 Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com> Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com> Reviewed-by: @mbelinky --- CHANGELOG.md | 9 ++- .../src/dreaming-narrative.test.ts | 70 ++++++++++++++++++- .../memory-core/src/dreaming-narrative.ts | 22 +++++- 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ff2ce8af2d..f49428b60bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,9 +29,15 @@ Docs: https://docs.openclaw.ai - Fireworks/FirePass: disable Kimi K2.5 Turbo reasoning output by forcing thinking off on the FirePass path and hardening the provider wrapper so hidden reasoning no longer leaks into visible replies. (#63607) Thanks @frankekn. - Sessions/model selection: preserve catalog-backed session model labels and keep already-qualified session model refs stable when catalog metadata is unavailable, so Control UI model selection survives reloads without bogus provider-prefixed values. (#61382) Thanks @Mule-ME. - Gateway/startup: keep WebSocket RPC available while channels and plugin sidecars start, hold `chat.history` unavailable until startup sidecars finish so synchronous history reads cannot stall startup (reported in #63450), refresh advertised gateway methods after deferred plugin reloads, and enforce the pre-auth WebSocket upgrade budget before the no-handler 503 path so upgrade floods cannot bypass connection limits during that window. (#63480) Thanks @neeravmakwana. +<<<<<<< HEAD - Dreaming/cron: reconcile managed dreaming cron from the resolved gateway startup config so boot-time schedule recovery respects the configured cadence and timezone. (#63873) Thanks @mbelinky. +||||||| parent of cc6ec8288a (Dreaming: harden atomic diary writes) +- Dreaming/diary: add idempotent narrative subagent runs and atomic `DREAMS.md` writes so repeated sweeps do not double-run the same narrative request or partially rewrite the diary. +======= +>>>>>>> cc6ec8288a (Dreaming: harden atomic diary writes) - Gateway/tailscale: start Tailscale exposure and the gateway update check before awaiting channel and plugin sidecar startup so remote operators are not locked out when startup sidecars stall. - QQBot/streaming: make block streaming configurable per QQ bot account via `streaming.mode` (`"partial"` | `"off"`, default `"partial"`) instead of hardcoding it off, so responses can be delivered incrementally. (#63746) +<<<<<<< HEAD - Dreaming/gateway: require `operator.admin` for persistent `/dreaming on|off` changes and treat missing gateway client scopes as unprivileged instead of silently allowing config writes. (#63872) Thanks @mbelinky. - Matrix/multi-account: keep room-level `account` scoping, inherited room overrides, and implicit account selection consistent across top-level default auth, named accounts, and cached-credential env setups. (#58449) thanks @Daanvdplas and @gumadeiras. - Gateway/pairing: prefer explicit QR bootstrap auth over earlier Tailscale auth classification so iOS `/pair qr` silent bootstrap pairing does not fall through to `pairing required`. (#59232) Thanks @ngutman. @@ -50,8 +56,9 @@ Docs: https://docs.openclaw.ai - Matrix/streaming: preserve ordered block flushes before tool, message, and agent boundaries, add explicit `channels.matrix.blockStreaming` opt-in so Matrix `streaming: "off"` stays final-only by default, and move MiniMax plain-text final handling into the MiniMax provider runtime instead of the shared core heuristic. (#59266) thanks @gumadeiras - Gateway/agents: fix stale run-context TTL cleanup so the new maintenance sweep compiles and resets orphaned run sequence state correctly. (#52731) thanks @artwalker - Memory/lancedb: accept `dreaming` config when `memory-lancedb` owns the memory slot so Dreaming surfaces can read slot-owner settings without schema rejection. (#63874) Thanks @mbelinky. +- Control UI/dreaming: keep the Dreaming trace area contained and scrollable so overlays no longer cover tabs or blow out the page layout. (#63875) Thanks @mbelinky. +- Dreaming/diary: add idempotent narrative subagent runs, preserve restrictive `DREAMS.md` permissions during atomic writes, and surface temp cleanup failures so repeated sweeps do not double-run the same narrative request or silently weaken diary safety. (#63876) Thanks @mbelinky. - Heartbeats/sessions: remove stale accumulated isolated heartbeat session keys when the next tick converges them back to the canonical sibling, so repaired sessions stop showing orphaned `:heartbeat:heartbeat` variants in session listings. (#59606) Thanks @rogerdigital. -- Control UI/dreaming: keep the Dreaming trace area contained and scrollable so overlays no longer cover tabs or blow out the page layout. - Cron/Telegram: collapse isolated announce delivery to the final assistant-visible text only for Telegram targets, while preserving existing multi-message direct delivery semantics for other channels. (#63228) Thanks @welfo-beo. - Gateway/thread routing: preserve Slack, Telegram, and Mattermost thread-child delivery targets so bound subagent completion messages land in the originating thread instead of top-level channels. (#54840) Thanks @yzzymt. diff --git a/extensions/memory-core/src/dreaming-narrative.test.ts b/extensions/memory-core/src/dreaming-narrative.test.ts index 7f832fe372c..b08b712b7b4 100644 --- a/extensions/memory-core/src/dreaming-narrative.test.ts +++ b/extensions/memory-core/src/dreaming-narrative.test.ts @@ -1,6 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { appendNarrativeEntry, buildBackfillDiaryEntry, @@ -18,6 +18,10 @@ import { createMemoryCoreTestHarness } from "./test-helpers.js"; const { createTempWorkspace } = createMemoryCoreTestHarness(); +afterEach(() => { + vi.restoreAllMocks(); +}); + describe("buildNarrativePrompt", () => { it("builds a prompt from snippets only", () => { const data: NarrativePhaseData = { @@ -312,6 +316,64 @@ describe("appendNarrativeEntry", () => { // Original content should still be there, after the diary. expect(content).toContain("# Existing"); }); + + it("keeps existing diary content intact when the atomic replace fails", async () => { + const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); + const dreamsPath = path.join(workspaceDir, "DREAMS.md"); + await fs.writeFile(dreamsPath, "# Existing\n", "utf-8"); + const renameError = Object.assign(new Error("replace failed"), { code: "ENOSPC" }); + const renameSpy = vi.spyOn(fs, "rename").mockRejectedValueOnce(renameError); + + await expect( + appendNarrativeEntry({ + workspaceDir, + narrative: "Appended dream.", + nowMs: Date.parse("2026-04-05T03:00:00Z"), + timezone: "UTC", + }), + ).rejects.toThrow("replace failed"); + + expect(renameSpy).toHaveBeenCalledOnce(); + await expect(fs.readFile(dreamsPath, "utf-8")).resolves.toBe("# Existing\n"); + }); + + it("preserves restrictive dreams file permissions across atomic replace", async () => { + const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); + const dreamsPath = path.join(workspaceDir, "DREAMS.md"); + await fs.writeFile(dreamsPath, "# Existing\n", { encoding: "utf-8", mode: 0o600 }); + await fs.chmod(dreamsPath, 0o600); + + await appendNarrativeEntry({ + workspaceDir, + narrative: "Appended dream.", + nowMs: Date.parse("2026-04-05T03:00:00Z"), + timezone: "UTC", + }); + + const stat = await fs.stat(dreamsPath); + expect(stat.mode & 0o777).toBe(0o600); + }); + + it("surfaces temp cleanup failure after atomic replace error", async () => { + const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); + const dreamsPath = path.join(workspaceDir, "DREAMS.md"); + await fs.writeFile(dreamsPath, "# Existing\n", "utf-8"); + vi.spyOn(fs, "rename").mockRejectedValueOnce( + Object.assign(new Error("replace failed"), { code: "ENOSPC" }), + ); + vi.spyOn(fs, "rm").mockRejectedValueOnce( + Object.assign(new Error("cleanup failed"), { code: "EACCES" }), + ); + + await expect( + appendNarrativeEntry({ + workspaceDir, + narrative: "Appended dream.", + nowMs: Date.parse("2026-04-05T03:00:00Z"), + timezone: "UTC", + }), + ).rejects.toThrow("cleanup also failed"); + }); }); describe("generateAndAppendDreamNarrative", () => { @@ -341,6 +403,8 @@ describe("generateAndAppendDreamNarrative", () => { const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); const subagent = createMockSubagent("The repository whispered of forgotten endpoints."); const logger = createMockLogger(); + const nowMs = Date.parse("2026-04-05T03:00:00Z"); + const expectedSessionKey = `dreaming-narrative-light-${nowMs}`; await generateAndAppendDreamNarrative({ subagent, @@ -349,13 +413,15 @@ describe("generateAndAppendDreamNarrative", () => { phase: "light", snippets: ["API endpoints need authentication"], }, - nowMs: Date.parse("2026-04-05T03:00:00Z"), + nowMs, timezone: "UTC", logger, }); expect(subagent.run).toHaveBeenCalledOnce(); expect(subagent.run.mock.calls[0][0]).toMatchObject({ + idempotencyKey: expectedSessionKey, + sessionKey: expectedSessionKey, deliver: false, }); expect(subagent.waitForRun).toHaveBeenCalledOnce(); diff --git a/extensions/memory-core/src/dreaming-narrative.ts b/extensions/memory-core/src/dreaming-narrative.ts index c15d2d87439..138fcf77d73 100644 --- a/extensions/memory-core/src/dreaming-narrative.ts +++ b/extensions/memory-core/src/dreaming-narrative.ts @@ -6,6 +6,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; type SubagentSurface = { run: (params: { + idempotencyKey: string; sessionKey: string; message: string; extraSystemPrompt?: string; @@ -277,12 +278,26 @@ async function assertSafeDreamsPath(dreamsPath: string): Promise { async function writeDreamsFileAtomic(dreamsPath: string, content: string): Promise { await assertSafeDreamsPath(dreamsPath); + const existing = await fs.stat(dreamsPath).catch((err: NodeJS.ErrnoException) => { + if (err.code === "ENOENT") { + return null; + } + throw err; + }); + const mode = existing?.mode ?? 0o600; const tempPath = `${dreamsPath}.${process.pid}.${Date.now()}.tmp`; - await fs.writeFile(tempPath, content, { encoding: "utf-8", flag: "wx" }); + await fs.writeFile(tempPath, content, { encoding: "utf-8", flag: "wx", mode }); + await fs.chmod(tempPath, mode).catch(() => undefined); try { await fs.rename(tempPath, dreamsPath); + await fs.chmod(dreamsPath, mode).catch(() => undefined); } catch (err) { - await fs.rm(tempPath, { force: true }).catch(() => {}); + const cleanupError = await fs.rm(tempPath, { force: true }).catch((rmErr) => rmErr); + if (cleanupError) { + throw new Error( + `Atomic DREAMS.md write failed (${formatErrorMessage(err)}); cleanup also failed (${formatErrorMessage(cleanupError)})`, + ); + } throw err; } } @@ -409,7 +424,7 @@ export async function appendNarrativeEntry(params: { } } - await fs.writeFile(dreamsPath, updated.endsWith("\n") ? updated : `${updated}\n`, "utf-8"); + await writeDreamsFileAtomic(dreamsPath, updated.endsWith("\n") ? updated : `${updated}\n`); return dreamsPath; } @@ -434,6 +449,7 @@ export async function generateAndAppendDreamNarrative(params: { try { const { runId } = await params.subagent.run({ + idempotencyKey: sessionKey, sessionKey, message, extraSystemPrompt: NARRATIVE_SYSTEM_PROMPT,