From a644e30245437534a4f6a8bb0b694fdce7780c1b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Apr 2026 06:15:28 +0100 Subject: [PATCH] fix(memory-core): retry unavailable dreaming model --- CHANGELOG.md | 4 + docs/concepts/dreaming.md | 4 +- docs/gateway/configuration-reference.md | 2 +- docs/reference/memory-config.md | 1 + .../src/dreaming-narrative.test.ts | 107 ++++++++++++ .../memory-core/src/dreaming-narrative.ts | 153 ++++++++++++++---- 6 files changed, 240 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b38ca8697b0..8f8da5daf35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Docs: https://docs.openclaw.ai - Gateway/chat: accept non-image attachments through `chat.send` by staging them as agent-readable media paths, while keeping unsupported RPC attachment paths explicit instead of silently dropping files. Fixes #48123. (#67572) Thanks @samzong. +### Fixes + +- Memory/Dreaming: retry Dream Diary once with the session default when a configured dreaming model is unavailable, while leaving subagent trust and allowlist errors visible instead of silently masking configuration problems. Refs #67409 and #69209. Thanks @Ghiggins18 and @everySympathy. + ## 2026.4.27 ### Changes diff --git a/docs/concepts/dreaming.md b/docs/concepts/dreaming.md index 341c6a001d8..ec1c69e4ccc 100644 --- a/docs/concepts/dreaming.md +++ b/docs/concepts/dreaming.md @@ -72,7 +72,7 @@ Dreaming can ingest redacted session transcripts into the dreaming corpus. When ## Dream Diary -Dreaming also keeps a narrative **Dream Diary** in `DREAMS.md`. After each phase has enough material, `memory-core` runs a best-effort background subagent turn and appends a short diary entry. It uses the default runtime model unless `dreaming.model` is configured. +Dreaming also keeps a narrative **Dream Diary** in `DREAMS.md`. After each phase has enough material, `memory-core` runs a best-effort background subagent turn and appends a short diary entry. It uses the default runtime model unless `dreaming.model` is configured. If the configured model is unavailable, Dream Diary retries once with the session default model. This diary is for human reading in the Dreams UI, not a promotion source. Dreaming-generated diary/report artifacts are excluded from short-term promotion. Only grounded memory snippets are eligible to promote into `MEMORY.md`. @@ -216,7 +216,7 @@ All settings live under `plugins.entries.memory-core.config.dreaming`. -`dreaming.model` requires `plugins.entries.memory-core.subagent.allowModelOverride: true`. To restrict it, also set `plugins.entries.memory-core.subagent.allowedModels`. +`dreaming.model` requires `plugins.entries.memory-core.subagent.allowModelOverride: true`. To restrict it, also set `plugins.entries.memory-core.subagent.allowedModels`. Trust or allowlist failures stay visible instead of falling back silently; the retry only covers model-unavailable errors. diff --git a/docs/gateway/configuration-reference.md b/docs/gateway/configuration-reference.md index c8d2fd57146..ded910bb30b 100644 --- a/docs/gateway/configuration-reference.md +++ b/docs/gateway/configuration-reference.md @@ -185,7 +185,7 @@ See [MCP](/cli/mcp#openclaw-as-an-mcp-client-registry) and - `plugins.entries.memory-core.config.dreaming`: memory dreaming settings. See [Dreaming](/concepts/dreaming) for phases and thresholds. - `enabled`: master dreaming switch (default `false`). - `frequency`: cron cadence for each full dreaming sweep (`"0 3 * * *"` by default). - - `model`: optional Dream Diary subagent model override. Requires `plugins.entries.memory-core.subagent.allowModelOverride: true`; pair with `allowedModels` to restrict targets. + - `model`: optional Dream Diary subagent model override. Requires `plugins.entries.memory-core.subagent.allowModelOverride: true`; pair with `allowedModels` to restrict targets. Model-unavailable errors retry once with the session default model; trust or allowlist failures do not fall back silently. - phase policy and thresholds are implementation details (not user-facing config keys). - Full memory config lives in [Memory configuration reference](/reference/memory-config): - `agents.defaults.memorySearch.*` diff --git a/docs/reference/memory-config.md b/docs/reference/memory-config.md index c213dbbc089..d7f2ed958ec 100644 --- a/docs/reference/memory-config.md +++ b/docs/reference/memory-config.md @@ -612,6 +612,7 @@ For conceptual behavior and slash commands, see [Dreaming](/concepts/dreaming). - Dreaming writes machine state to `memory/.dreams/`. - Dreaming writes human-readable narrative output to `DREAMS.md` (or existing `dreams.md`). - `dreaming.model` uses the existing plugin subagent trust gate; set `plugins.entries.memory-core.subagent.allowModelOverride: true` before enabling it. +- Dream Diary retries once with the session default model when the configured model is unavailable. Trust or allowlist failures are logged and are not silently retried. - The light/deep/REM phase policy and thresholds are internal behavior, not user-facing config. diff --git a/extensions/memory-core/src/dreaming-narrative.test.ts b/extensions/memory-core/src/dreaming-narrative.test.ts index acb14a50c0e..82f2404dd51 100644 --- a/extensions/memory-core/src/dreaming-narrative.test.ts +++ b/extensions/memory-core/src/dreaming-narrative.test.ts @@ -618,6 +618,113 @@ describe("generateAndAppendDreamNarrative", () => { expect(logger.info).toHaveBeenCalled(); }); + it("retries with the session default when the configured model cannot start", async () => { + const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); + const subagent = createMockSubagent("The default model carried the diary home."); + subagent.run.mockRejectedValueOnce(new Error("model unavailable")); + const logger = createMockLogger(); + const nowMs = Date.parse("2026-04-05T03:00:00Z"); + const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12); + const expectedSessionKey = `dreaming-narrative-light-${workspaceHash}-${nowMs}`; + const retrySessionKey = `${expectedSessionKey}-retry-1`; + + await generateAndAppendDreamNarrative({ + subagent, + workspaceDir, + data: { + phase: "light", + snippets: ["API endpoints need authentication"], + }, + nowMs, + timezone: "UTC", + model: "ollama/missing-model", + logger, + }); + + expect(subagent.run).toHaveBeenCalledTimes(2); + expect(subagent.run.mock.calls[0]?.[0]).toMatchObject({ + sessionKey: expectedSessionKey, + model: "ollama/missing-model", + }); + expect(subagent.run.mock.calls[1]?.[0]).toMatchObject({ + sessionKey: retrySessionKey, + }); + expect(subagent.run.mock.calls[1]?.[0]).not.toHaveProperty("model"); + expect(subagent.getSessionMessages).toHaveBeenCalledWith({ + sessionKey: retrySessionKey, + limit: 5, + }); + expect(subagent.deleteSession).toHaveBeenCalledOnce(); + expect(subagent.deleteSession).toHaveBeenCalledWith({ sessionKey: retrySessionKey }); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("session default")); + }); + + it("retries with the session default when the configured model run ends unavailable", async () => { + const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); + const subagent = createMockSubagent("The default model carried the diary home."); + subagent.run + .mockResolvedValueOnce({ runId: "run-configured" }) + .mockResolvedValueOnce({ runId: "run-default" }); + subagent.waitForRun + .mockResolvedValueOnce({ status: "error", error: "unknown model: ollama/missing-model" }) + .mockResolvedValueOnce({ status: "ok" }); + const logger = createMockLogger(); + const nowMs = Date.parse("2026-04-05T03:00:00Z"); + const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12); + const expectedSessionKey = `dreaming-narrative-rem-${workspaceHash}-${nowMs}`; + const retrySessionKey = `${expectedSessionKey}-retry-1`; + + await generateAndAppendDreamNarrative({ + subagent, + workspaceDir, + data: { + phase: "rem", + snippets: ["The index remembered a missing provider."], + }, + nowMs, + timezone: "UTC", + model: "ollama/missing-model", + logger, + }); + + expect(subagent.waitForRun).toHaveBeenCalledTimes(2); + expect(subagent.getSessionMessages).toHaveBeenCalledWith({ + sessionKey: retrySessionKey, + limit: 5, + }); + expect(subagent.deleteSession).toHaveBeenCalledTimes(2); + expect(subagent.deleteSession.mock.calls[0]?.[0]).toEqual({ sessionKey: expectedSessionKey }); + expect(subagent.deleteSession.mock.calls[1]?.[0]).toEqual({ sessionKey: retrySessionKey }); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("unknown model")); + }); + + it("does not hide configured model authorization failures by retrying", async () => { + const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); + const subagent = createMockSubagent(""); + subagent.run.mockRejectedValue( + new Error("provider/model override is not authorized for this plugin subagent run."), + ); + const logger = createMockLogger(); + + await generateAndAppendDreamNarrative({ + subagent, + workspaceDir, + data: { + phase: "light", + snippets: ["API endpoints need authentication"], + }, + model: "ollama/missing-model", + logger, + }); + + expect(subagent.run).toHaveBeenCalledOnce(); + expect(subagent.waitForRun).not.toHaveBeenCalled(); + expect(subagent.deleteSession).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining("narrative generation failed"), + ); + }); + it("skips narrative when no snippets are available", async () => { const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-"); const subagent = createMockSubagent("Should not appear."); diff --git a/extensions/memory-core/src/dreaming-narrative.ts b/extensions/memory-core/src/dreaming-narrative.ts index f67e7e65472..29fd53b7e5d 100644 --- a/extensions/memory-core/src/dreaming-narrative.ts +++ b/extensions/memory-core/src/dreaming-narrative.ts @@ -145,6 +145,56 @@ function buildRequestScopedFallbackNarrative(data: NarrativePhaseData): string { ); } +function buildNarrativeAttemptSessionKey(baseSessionKey: string, attempt: number): string { + return attempt === 0 ? baseSessionKey : `${baseSessionKey}-retry-${attempt}`; +} + +function isConfiguredModelUnavailableNarrativeError(raw: string): boolean { + const message = raw.trim(); + if (!message) { + return false; + } + if (/requested model may be(?: temporarily)? unavailable/i.test(message)) { + return true; + } + if (/model unavailable/i.test(message)) { + return true; + } + if (/no endpoints found for/i.test(message)) { + return true; + } + if (/unknown model/i.test(message)) { + return true; + } + if (/model(?:[_\-\s])?not(?:[_\-\s])?found/i.test(message)) { + return true; + } + if (/\b404\b/.test(message) && /not(?:[_\-\s])?found/i.test(message)) { + return true; + } + if (/not_found_error/i.test(message)) { + return true; + } + if (/models\/[^\s]+ is not found/i.test(message)) { + return true; + } + if (/model/i.test(message) && /does not exist/i.test(message)) { + return true; + } + if (/unsupported model/i.test(message)) { + return true; + } + if (/is not a valid model id/i.test(message)) { + return true; + } + return false; +} + +function formatNarrativeTerminalStatus(params: { status: string; error?: string }): string { + const detail = params.error?.trim(); + return detail ? `status=${params.status} (${detail})` : `status=${params.status}`; +} + async function startNarrativeRunOrFallback(params: { subagent: SubagentSurface; sessionKey: string; @@ -869,39 +919,81 @@ export async function generateAndAppendDreamNarrative(params: { nowMs, }); const message = buildNarrativePrompt(params.data); - let runId: string | null = null; - let shouldDeleteSession = false; + const attempts: Array<{ sessionKey: string; runId: string | null }> = []; + let successfulSessionKey: string | null = null; try { - runId = await startNarrativeRunOrFallback({ - subagent: params.subagent, - sessionKey, - message, - data: params.data, - workspaceDir: params.workspaceDir, - nowMs, - timezone: params.timezone, - model: params.model, - logger: params.logger, - }); - if (!runId) { - return; + const attemptModels = params.model ? [params.model, undefined] : [undefined]; + + for (const [attemptIndex, attemptModel] of attemptModels.entries()) { + const attemptSessionKey = buildNarrativeAttemptSessionKey(sessionKey, attemptIndex); + const attempt = { sessionKey: attemptSessionKey, runId: null as string | null }; + attempts.push(attempt); + + try { + const runId = await startNarrativeRunOrFallback({ + subagent: params.subagent, + sessionKey: attemptSessionKey, + message, + data: params.data, + workspaceDir: params.workspaceDir, + nowMs, + timezone: params.timezone, + model: attemptModel, + logger: params.logger, + }); + if (!runId) { + return; + } + attempt.runId = runId; + + const result = await params.subagent.waitForRun({ + runId, + timeoutMs: NARRATIVE_TIMEOUT_MS, + }); + + if (result.status === "ok") { + successfulSessionKey = attemptSessionKey; + break; + } + + if ( + attemptModel && + result.status === "error" && + isConfiguredModelUnavailableNarrativeError(result.error ?? "") + ) { + params.logger.warn( + `memory-core: narrative generation ended with ${formatNarrativeTerminalStatus({ + status: result.status, + error: result.error, + })} for ${params.data.phase} phase using configured model "${attemptModel}"; retrying with the session default.`, + ); + continue; + } + + params.logger.warn( + `memory-core: narrative generation ended with ${formatNarrativeTerminalStatus({ + status: result.status, + error: result.error, + })} for ${params.data.phase} phase.`, + ); + return; + } catch (err) { + if (attemptModel && isConfiguredModelUnavailableNarrativeError(formatErrorMessage(err))) { + params.logger.warn( + `memory-core: narrative generation could not start with configured model "${attemptModel}" for ${params.data.phase} phase; retrying with the session default (${formatErrorMessage(err)}).`, + ); + continue; + } + throw err; + } } - shouldDeleteSession = true; - const result = await params.subagent.waitForRun({ - runId, - timeoutMs: NARRATIVE_TIMEOUT_MS, - }); - - if (result.status !== "ok") { - params.logger.warn( - `memory-core: narrative generation ended with status=${result.status} for ${params.data.phase} phase.`, - ); + if (!successfulSessionKey) { return; } const { messages } = await params.subagent.getSessionMessages({ - sessionKey, + sessionKey: successfulSessionKey, limit: 5, }); @@ -931,9 +1023,14 @@ export async function generateAndAppendDreamNarrative(params: { } finally { // Only cleanup after a run was accepted. Request-scoped fallback writes a // local diary entry without creating a subagent session. - if (shouldDeleteSession && params.subagent) { + const cleanedSessionKeys = new Set(); + for (const attempt of attempts) { + if (!attempt.runId || cleanedSessionKeys.has(attempt.sessionKey)) { + continue; + } + cleanedSessionKeys.add(attempt.sessionKey); try { - await params.subagent.deleteSession({ sessionKey }); + await params.subagent.deleteSession({ sessionKey: attempt.sessionKey }); } catch (cleanupErr) { params.logger.warn( `memory-core: narrative session cleanup failed for ${params.data.phase} phase: ${formatErrorMessage(cleanupErr)}`,