mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:50:43 +00:00
fix(memory-core): retry unavailable dreaming model
This commit is contained in:
@@ -8,6 +8,10 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
- Gateway/chat: accept non-image attachments through `chat.send` by staging them as agent-readable media paths, while keeping unsupported RPC attachment paths explicit instead of silently dropping files. Fixes #48123. (#67572) Thanks @samzong.
|
||||
|
||||
### Fixes
|
||||
|
||||
- Memory/Dreaming: retry Dream Diary once with the session default when a configured dreaming model is unavailable, while leaving subagent trust and allowlist errors visible instead of silently masking configuration problems. Refs #67409 and #69209. Thanks @Ghiggins18 and @everySympathy.
|
||||
|
||||
## 2026.4.27
|
||||
|
||||
### Changes
|
||||
|
||||
@@ -72,7 +72,7 @@ Dreaming can ingest redacted session transcripts into the dreaming corpus. When
|
||||
|
||||
## Dream Diary
|
||||
|
||||
Dreaming also keeps a narrative **Dream Diary** in `DREAMS.md`. After each phase has enough material, `memory-core` runs a best-effort background subagent turn and appends a short diary entry. It uses the default runtime model unless `dreaming.model` is configured.
|
||||
Dreaming also keeps a narrative **Dream Diary** in `DREAMS.md`. After each phase has enough material, `memory-core` runs a best-effort background subagent turn and appends a short diary entry. It uses the default runtime model unless `dreaming.model` is configured. If the configured model is unavailable, Dream Diary retries once with the session default model.
|
||||
|
||||
<Note>
|
||||
This diary is for human reading in the Dreams UI, not a promotion source. Dreaming-generated diary/report artifacts are excluded from short-term promotion. Only grounded memory snippets are eligible to promote into `MEMORY.md`.
|
||||
@@ -216,7 +216,7 @@ All settings live under `plugins.entries.memory-core.config.dreaming`.
|
||||
</ParamField>
|
||||
|
||||
<Warning>
|
||||
`dreaming.model` requires `plugins.entries.memory-core.subagent.allowModelOverride: true`. To restrict it, also set `plugins.entries.memory-core.subagent.allowedModels`.
|
||||
`dreaming.model` requires `plugins.entries.memory-core.subagent.allowModelOverride: true`. To restrict it, also set `plugins.entries.memory-core.subagent.allowedModels`. Trust or allowlist failures stay visible instead of falling back silently; the retry only covers model-unavailable errors.
|
||||
</Warning>
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -185,7 +185,7 @@ See [MCP](/cli/mcp#openclaw-as-an-mcp-client-registry) and
|
||||
- `plugins.entries.memory-core.config.dreaming`: memory dreaming settings. See [Dreaming](/concepts/dreaming) for phases and thresholds.
|
||||
- `enabled`: master dreaming switch (default `false`).
|
||||
- `frequency`: cron cadence for each full dreaming sweep (`"0 3 * * *"` by default).
|
||||
- `model`: optional Dream Diary subagent model override. Requires `plugins.entries.memory-core.subagent.allowModelOverride: true`; pair with `allowedModels` to restrict targets.
|
||||
- `model`: optional Dream Diary subagent model override. Requires `plugins.entries.memory-core.subagent.allowModelOverride: true`; pair with `allowedModels` to restrict targets. Model-unavailable errors retry once with the session default model; trust or allowlist failures do not fall back silently.
|
||||
- phase policy and thresholds are implementation details (not user-facing config keys).
|
||||
- Full memory config lives in [Memory configuration reference](/reference/memory-config):
|
||||
- `agents.defaults.memorySearch.*`
|
||||
|
||||
@@ -612,6 +612,7 @@ For conceptual behavior and slash commands, see [Dreaming](/concepts/dreaming).
|
||||
- Dreaming writes machine state to `memory/.dreams/`.
|
||||
- Dreaming writes human-readable narrative output to `DREAMS.md` (or existing `dreams.md`).
|
||||
- `dreaming.model` uses the existing plugin subagent trust gate; set `plugins.entries.memory-core.subagent.allowModelOverride: true` before enabling it.
|
||||
- Dream Diary retries once with the session default model when the configured model is unavailable. Trust or allowlist failures are logged and are not silently retried.
|
||||
- The light/deep/REM phase policy and thresholds are internal behavior, not user-facing config.
|
||||
</Note>
|
||||
|
||||
|
||||
@@ -618,6 +618,113 @@ describe("generateAndAppendDreamNarrative", () => {
|
||||
expect(logger.info).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries with the session default when the configured model cannot start", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("The default model carried the diary home.");
|
||||
subagent.run.mockRejectedValueOnce(new Error("model unavailable"));
|
||||
const logger = createMockLogger();
|
||||
const nowMs = Date.parse("2026-04-05T03:00:00Z");
|
||||
const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12);
|
||||
const expectedSessionKey = `dreaming-narrative-light-${workspaceHash}-${nowMs}`;
|
||||
const retrySessionKey = `${expectedSessionKey}-retry-1`;
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: {
|
||||
phase: "light",
|
||||
snippets: ["API endpoints need authentication"],
|
||||
},
|
||||
nowMs,
|
||||
timezone: "UTC",
|
||||
model: "ollama/missing-model",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.run).toHaveBeenCalledTimes(2);
|
||||
expect(subagent.run.mock.calls[0]?.[0]).toMatchObject({
|
||||
sessionKey: expectedSessionKey,
|
||||
model: "ollama/missing-model",
|
||||
});
|
||||
expect(subagent.run.mock.calls[1]?.[0]).toMatchObject({
|
||||
sessionKey: retrySessionKey,
|
||||
});
|
||||
expect(subagent.run.mock.calls[1]?.[0]).not.toHaveProperty("model");
|
||||
expect(subagent.getSessionMessages).toHaveBeenCalledWith({
|
||||
sessionKey: retrySessionKey,
|
||||
limit: 5,
|
||||
});
|
||||
expect(subagent.deleteSession).toHaveBeenCalledOnce();
|
||||
expect(subagent.deleteSession).toHaveBeenCalledWith({ sessionKey: retrySessionKey });
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("session default"));
|
||||
});
|
||||
|
||||
it("retries with the session default when the configured model run ends unavailable", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("The default model carried the diary home.");
|
||||
subagent.run
|
||||
.mockResolvedValueOnce({ runId: "run-configured" })
|
||||
.mockResolvedValueOnce({ runId: "run-default" });
|
||||
subagent.waitForRun
|
||||
.mockResolvedValueOnce({ status: "error", error: "unknown model: ollama/missing-model" })
|
||||
.mockResolvedValueOnce({ status: "ok" });
|
||||
const logger = createMockLogger();
|
||||
const nowMs = Date.parse("2026-04-05T03:00:00Z");
|
||||
const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12);
|
||||
const expectedSessionKey = `dreaming-narrative-rem-${workspaceHash}-${nowMs}`;
|
||||
const retrySessionKey = `${expectedSessionKey}-retry-1`;
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: {
|
||||
phase: "rem",
|
||||
snippets: ["The index remembered a missing provider."],
|
||||
},
|
||||
nowMs,
|
||||
timezone: "UTC",
|
||||
model: "ollama/missing-model",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.waitForRun).toHaveBeenCalledTimes(2);
|
||||
expect(subagent.getSessionMessages).toHaveBeenCalledWith({
|
||||
sessionKey: retrySessionKey,
|
||||
limit: 5,
|
||||
});
|
||||
expect(subagent.deleteSession).toHaveBeenCalledTimes(2);
|
||||
expect(subagent.deleteSession.mock.calls[0]?.[0]).toEqual({ sessionKey: expectedSessionKey });
|
||||
expect(subagent.deleteSession.mock.calls[1]?.[0]).toEqual({ sessionKey: retrySessionKey });
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("unknown model"));
|
||||
});
|
||||
|
||||
it("does not hide configured model authorization failures by retrying", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.run.mockRejectedValue(
|
||||
new Error("provider/model override is not authorized for this plugin subagent run."),
|
||||
);
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: {
|
||||
phase: "light",
|
||||
snippets: ["API endpoints need authentication"],
|
||||
},
|
||||
model: "ollama/missing-model",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.run).toHaveBeenCalledOnce();
|
||||
expect(subagent.waitForRun).not.toHaveBeenCalled();
|
||||
expect(subagent.deleteSession).not.toHaveBeenCalled();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining("narrative generation failed"),
|
||||
);
|
||||
});
|
||||
|
||||
it("skips narrative when no snippets are available", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("Should not appear.");
|
||||
|
||||
@@ -145,6 +145,56 @@ function buildRequestScopedFallbackNarrative(data: NarrativePhaseData): string {
|
||||
);
|
||||
}
|
||||
|
||||
function buildNarrativeAttemptSessionKey(baseSessionKey: string, attempt: number): string {
|
||||
return attempt === 0 ? baseSessionKey : `${baseSessionKey}-retry-${attempt}`;
|
||||
}
|
||||
|
||||
function isConfiguredModelUnavailableNarrativeError(raw: string): boolean {
|
||||
const message = raw.trim();
|
||||
if (!message) {
|
||||
return false;
|
||||
}
|
||||
if (/requested model may be(?: temporarily)? unavailable/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/model unavailable/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/no endpoints found for/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/unknown model/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/model(?:[_\-\s])?not(?:[_\-\s])?found/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/\b404\b/.test(message) && /not(?:[_\-\s])?found/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/not_found_error/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/models\/[^\s]+ is not found/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/model/i.test(message) && /does not exist/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/unsupported model/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/is not a valid model id/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function formatNarrativeTerminalStatus(params: { status: string; error?: string }): string {
|
||||
const detail = params.error?.trim();
|
||||
return detail ? `status=${params.status} (${detail})` : `status=${params.status}`;
|
||||
}
|
||||
|
||||
async function startNarrativeRunOrFallback(params: {
|
||||
subagent: SubagentSurface;
|
||||
sessionKey: string;
|
||||
@@ -869,39 +919,81 @@ export async function generateAndAppendDreamNarrative(params: {
|
||||
nowMs,
|
||||
});
|
||||
const message = buildNarrativePrompt(params.data);
|
||||
let runId: string | null = null;
|
||||
let shouldDeleteSession = false;
|
||||
const attempts: Array<{ sessionKey: string; runId: string | null }> = [];
|
||||
let successfulSessionKey: string | null = null;
|
||||
try {
|
||||
runId = await startNarrativeRunOrFallback({
|
||||
subagent: params.subagent,
|
||||
sessionKey,
|
||||
message,
|
||||
data: params.data,
|
||||
workspaceDir: params.workspaceDir,
|
||||
nowMs,
|
||||
timezone: params.timezone,
|
||||
model: params.model,
|
||||
logger: params.logger,
|
||||
});
|
||||
if (!runId) {
|
||||
return;
|
||||
const attemptModels = params.model ? [params.model, undefined] : [undefined];
|
||||
|
||||
for (const [attemptIndex, attemptModel] of attemptModels.entries()) {
|
||||
const attemptSessionKey = buildNarrativeAttemptSessionKey(sessionKey, attemptIndex);
|
||||
const attempt = { sessionKey: attemptSessionKey, runId: null as string | null };
|
||||
attempts.push(attempt);
|
||||
|
||||
try {
|
||||
const runId = await startNarrativeRunOrFallback({
|
||||
subagent: params.subagent,
|
||||
sessionKey: attemptSessionKey,
|
||||
message,
|
||||
data: params.data,
|
||||
workspaceDir: params.workspaceDir,
|
||||
nowMs,
|
||||
timezone: params.timezone,
|
||||
model: attemptModel,
|
||||
logger: params.logger,
|
||||
});
|
||||
if (!runId) {
|
||||
return;
|
||||
}
|
||||
attempt.runId = runId;
|
||||
|
||||
const result = await params.subagent.waitForRun({
|
||||
runId,
|
||||
timeoutMs: NARRATIVE_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
if (result.status === "ok") {
|
||||
successfulSessionKey = attemptSessionKey;
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
attemptModel &&
|
||||
result.status === "error" &&
|
||||
isConfiguredModelUnavailableNarrativeError(result.error ?? "")
|
||||
) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation ended with ${formatNarrativeTerminalStatus({
|
||||
status: result.status,
|
||||
error: result.error,
|
||||
})} for ${params.data.phase} phase using configured model "${attemptModel}"; retrying with the session default.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation ended with ${formatNarrativeTerminalStatus({
|
||||
status: result.status,
|
||||
error: result.error,
|
||||
})} for ${params.data.phase} phase.`,
|
||||
);
|
||||
return;
|
||||
} catch (err) {
|
||||
if (attemptModel && isConfiguredModelUnavailableNarrativeError(formatErrorMessage(err))) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation could not start with configured model "${attemptModel}" for ${params.data.phase} phase; retrying with the session default (${formatErrorMessage(err)}).`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
shouldDeleteSession = true;
|
||||
|
||||
const result = await params.subagent.waitForRun({
|
||||
runId,
|
||||
timeoutMs: NARRATIVE_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
if (result.status !== "ok") {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation ended with status=${result.status} for ${params.data.phase} phase.`,
|
||||
);
|
||||
if (!successfulSessionKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { messages } = await params.subagent.getSessionMessages({
|
||||
sessionKey,
|
||||
sessionKey: successfulSessionKey,
|
||||
limit: 5,
|
||||
});
|
||||
|
||||
@@ -931,9 +1023,14 @@ export async function generateAndAppendDreamNarrative(params: {
|
||||
} finally {
|
||||
// Only cleanup after a run was accepted. Request-scoped fallback writes a
|
||||
// local diary entry without creating a subagent session.
|
||||
if (shouldDeleteSession && params.subagent) {
|
||||
const cleanedSessionKeys = new Set<string>();
|
||||
for (const attempt of attempts) {
|
||||
if (!attempt.runId || cleanedSessionKeys.has(attempt.sessionKey)) {
|
||||
continue;
|
||||
}
|
||||
cleanedSessionKeys.add(attempt.sessionKey);
|
||||
try {
|
||||
await params.subagent.deleteSession({ sessionKey });
|
||||
await params.subagent.deleteSession({ sessionKey: attempt.sessionKey });
|
||||
} catch (cleanupErr) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative session cleanup failed for ${params.data.phase} phase: ${formatErrorMessage(cleanupErr)}`,
|
||||
|
||||
Reference in New Issue
Block a user