fix(memory-core): retry unavailable dreaming model

This commit is contained in:
Peter Steinberger
2026-04-28 06:15:28 +01:00
parent 017b8db616
commit a644e30245
6 changed files with 240 additions and 31 deletions

View File

@@ -618,6 +618,113 @@ describe("generateAndAppendDreamNarrative", () => {
expect(logger.info).toHaveBeenCalled();
});
it("retries with the session default when the configured model cannot start", async () => {
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
const subagent = createMockSubagent("The default model carried the diary home.");
subagent.run.mockRejectedValueOnce(new Error("model unavailable"));
const logger = createMockLogger();
const nowMs = Date.parse("2026-04-05T03:00:00Z");
const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12);
const expectedSessionKey = `dreaming-narrative-light-${workspaceHash}-${nowMs}`;
const retrySessionKey = `${expectedSessionKey}-retry-1`;
await generateAndAppendDreamNarrative({
subagent,
workspaceDir,
data: {
phase: "light",
snippets: ["API endpoints need authentication"],
},
nowMs,
timezone: "UTC",
model: "ollama/missing-model",
logger,
});
expect(subagent.run).toHaveBeenCalledTimes(2);
expect(subagent.run.mock.calls[0]?.[0]).toMatchObject({
sessionKey: expectedSessionKey,
model: "ollama/missing-model",
});
expect(subagent.run.mock.calls[1]?.[0]).toMatchObject({
sessionKey: retrySessionKey,
});
expect(subagent.run.mock.calls[1]?.[0]).not.toHaveProperty("model");
expect(subagent.getSessionMessages).toHaveBeenCalledWith({
sessionKey: retrySessionKey,
limit: 5,
});
expect(subagent.deleteSession).toHaveBeenCalledOnce();
expect(subagent.deleteSession).toHaveBeenCalledWith({ sessionKey: retrySessionKey });
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("session default"));
});
it("retries with the session default when the configured model run ends unavailable", async () => {
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
const subagent = createMockSubagent("The default model carried the diary home.");
subagent.run
.mockResolvedValueOnce({ runId: "run-configured" })
.mockResolvedValueOnce({ runId: "run-default" });
subagent.waitForRun
.mockResolvedValueOnce({ status: "error", error: "unknown model: ollama/missing-model" })
.mockResolvedValueOnce({ status: "ok" });
const logger = createMockLogger();
const nowMs = Date.parse("2026-04-05T03:00:00Z");
const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12);
const expectedSessionKey = `dreaming-narrative-rem-${workspaceHash}-${nowMs}`;
const retrySessionKey = `${expectedSessionKey}-retry-1`;
await generateAndAppendDreamNarrative({
subagent,
workspaceDir,
data: {
phase: "rem",
snippets: ["The index remembered a missing provider."],
},
nowMs,
timezone: "UTC",
model: "ollama/missing-model",
logger,
});
expect(subagent.waitForRun).toHaveBeenCalledTimes(2);
expect(subagent.getSessionMessages).toHaveBeenCalledWith({
sessionKey: retrySessionKey,
limit: 5,
});
expect(subagent.deleteSession).toHaveBeenCalledTimes(2);
expect(subagent.deleteSession.mock.calls[0]?.[0]).toEqual({ sessionKey: expectedSessionKey });
expect(subagent.deleteSession.mock.calls[1]?.[0]).toEqual({ sessionKey: retrySessionKey });
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("unknown model"));
});
it("does not hide configured model authorization failures by retrying", async () => {
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
const subagent = createMockSubagent("");
subagent.run.mockRejectedValue(
new Error("provider/model override is not authorized for this plugin subagent run."),
);
const logger = createMockLogger();
await generateAndAppendDreamNarrative({
subagent,
workspaceDir,
data: {
phase: "light",
snippets: ["API endpoints need authentication"],
},
model: "ollama/missing-model",
logger,
});
expect(subagent.run).toHaveBeenCalledOnce();
expect(subagent.waitForRun).not.toHaveBeenCalled();
expect(subagent.deleteSession).not.toHaveBeenCalled();
expect(logger.warn).toHaveBeenCalledWith(
expect.stringContaining("narrative generation failed"),
);
});
it("skips narrative when no snippets are available", async () => {
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
const subagent = createMockSubagent("Should not appear.");

View File

@@ -145,6 +145,56 @@ function buildRequestScopedFallbackNarrative(data: NarrativePhaseData): string {
);
}
function buildNarrativeAttemptSessionKey(baseSessionKey: string, attempt: number): string {
return attempt === 0 ? baseSessionKey : `${baseSessionKey}-retry-${attempt}`;
}
function isConfiguredModelUnavailableNarrativeError(raw: string): boolean {
const message = raw.trim();
if (!message) {
return false;
}
if (/requested model may be(?: temporarily)? unavailable/i.test(message)) {
return true;
}
if (/model unavailable/i.test(message)) {
return true;
}
if (/no endpoints found for/i.test(message)) {
return true;
}
if (/unknown model/i.test(message)) {
return true;
}
if (/model(?:[_\-\s])?not(?:[_\-\s])?found/i.test(message)) {
return true;
}
if (/\b404\b/.test(message) && /not(?:[_\-\s])?found/i.test(message)) {
return true;
}
if (/not_found_error/i.test(message)) {
return true;
}
if (/models\/[^\s]+ is not found/i.test(message)) {
return true;
}
if (/model/i.test(message) && /does not exist/i.test(message)) {
return true;
}
if (/unsupported model/i.test(message)) {
return true;
}
if (/is not a valid model id/i.test(message)) {
return true;
}
return false;
}
function formatNarrativeTerminalStatus(params: { status: string; error?: string }): string {
const detail = params.error?.trim();
return detail ? `status=${params.status} (${detail})` : `status=${params.status}`;
}
async function startNarrativeRunOrFallback(params: {
subagent: SubagentSurface;
sessionKey: string;
@@ -869,39 +919,81 @@ export async function generateAndAppendDreamNarrative(params: {
nowMs,
});
const message = buildNarrativePrompt(params.data);
let runId: string | null = null;
let shouldDeleteSession = false;
const attempts: Array<{ sessionKey: string; runId: string | null }> = [];
let successfulSessionKey: string | null = null;
try {
runId = await startNarrativeRunOrFallback({
subagent: params.subagent,
sessionKey,
message,
data: params.data,
workspaceDir: params.workspaceDir,
nowMs,
timezone: params.timezone,
model: params.model,
logger: params.logger,
});
if (!runId) {
return;
const attemptModels = params.model ? [params.model, undefined] : [undefined];
for (const [attemptIndex, attemptModel] of attemptModels.entries()) {
const attemptSessionKey = buildNarrativeAttemptSessionKey(sessionKey, attemptIndex);
const attempt = { sessionKey: attemptSessionKey, runId: null as string | null };
attempts.push(attempt);
try {
const runId = await startNarrativeRunOrFallback({
subagent: params.subagent,
sessionKey: attemptSessionKey,
message,
data: params.data,
workspaceDir: params.workspaceDir,
nowMs,
timezone: params.timezone,
model: attemptModel,
logger: params.logger,
});
if (!runId) {
return;
}
attempt.runId = runId;
const result = await params.subagent.waitForRun({
runId,
timeoutMs: NARRATIVE_TIMEOUT_MS,
});
if (result.status === "ok") {
successfulSessionKey = attemptSessionKey;
break;
}
if (
attemptModel &&
result.status === "error" &&
isConfiguredModelUnavailableNarrativeError(result.error ?? "")
) {
params.logger.warn(
`memory-core: narrative generation ended with ${formatNarrativeTerminalStatus({
status: result.status,
error: result.error,
})} for ${params.data.phase} phase using configured model "${attemptModel}"; retrying with the session default.`,
);
continue;
}
params.logger.warn(
`memory-core: narrative generation ended with ${formatNarrativeTerminalStatus({
status: result.status,
error: result.error,
})} for ${params.data.phase} phase.`,
);
return;
} catch (err) {
if (attemptModel && isConfiguredModelUnavailableNarrativeError(formatErrorMessage(err))) {
params.logger.warn(
`memory-core: narrative generation could not start with configured model "${attemptModel}" for ${params.data.phase} phase; retrying with the session default (${formatErrorMessage(err)}).`,
);
continue;
}
throw err;
}
}
shouldDeleteSession = true;
const result = await params.subagent.waitForRun({
runId,
timeoutMs: NARRATIVE_TIMEOUT_MS,
});
if (result.status !== "ok") {
params.logger.warn(
`memory-core: narrative generation ended with status=${result.status} for ${params.data.phase} phase.`,
);
if (!successfulSessionKey) {
return;
}
const { messages } = await params.subagent.getSessionMessages({
sessionKey,
sessionKey: successfulSessionKey,
limit: 5,
});
@@ -931,9 +1023,14 @@ export async function generateAndAppendDreamNarrative(params: {
} finally {
// Only cleanup after a run was accepted. Request-scoped fallback writes a
// local diary entry without creating a subagent session.
if (shouldDeleteSession && params.subagent) {
const cleanedSessionKeys = new Set<string>();
for (const attempt of attempts) {
if (!attempt.runId || cleanedSessionKeys.has(attempt.sessionKey)) {
continue;
}
cleanedSessionKeys.add(attempt.sessionKey);
try {
await params.subagent.deleteSession({ sessionKey });
await params.subagent.deleteSession({ sessionKey: attempt.sessionKey });
} catch (cleanupErr) {
params.logger.warn(
`memory-core: narrative session cleanup failed for ${params.data.phase} phase: ${formatErrorMessage(cleanupErr)}`,