mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 17:20:45 +00:00
fix(memory-core): retry unavailable dreaming model
This commit is contained in:
@@ -618,6 +618,113 @@ describe("generateAndAppendDreamNarrative", () => {
|
||||
expect(logger.info).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries with the session default when the configured model cannot start", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("The default model carried the diary home.");
|
||||
subagent.run.mockRejectedValueOnce(new Error("model unavailable"));
|
||||
const logger = createMockLogger();
|
||||
const nowMs = Date.parse("2026-04-05T03:00:00Z");
|
||||
const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12);
|
||||
const expectedSessionKey = `dreaming-narrative-light-${workspaceHash}-${nowMs}`;
|
||||
const retrySessionKey = `${expectedSessionKey}-retry-1`;
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: {
|
||||
phase: "light",
|
||||
snippets: ["API endpoints need authentication"],
|
||||
},
|
||||
nowMs,
|
||||
timezone: "UTC",
|
||||
model: "ollama/missing-model",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.run).toHaveBeenCalledTimes(2);
|
||||
expect(subagent.run.mock.calls[0]?.[0]).toMatchObject({
|
||||
sessionKey: expectedSessionKey,
|
||||
model: "ollama/missing-model",
|
||||
});
|
||||
expect(subagent.run.mock.calls[1]?.[0]).toMatchObject({
|
||||
sessionKey: retrySessionKey,
|
||||
});
|
||||
expect(subagent.run.mock.calls[1]?.[0]).not.toHaveProperty("model");
|
||||
expect(subagent.getSessionMessages).toHaveBeenCalledWith({
|
||||
sessionKey: retrySessionKey,
|
||||
limit: 5,
|
||||
});
|
||||
expect(subagent.deleteSession).toHaveBeenCalledOnce();
|
||||
expect(subagent.deleteSession).toHaveBeenCalledWith({ sessionKey: retrySessionKey });
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("session default"));
|
||||
});
|
||||
|
||||
it("retries with the session default when the configured model run ends unavailable", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("The default model carried the diary home.");
|
||||
subagent.run
|
||||
.mockResolvedValueOnce({ runId: "run-configured" })
|
||||
.mockResolvedValueOnce({ runId: "run-default" });
|
||||
subagent.waitForRun
|
||||
.mockResolvedValueOnce({ status: "error", error: "unknown model: ollama/missing-model" })
|
||||
.mockResolvedValueOnce({ status: "ok" });
|
||||
const logger = createMockLogger();
|
||||
const nowMs = Date.parse("2026-04-05T03:00:00Z");
|
||||
const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12);
|
||||
const expectedSessionKey = `dreaming-narrative-rem-${workspaceHash}-${nowMs}`;
|
||||
const retrySessionKey = `${expectedSessionKey}-retry-1`;
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: {
|
||||
phase: "rem",
|
||||
snippets: ["The index remembered a missing provider."],
|
||||
},
|
||||
nowMs,
|
||||
timezone: "UTC",
|
||||
model: "ollama/missing-model",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.waitForRun).toHaveBeenCalledTimes(2);
|
||||
expect(subagent.getSessionMessages).toHaveBeenCalledWith({
|
||||
sessionKey: retrySessionKey,
|
||||
limit: 5,
|
||||
});
|
||||
expect(subagent.deleteSession).toHaveBeenCalledTimes(2);
|
||||
expect(subagent.deleteSession.mock.calls[0]?.[0]).toEqual({ sessionKey: expectedSessionKey });
|
||||
expect(subagent.deleteSession.mock.calls[1]?.[0]).toEqual({ sessionKey: retrySessionKey });
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("unknown model"));
|
||||
});
|
||||
|
||||
it("does not hide configured model authorization failures by retrying", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.run.mockRejectedValue(
|
||||
new Error("provider/model override is not authorized for this plugin subagent run."),
|
||||
);
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: {
|
||||
phase: "light",
|
||||
snippets: ["API endpoints need authentication"],
|
||||
},
|
||||
model: "ollama/missing-model",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.run).toHaveBeenCalledOnce();
|
||||
expect(subagent.waitForRun).not.toHaveBeenCalled();
|
||||
expect(subagent.deleteSession).not.toHaveBeenCalled();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining("narrative generation failed"),
|
||||
);
|
||||
});
|
||||
|
||||
it("skips narrative when no snippets are available", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("Should not appear.");
|
||||
|
||||
@@ -145,6 +145,56 @@ function buildRequestScopedFallbackNarrative(data: NarrativePhaseData): string {
|
||||
);
|
||||
}
|
||||
|
||||
function buildNarrativeAttemptSessionKey(baseSessionKey: string, attempt: number): string {
|
||||
return attempt === 0 ? baseSessionKey : `${baseSessionKey}-retry-${attempt}`;
|
||||
}
|
||||
|
||||
function isConfiguredModelUnavailableNarrativeError(raw: string): boolean {
|
||||
const message = raw.trim();
|
||||
if (!message) {
|
||||
return false;
|
||||
}
|
||||
if (/requested model may be(?: temporarily)? unavailable/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/model unavailable/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/no endpoints found for/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/unknown model/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/model(?:[_\-\s])?not(?:[_\-\s])?found/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/\b404\b/.test(message) && /not(?:[_\-\s])?found/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/not_found_error/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/models\/[^\s]+ is not found/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/model/i.test(message) && /does not exist/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/unsupported model/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
if (/is not a valid model id/i.test(message)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function formatNarrativeTerminalStatus(params: { status: string; error?: string }): string {
|
||||
const detail = params.error?.trim();
|
||||
return detail ? `status=${params.status} (${detail})` : `status=${params.status}`;
|
||||
}
|
||||
|
||||
async function startNarrativeRunOrFallback(params: {
|
||||
subagent: SubagentSurface;
|
||||
sessionKey: string;
|
||||
@@ -869,39 +919,81 @@ export async function generateAndAppendDreamNarrative(params: {
|
||||
nowMs,
|
||||
});
|
||||
const message = buildNarrativePrompt(params.data);
|
||||
let runId: string | null = null;
|
||||
let shouldDeleteSession = false;
|
||||
const attempts: Array<{ sessionKey: string; runId: string | null }> = [];
|
||||
let successfulSessionKey: string | null = null;
|
||||
try {
|
||||
runId = await startNarrativeRunOrFallback({
|
||||
subagent: params.subagent,
|
||||
sessionKey,
|
||||
message,
|
||||
data: params.data,
|
||||
workspaceDir: params.workspaceDir,
|
||||
nowMs,
|
||||
timezone: params.timezone,
|
||||
model: params.model,
|
||||
logger: params.logger,
|
||||
});
|
||||
if (!runId) {
|
||||
return;
|
||||
const attemptModels = params.model ? [params.model, undefined] : [undefined];
|
||||
|
||||
for (const [attemptIndex, attemptModel] of attemptModels.entries()) {
|
||||
const attemptSessionKey = buildNarrativeAttemptSessionKey(sessionKey, attemptIndex);
|
||||
const attempt = { sessionKey: attemptSessionKey, runId: null as string | null };
|
||||
attempts.push(attempt);
|
||||
|
||||
try {
|
||||
const runId = await startNarrativeRunOrFallback({
|
||||
subagent: params.subagent,
|
||||
sessionKey: attemptSessionKey,
|
||||
message,
|
||||
data: params.data,
|
||||
workspaceDir: params.workspaceDir,
|
||||
nowMs,
|
||||
timezone: params.timezone,
|
||||
model: attemptModel,
|
||||
logger: params.logger,
|
||||
});
|
||||
if (!runId) {
|
||||
return;
|
||||
}
|
||||
attempt.runId = runId;
|
||||
|
||||
const result = await params.subagent.waitForRun({
|
||||
runId,
|
||||
timeoutMs: NARRATIVE_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
if (result.status === "ok") {
|
||||
successfulSessionKey = attemptSessionKey;
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
attemptModel &&
|
||||
result.status === "error" &&
|
||||
isConfiguredModelUnavailableNarrativeError(result.error ?? "")
|
||||
) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation ended with ${formatNarrativeTerminalStatus({
|
||||
status: result.status,
|
||||
error: result.error,
|
||||
})} for ${params.data.phase} phase using configured model "${attemptModel}"; retrying with the session default.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation ended with ${formatNarrativeTerminalStatus({
|
||||
status: result.status,
|
||||
error: result.error,
|
||||
})} for ${params.data.phase} phase.`,
|
||||
);
|
||||
return;
|
||||
} catch (err) {
|
||||
if (attemptModel && isConfiguredModelUnavailableNarrativeError(formatErrorMessage(err))) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation could not start with configured model "${attemptModel}" for ${params.data.phase} phase; retrying with the session default (${formatErrorMessage(err)}).`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
shouldDeleteSession = true;
|
||||
|
||||
const result = await params.subagent.waitForRun({
|
||||
runId,
|
||||
timeoutMs: NARRATIVE_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
if (result.status !== "ok") {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation ended with status=${result.status} for ${params.data.phase} phase.`,
|
||||
);
|
||||
if (!successfulSessionKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { messages } = await params.subagent.getSessionMessages({
|
||||
sessionKey,
|
||||
sessionKey: successfulSessionKey,
|
||||
limit: 5,
|
||||
});
|
||||
|
||||
@@ -931,9 +1023,14 @@ export async function generateAndAppendDreamNarrative(params: {
|
||||
} finally {
|
||||
// Only cleanup after a run was accepted. Request-scoped fallback writes a
|
||||
// local diary entry without creating a subagent session.
|
||||
if (shouldDeleteSession && params.subagent) {
|
||||
const cleanedSessionKeys = new Set<string>();
|
||||
for (const attempt of attempts) {
|
||||
if (!attempt.runId || cleanedSessionKeys.has(attempt.sessionKey)) {
|
||||
continue;
|
||||
}
|
||||
cleanedSessionKeys.add(attempt.sessionKey);
|
||||
try {
|
||||
await params.subagent.deleteSession({ sessionKey });
|
||||
await params.subagent.deleteSession({ sessionKey: attempt.sessionKey });
|
||||
} catch (cleanupErr) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative session cleanup failed for ${params.data.phase} phase: ${formatErrorMessage(cleanupErr)}`,
|
||||
|
||||
Reference in New Issue
Block a user