From ae82a39150f9e02668ea34ca5955ccfc9bd67b70 Mon Sep 17 00:00:00 2001 From: "clawsweeper[bot]" <274271284+clawsweeper[bot]@users.noreply.github.com> Date: Sat, 2 May 2026 22:23:02 +0000 Subject: [PATCH] fix(active-memory): fast-fail stalled recall paths (#76183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: - This PR adds Active Memory transcript polling to fast-fail terminal zero-hit or unavailable recall tool results, filters timeout boilerplate, extends focused regressions, and adds a changelog fix entry. - Reproducibility: yes. The PR includes focused regressions that reproduce terminal zero-hit search, unavailab ... rch, non-empty `details.results` with `debug.hits: 0`, memory_get misses, and timeout boilerplate behavior. ClawSweeper fixups: - Included follow-up commit: fix(active-memory): fast-fail stalled recall paths - Included follow-up commit: fix(clawsweeper): address review for automerge-openclaw-openclaw-7576… - Included follow-up commit: fix(clawsweeper): reconcile automerge-openclaw-openclaw-75761 with ma… - Ran the ClawSweeper repair loop before final review. Validation: - ClawSweeper review passed for head e5ea3f1a7a4e84f55422c426b6471179d23b2c53. - Required merge gates passed before the squash merge. Prepared head SHA: e5ea3f1a7a4e84f55422c426b6471179d23b2c53 Review: https://github.com/openclaw/openclaw/pull/76183#issuecomment-4364369591 Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: codexGW <9350182+codexGW@users.noreply.github.com> --- CHANGELOG.md | 1 + extensions/active-memory/index.test.ts | 209 +++++++++++++++++++++++++ extensions/active-memory/index.ts | 196 ++++++++++++++++++++++- 3 files changed, 403 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edf34f856ec..635f5d4384a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Docs: https://docs.openclaw.ai - CLI/plugins: stop treating the non-plugin `auth` command root as a bundled plugin id, so restrictive `plugins.allow` configs no longer tell users to add stale `auth` plugin entries. - Doctor/plugins: update configured plugin installs whose stale manifests still declare channels without `channelConfigs`, so beta upgrades repair old Discord-style package payloads during `doctor --fix`. +- Active Memory: keep non-empty `memory_search` results from being fast-failed as empty when debug telemetry reports zero hits. - Plugins/externalization: repair missing configured plugin installs from npm by default, reserve ClawHub downloads for explicit `clawhubSpec` metadata, and cover agent-runtime/env-selected plugin repair. Thanks @vincentkoc. - Upgrade/config: validate configured web-search providers and statically suppressed model/provider pairs against the active plugin set at config load, so stale plugin state fails loud before runtime fallback. - Status/update: resolve beta update-channel checks from the installed version when config still says `stable`, and let `status --deep` reuse live gateway channel credential state instead of warning on command-path-only token misses. diff --git a/extensions/active-memory/index.test.ts b/extensions/active-memory/index.test.ts index 88f0861d45b..a4cdea89ff8 100644 --- a/extensions/active-memory/index.test.ts +++ b/extensions/active-memory/index.test.ts @@ -1789,6 +1789,50 @@ describe("active-memory plugin", () => { expect(lines.some((line) => line.includes("timeout_partial"))).toBe(false); }); + it("does not inject embedded timeout boilerplate from partial transcripts", async () => { + __testing.setMinimumTimeoutMsForTests(1); + __testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:timeout-boilerplate-transcript"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-timeout-boilerplate-transcript", + updatedAt: 0, + }; + runEmbeddedPiAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + type: "message", + message: { + role: "assistant", + content: "LLM request timed out after 15000 ms.", + }, + }, + ]); + await new Promise(() => {}); + }); + + const result = await hooks.before_prompt_build( + { prompt: "what wings should i order? timeout boilerplate", messages: [] }, + { + agentId: "main", + trigger: "user", + sessionKey, + messageProvider: "webchat", + }, + ); + + expect(result).toBeUndefined(); + const lines = getActiveMemoryLines(sessionKey); + expect(lines).toEqual([expect.stringContaining("🧩 Active Memory: status=timeout")]); + expect(lines.some((line) => line.includes("timeout_partial"))).toBe(false); + expect(lines.some((line) => line.includes("LLM request timed out"))).toBe(false); + }); + it("returns partial transcript text when an aborted subagent rejects before the race timeout wins", async () => { __testing.setMinimumTimeoutMsForTests(1); api.pluginConfig = { @@ -2258,6 +2302,171 @@ describe("active-memory plugin", () => { expect(wallClockMs).toBeLessThan(CONFIGURED_TIMEOUT_MS + HARD_DEADLINE_MARGIN_MS); }); + it("fast-fails terminal zero-hit memory_search results without waiting for recall timeout", async () => { + const CONFIGURED_TIMEOUT_MS = 1_000; + __testing.setMinimumTimeoutMsForTests(1); + __testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: CONFIGURED_TIMEOUT_MS, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:terminal-zero-hit"; + hoisted.sessionStore[sessionKey] = { sessionId: "s-terminal-zero-hit", updatedAt: 0 }; + runEmbeddedPiAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { results: [], debug: { backend: "qmd", hits: 0, searchMs: 8 } }, + }, + }, + ]); + await new Promise(() => {}); + }); + + const startedAt = Date.now(); + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? zero hit", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + const wallClockMs = Date.now() - startedAt; + + expect(result).toBeUndefined(); + expect(wallClockMs).toBeLessThan(CONFIGURED_TIMEOUT_MS); + expect(getActiveMemoryLines(sessionKey)).toEqual([ + expect.stringContaining("🧩 Active Memory: status=empty"), + expect.stringContaining("🔎 Active Memory Debug: backend=qmd searchMs=8 hits=0"), + ]); + }); + + it("does not fast-fail memory_search results solely because debug hits is zero", async () => { + __testing.setMinimumTimeoutMsForTests(1); + __testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 500, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:terminal-zero-hit-with-results"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-terminal-zero-hit-with-results", + updatedAt: 0, + }; + runEmbeddedPiAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + results: [{ path: "memory/food.md", text: "User usually orders ramen." }], + debug: { backend: "qmd", hits: 0, searchMs: 8 }, + }, + }, + }, + ]); + await new Promise((resolve) => setTimeout(resolve, 50)); + return { payloads: [{ text: "User usually orders ramen." }] }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? zero hit with results", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result?.prependContext).toContain("User usually orders ramen."); + expect(getActiveMemoryLines(sessionKey)).toEqual([ + expect.stringContaining("🧩 Active Memory: status=ok"), + expect.stringContaining("🔎 Active Memory Debug: backend=qmd searchMs=8 hits=0"), + ]); + }); + + it("fast-fails unavailable memory_search results without injecting provider errors", async () => { + const CONFIGURED_TIMEOUT_MS = 1_000; + __testing.setMinimumTimeoutMsForTests(1); + __testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: CONFIGURED_TIMEOUT_MS, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:terminal-unavailable"; + hoisted.sessionStore[sessionKey] = { sessionId: "s-terminal-unavailable", updatedAt: 0 }; + runEmbeddedPiAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + disabled: true, + warning: "Memory search is unavailable due to an embedding/provider error.", + action: "Check the embedding provider configuration, then retry memory_search.", + error: "embedding request failed", + }, + }, + }, + ]); + await new Promise(() => {}); + }); + + const startedAt = Date.now(); + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? unavailable", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + const wallClockMs = Date.now() - startedAt; + + expect(result).toBeUndefined(); + expect(wallClockMs).toBeLessThan(CONFIGURED_TIMEOUT_MS); + expect(getActiveMemoryLines(sessionKey)).toEqual([ + expect.stringContaining("🧩 Active Memory: status=empty"), + expect.stringContaining( + "🔎 Active Memory Debug: Memory search is unavailable due to an embedding/provider error. Check the embedding provider configuration, then retry memory_search.", + ), + ]); + }); + + it("does not treat memory_get misses as terminal recall results", async () => { + __testing.setMinimumTimeoutMsForTests(1); + __testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 500, + }; + plugin.register(api as unknown as OpenClawPluginApi); + runEmbeddedPiAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_get", + details: { path: "memory/missing.md", text: "", disabled: true, error: "not found" }, + }, + }, + ]); + await new Promise((resolve) => setTimeout(resolve, 50)); + return { payloads: [{ text: "User usually orders ramen after late flights." }] }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? memory get miss", messages: [] }, + { + agentId: "main", + trigger: "user", + sessionKey: "agent:main:memory-get-miss", + messageProvider: "webchat", + }, + ); + + expect(result?.prependContext).toContain("User usually orders ramen after late flights."); + }); + it("returns undefined instead of throwing when an unexpected error escapes prompt building", async () => { const result = await hooks.before_prompt_build( { prompt: "what should i eat? escape test", messages: undefined as never }, diff --git a/extensions/active-memory/index.ts b/extensions/active-memory/index.ts index 3b494433856..825f8e6a82c 100644 --- a/extensions/active-memory/index.ts +++ b/extensions/active-memory/index.ts @@ -46,6 +46,7 @@ const DEFAULT_PARTIAL_TRANSCRIPT_MAX_CHARS = 32_000; const DEFAULT_TRANSCRIPT_READ_MAX_LINES = 2_000; const DEFAULT_TRANSCRIPT_READ_MAX_BYTES = 50 * 1024 * 1024; const TIMEOUT_PARTIAL_DATA_GRACE_MS = 50; +const TERMINAL_MEMORY_SEARCH_POLL_INTERVAL_MS = 25; const NO_RECALL_VALUES = new Set([ "", @@ -56,12 +57,21 @@ const NO_RECALL_VALUES = new Set([ "no relevant memory", "no relevant memories", "timeout", + "timed out", + "request timed out", + "llm request timed out", + "the llm request timed out", "[]", "{}", "null", "n/a", ]); +const TIMEOUT_BOILERPLATE_PATTERNS = [ + /^(?:error:\s*)?(?:the\s+)?(?:llm|model|request|operation|agent)\s+(?:request\s+)?timed out\b/i, + /^(?:error:\s*)?active-memory timeout after \d+ms\b/i, +]; + const RECALLED_CONTEXT_LINE_PATTERNS = [ /^🧩\s*active memory:/i, /^🔎\s*active memory debug:/i, @@ -209,6 +219,16 @@ type RecallSubagentResult = { searchDebug?: ActiveMemorySearchDebug; }; +type TerminalMemorySearchResult = { + status: "empty"; + searchDebug?: ActiveMemorySearchDebug; +}; + +type TerminalMemorySearchWatch = { + promise: Promise; + stop: () => void; +}; + type CachedActiveRecallResult = { expiresAt: number; result: ActiveRecallResult; @@ -1549,6 +1569,41 @@ function extractActiveMemorySearchDebugFromSessionRecord( }; } +function extractTerminalMemorySearchResultFromSessionRecord( + value: unknown, +): TerminalMemorySearchResult | undefined { + const record = asRecord(value); + const nestedMessage = asRecord(record?.message); + const topLevelMessage = + record?.role === "toolResult" || + record?.toolName === "memory_search" || + record?.toolName === "memory_recall" + ? record + : undefined; + const message = nestedMessage ?? topLevelMessage; + if (!message) { + return undefined; + } + const role = normalizeOptionalString(message.role); + const toolName = normalizeOptionalString(message.toolName); + if (role !== "toolResult" || (toolName !== "memory_search" && toolName !== "memory_recall")) { + return undefined; + } + const details = asRecord(message.details); + const debug = extractActiveMemorySearchDebugFromSessionRecord(value); + const results = Array.isArray(details?.results) ? details.results : undefined; + const disabled = details?.disabled === true; + const unavailable = + disabled || Boolean(debug?.warning) || Boolean(debug?.error) || Boolean(details?.error); + const debugHits = + typeof debug?.hits === "number" && Number.isFinite(debug.hits) ? debug.hits : undefined; + const zeroHitSearch = results !== undefined ? results.length === 0 : debugHits === 0; + if (unavailable || zeroHitSearch) { + return { status: "empty", searchDebug: debug }; + } + return undefined; +} + async function readActiveMemorySearchDebug( sessionFile: string, limits?: TranscriptReadLimits, @@ -1567,6 +1622,93 @@ async function readActiveMemorySearchDebug( return found; } +async function readTerminalMemorySearchResult( + sessionFile: string, + limits?: TranscriptReadLimits, +): Promise { + let found: TerminalMemorySearchResult | undefined; + await streamBoundedTranscriptJsonl({ + sessionFile, + limits, + onRecord: (record) => { + const result = extractTerminalMemorySearchResultFromSessionRecord(record); + if (result) { + found = result; + return true; + } + return false; + }, + }); + return found; +} + +function watchTerminalMemorySearchResult(params: { + getSessionFile: () => string | undefined; + abortSignal: AbortSignal; +}): TerminalMemorySearchWatch { + let stopped = false; + let timeoutId: ReturnType | undefined; + let inFlight = false; + let resolveWatch: (result: TerminalMemorySearchResult) => void = () => {}; + const stop = () => { + if (stopped) { + return; + } + stopped = true; + if (timeoutId) { + clearTimeout(timeoutId); + timeoutId = undefined; + } + params.abortSignal.removeEventListener("abort", onAbort); + }; + const finish = (result: TerminalMemorySearchResult) => { + stop(); + resolveWatch(result); + }; + const schedule = () => { + if (stopped) { + return; + } + timeoutId = setTimeout(tick, TERMINAL_MEMORY_SEARCH_POLL_INTERVAL_MS); + timeoutId.unref?.(); + }; + const tick = async () => { + if (stopped || inFlight) { + return; + } + if (params.abortSignal.aborted) { + stop(); + return; + } + inFlight = true; + try { + const sessionFile = params.getSessionFile(); + const result = sessionFile ? await readTerminalMemorySearchResult(sessionFile) : undefined; + if (result) { + finish(result); + return; + } + } catch { + // Transcript polling is opportunistic; normal timeout handling remains authoritative. + } finally { + inFlight = false; + } + schedule(); + }; + function onAbort() { + stop(); + } + const promise = new Promise((resolve) => { + resolveWatch = resolve; + params.abortSignal.addEventListener("abort", onAbort, { once: true }); + void tick(); + }); + return { + promise, + stop, + }; +} + function normalizeSearchDebug(value: unknown): ActiveMemorySearchDebug | undefined { const debug = asRecord(value); if (!debug) { @@ -1777,13 +1919,21 @@ function normalizeNoRecallValue(value: string): boolean { return NO_RECALL_VALUES.has(value.trim().toLowerCase()); } +function isTimeoutBoilerplateSummary(value: string): boolean { + return TIMEOUT_BOILERPLATE_PATTERNS.some((pattern) => pattern.test(value)); +} + function normalizeActiveSummary(rawReply: string): string | null { const trimmed = rawReply.trim(); if (normalizeNoRecallValue(trimmed)) { return null; } const singleLine = trimmed.replace(/\s+/g, " ").trim(); - if (!singleLine || normalizeNoRecallValue(singleLine)) { + if ( + !singleLine || + normalizeNoRecallValue(singleLine) || + isTimeoutBoilerplateSummary(singleLine) + ) { return null; } return singleLine; @@ -2299,6 +2449,7 @@ async function maybeResolveActiveRecall(params: { ); }); + let terminalMemorySearchWatch: TerminalMemorySearchWatch | undefined; try { const subagentPromise = runRecallSubagent({ ...params, @@ -2308,11 +2459,20 @@ async function maybeResolveActiveRecall(params: { sessionFile = value; }, }); + terminalMemorySearchWatch = watchTerminalMemorySearchResult({ + getSessionFile: () => sessionFile, + abortSignal: controller.signal, + }); // Silently catch late rejections after timeout so they don't become // unhandled promise rejections. subagentPromise.catch(() => undefined); - const raceResult = await Promise.race([subagentPromise, timeoutPromise]); + const raceResult = await Promise.race([ + subagentPromise, + timeoutPromise, + terminalMemorySearchWatch.promise, + ]); + terminalMemorySearchWatch.stop(); if (raceResult === TIMEOUT_SENTINEL) { const result = await buildTimeoutRecallResult({ @@ -2338,6 +2498,33 @@ async function maybeResolveActiveRecall(params: { return result; } + if ("status" in raceResult) { + controller.abort(new Error("active-memory terminal memory search result")); + const result: ActiveRecallResult = { + status: raceResult.status, + elapsedMs: Date.now() - startedAt, + summary: null, + searchDebug: raceResult.searchDebug, + }; + if (params.config.logging) { + params.api.logger.info?.( + `${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=0`, + ); + } + await persistPluginStatusLines({ + api: params.api, + agentId: params.agentId, + sessionKey: params.sessionKey, + statusLine: buildPluginStatusLine({ result, config: params.config }), + searchDebug: result.searchDebug, + }); + if (shouldCacheResult(result)) { + setCachedResult(cacheKey, result, params.config.cacheTtlMs); + } + resetCircuitBreaker(cbKey); + return result; + } + const { rawReply, transcriptPath, searchDebug } = raceResult; const summary = truncateSummary( normalizeActiveSummary(rawReply) ?? "", @@ -2423,6 +2610,7 @@ async function maybeResolveActiveRecall(params: { }); return result; } finally { + terminalMemorySearchWatch?.stop(); clearTimeout(timeoutId); } } @@ -2649,7 +2837,7 @@ export default definePluginEntry({ }, }); -export const __testing = { +const testing = { buildCacheKey, buildCircuitBreakerKey, buildMetadata, @@ -2679,3 +2867,5 @@ export const __testing = { return timeoutCircuitBreaker.get(key); }, }; + +export { testing as __testing };