From c0cba7fb72ea7490b89ab194041287bea4017f3e Mon Sep 17 00:00:00 2001 From: Julia Barth <72460857+Julbarth@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:34:46 -0700 Subject: [PATCH] Fix one-shot exit hangs by tearing down cached memory managers (#40389) Merged via squash. Prepared head SHA: 0e600e89cf10f5086ab9d93f445587373a54dcec Co-authored-by: Julbarth <72460857+Julbarth@users.noreply.github.com> Co-authored-by: frankekn <4488090+frankekn@users.noreply.github.com> Reviewed-by: @frankekn --- CHANGELOG.md | 1 + src/cli/run-main.exit.test.ts | 6 ++ src/cli/run-main.ts | 111 ++++++++++++--------- src/memory/index.ts | 6 +- src/memory/manager-runtime.ts | 2 +- src/memory/manager.get-concurrency.test.ts | 37 +++++++ src/memory/manager.ts | 16 +++ src/memory/search-manager.test.ts | 107 +++++++++++++------- src/memory/search-manager.ts | 16 +++ 9 files changed, 214 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f705ed77a3..ce8a07061ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Docs: https://docs.openclaw.ai - Logging/probe observations: suppress structured embedded and model-fallback probe warnings on the console without hiding error or fatal output. (#41338) thanks @altaywtf. - Agents/fallback: treat HTTP 499 responses as transient in both raw-text and structured failover paths so Anthropic-style client-closed overload responses trigger model fallback reliably. (#41468) thanks @zeroasterisk. - Plugins/context-engine model auth: expose `runtime.modelAuth` and plugin-sdk auth helpers so plugins can resolve provider/model API keys through the normal auth pipeline. (#41090) thanks @xinhuagu. +- CLI/memory teardown: close cached memory search/index managers in the one-shot CLI shutdown path so watcher-backed memory caches no longer keep completed CLI runs alive after output finishes. (#40389) thanks @Julbarth. ## 2026.3.8 diff --git a/src/cli/run-main.exit.test.ts b/src/cli/run-main.exit.test.ts index 86d74f09640..3e56c1ce794 100644 --- a/src/cli/run-main.exit.test.ts +++ b/src/cli/run-main.exit.test.ts @@ -6,6 +6,7 @@ const loadDotEnvMock = vi.hoisted(() => vi.fn()); const normalizeEnvMock = vi.hoisted(() => vi.fn()); const ensurePathMock = vi.hoisted(() => vi.fn()); const assertRuntimeMock = vi.hoisted(() => vi.fn()); +const closeAllMemorySearchManagersMock = vi.hoisted(() => vi.fn(async () => {})); vi.mock("./route.js", () => ({ tryRouteCli: tryRouteCliMock, @@ -27,6 +28,10 @@ vi.mock("../infra/runtime-guard.js", () => ({ assertSupportedRuntime: assertRuntimeMock, })); +vi.mock("../memory/search-manager.js", () => ({ + closeAllMemorySearchManagers: closeAllMemorySearchManagersMock, +})); + const { runCli } = await import("./run-main.js"); describe("runCli exit behavior", () => { @@ -43,6 +48,7 @@ describe("runCli exit behavior", () => { await runCli(["node", "openclaw", "status"]); expect(tryRouteCliMock).toHaveBeenCalledWith(["node", "openclaw", "status"]); + expect(closeAllMemorySearchManagersMock).toHaveBeenCalledTimes(1); expect(exitSpy).not.toHaveBeenCalled(); exitSpy.mockRestore(); }); diff --git a/src/cli/run-main.ts b/src/cli/run-main.ts index e80ce97b845..c0673ddf2af 100644 --- a/src/cli/run-main.ts +++ b/src/cli/run-main.ts @@ -13,6 +13,15 @@ import { applyCliProfileEnv, parseCliProfileArgs } from "./profile.js"; import { tryRouteCli } from "./route.js"; import { normalizeWindowsArgv } from "./windows-argv.js"; +async function closeCliMemoryManagers(): Promise { + try { + const { closeAllMemorySearchManagers } = await import("../memory/search-manager.js"); + await closeAllMemorySearchManagers(); + } catch { + // Best-effort teardown for short-lived CLI processes. + } +} + export function rewriteUpdateFlagArgv(argv: string[]): string[] { const index = argv.indexOf("--update"); if (index === -1) { @@ -82,59 +91,63 @@ export async function runCli(argv: string[] = process.argv) { // Enforce the minimum supported runtime before doing any work. assertSupportedRuntime(); - if (await tryRouteCli(normalizedArgv)) { - return; - } - - // Capture all console output into structured logs while keeping stdout/stderr behavior. - enableConsoleCapture(); - - const { buildProgram } = await import("./program.js"); - const program = buildProgram(); - - // Global error handlers to prevent silent crashes from unhandled rejections/exceptions. - // These log the error and exit gracefully instead of crashing without trace. - installUnhandledRejectionHandler(); - - process.on("uncaughtException", (error) => { - console.error("[openclaw] Uncaught exception:", formatUncaughtError(error)); - process.exit(1); - }); - - const parseArgv = rewriteUpdateFlagArgv(normalizedArgv); - // Register the primary command (builtin or subcli) so help and command parsing - // are correct even with lazy command registration. - const primary = getPrimaryCommand(parseArgv); - if (primary) { - const { getProgramContext } = await import("./program/program-context.js"); - const ctx = getProgramContext(program); - if (ctx) { - const { registerCoreCliByName } = await import("./program/command-registry.js"); - await registerCoreCliByName(program, ctx, primary, parseArgv); + try { + if (await tryRouteCli(normalizedArgv)) { + return; } - const { registerSubCliByName } = await import("./program/register.subclis.js"); - await registerSubCliByName(program, primary); - } - const hasBuiltinPrimary = - primary !== null && program.commands.some((command) => command.name() === primary); - const shouldSkipPluginRegistration = shouldSkipPluginCommandRegistration({ - argv: parseArgv, - primary, - hasBuiltinPrimary, - }); - if (!shouldSkipPluginRegistration) { - // Register plugin CLI commands before parsing - const { registerPluginCliCommands } = await import("../plugins/cli.js"); - const { loadValidatedConfigForPluginRegistration } = - await import("./program/register.subclis.js"); - const config = await loadValidatedConfigForPluginRegistration(); - if (config) { - registerPluginCliCommands(program, config); + // Capture all console output into structured logs while keeping stdout/stderr behavior. + enableConsoleCapture(); + + const { buildProgram } = await import("./program.js"); + const program = buildProgram(); + + // Global error handlers to prevent silent crashes from unhandled rejections/exceptions. + // These log the error and exit gracefully instead of crashing without trace. + installUnhandledRejectionHandler(); + + process.on("uncaughtException", (error) => { + console.error("[openclaw] Uncaught exception:", formatUncaughtError(error)); + process.exit(1); + }); + + const parseArgv = rewriteUpdateFlagArgv(normalizedArgv); + // Register the primary command (builtin or subcli) so help and command parsing + // are correct even with lazy command registration. + const primary = getPrimaryCommand(parseArgv); + if (primary) { + const { getProgramContext } = await import("./program/program-context.js"); + const ctx = getProgramContext(program); + if (ctx) { + const { registerCoreCliByName } = await import("./program/command-registry.js"); + await registerCoreCliByName(program, ctx, primary, parseArgv); + } + const { registerSubCliByName } = await import("./program/register.subclis.js"); + await registerSubCliByName(program, primary); } - } - await program.parseAsync(parseArgv); + const hasBuiltinPrimary = + primary !== null && program.commands.some((command) => command.name() === primary); + const shouldSkipPluginRegistration = shouldSkipPluginCommandRegistration({ + argv: parseArgv, + primary, + hasBuiltinPrimary, + }); + if (!shouldSkipPluginRegistration) { + // Register plugin CLI commands before parsing + const { registerPluginCliCommands } = await import("../plugins/cli.js"); + const { loadValidatedConfigForPluginRegistration } = + await import("./program/register.subclis.js"); + const config = await loadValidatedConfigForPluginRegistration(); + if (config) { + registerPluginCliCommands(program, config); + } + } + + await program.parseAsync(parseArgv); + } finally { + await closeCliMemoryManagers(); + } } export function isCliMainModule(): boolean { diff --git a/src/memory/index.ts b/src/memory/index.ts index 4d2df05a399..86ca52e1d27 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -4,4 +4,8 @@ export type { MemorySearchManager, MemorySearchResult, } from "./types.js"; -export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js"; +export { + closeAllMemorySearchManagers, + getMemorySearchManager, + type MemorySearchManagerResult, +} from "./search-manager.js"; diff --git a/src/memory/manager-runtime.ts b/src/memory/manager-runtime.ts index b46b3708a6e..3e910b5676a 100644 --- a/src/memory/manager-runtime.ts +++ b/src/memory/manager-runtime.ts @@ -1 +1 @@ -export { MemoryIndexManager } from "./manager.js"; +export { closeAllMemoryIndexManagers, MemoryIndexManager } from "./manager.js"; diff --git a/src/memory/manager.get-concurrency.test.ts b/src/memory/manager.get-concurrency.test.ts index e7d040217a8..67b10768fc3 100644 --- a/src/memory/manager.get-concurrency.test.ts +++ b/src/memory/manager.get-concurrency.test.ts @@ -4,6 +4,10 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { getMemorySearchManager, type MemoryIndexManager } from "./index.js"; +import { + closeAllMemoryIndexManagers, + MemoryIndexManager as RawMemoryIndexManager, +} from "./manager.js"; import "./test-runtime-mocks.js"; const hoisted = vi.hoisted(() => ({ @@ -78,4 +82,37 @@ describe("memory manager cache hydration", () => { await managers[0].close(); }); + + it("drains in-flight manager creation during global teardown", async () => { + const indexPath = path.join(workspaceDir, "index.sqlite"); + const cfg = { + agents: { + defaults: { + workspace: workspaceDir, + memorySearch: { + provider: "openai", + model: "mock-embed", + store: { path: indexPath, vector: { enabled: false } }, + sync: { watch: false, onSessionStart: false, onSearch: false }, + }, + }, + list: [{ id: "main", default: true }], + }, + } as OpenClawConfig; + + hoisted.providerDelayMs = 100; + + const pendingResult = RawMemoryIndexManager.get({ cfg, agentId: "main" }); + await closeAllMemoryIndexManagers(); + const firstManager = await pendingResult; + + const secondManager = await RawMemoryIndexManager.get({ cfg, agentId: "main" }); + + expect(firstManager).toBeTruthy(); + expect(secondManager).toBeTruthy(); + expect(Object.is(secondManager, firstManager)).toBe(false); + expect(hoisted.providerCreateCalls).toBe(2); + + await secondManager?.close?.(); + }); }); diff --git a/src/memory/manager.ts b/src/memory/manager.ts index 1d2fb49e88b..9b1ff74e54c 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -42,6 +42,22 @@ const log = createSubsystemLogger("memory"); const INDEX_CACHE = new Map(); const INDEX_CACHE_PENDING = new Map>(); +export async function closeAllMemoryIndexManagers(): Promise { + const pending = Array.from(INDEX_CACHE_PENDING.values()); + if (pending.length > 0) { + await Promise.allSettled(pending); + } + const managers = Array.from(INDEX_CACHE.values()); + INDEX_CACHE.clear(); + for (const manager of managers) { + try { + await manager.close(); + } catch (err) { + log.warn(`failed to close memory index manager: ${String(err)}`); + } + } +} + export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements MemorySearchManager { private readonly cacheKey: string; protected readonly cfg: OpenClawConfig; diff --git a/src/memory/search-manager.test.ts b/src/memory/search-manager.test.ts index d853f5af1fa..1f705aeddcf 100644 --- a/src/memory/search-manager.test.ts +++ b/src/memory/search-manager.test.ts @@ -29,53 +29,53 @@ function createManagerStatus(params: { }; } -const qmdManagerStatus = createManagerStatus({ - backend: "qmd", - provider: "qmd", - model: "qmd", - requestedProvider: "qmd", - withMemorySourceCounts: true, -}); - -const fallbackManagerStatus = createManagerStatus({ - backend: "builtin", - provider: "openai", - model: "text-embedding-3-small", - requestedProvider: "openai", -}); - -const mockPrimary = { +const mockPrimary = vi.hoisted(() => ({ search: vi.fn(async () => []), readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })), - status: vi.fn(() => qmdManagerStatus), + status: vi.fn(() => + createManagerStatus({ + backend: "qmd", + provider: "qmd", + model: "qmd", + requestedProvider: "qmd", + withMemorySourceCounts: true, + }), + ), sync: vi.fn(async () => {}), probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })), probeVectorAvailability: vi.fn(async () => true), close: vi.fn(async () => {}), -}; +})); -const fallbackSearch = vi.fn(async () => [ - { - path: "MEMORY.md", - startLine: 1, - endLine: 1, - score: 1, - snippet: "fallback", - source: "memory" as const, - }, -]); - -const fallbackManager = { - search: fallbackSearch, +const fallbackManager = vi.hoisted(() => ({ + search: vi.fn(async () => [ + { + path: "MEMORY.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "fallback", + source: "memory" as const, + }, + ]), readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })), - status: vi.fn(() => fallbackManagerStatus), + status: vi.fn(() => + createManagerStatus({ + backend: "builtin", + provider: "openai", + model: "text-embedding-3-small", + requestedProvider: "openai", + }), + ), sync: vi.fn(async () => {}), probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })), probeVectorAvailability: vi.fn(async () => true), close: vi.fn(async () => {}), -}; +})); -const mockMemoryIndexGet = vi.fn(async () => fallbackManager); +const fallbackSearch = fallbackManager.search; +const mockMemoryIndexGet = vi.hoisted(() => vi.fn(async () => fallbackManager)); +const mockCloseAllMemoryIndexManagers = vi.hoisted(() => vi.fn(async () => {})); vi.mock("./qmd-manager.js", () => ({ QmdMemoryManager: { @@ -83,14 +83,15 @@ vi.mock("./qmd-manager.js", () => ({ }, })); -vi.mock("./manager.js", () => ({ +vi.mock("./manager-runtime.js", () => ({ MemoryIndexManager: { get: mockMemoryIndexGet, }, + closeAllMemoryIndexManagers: mockCloseAllMemoryIndexManagers, })); import { QmdMemoryManager } from "./qmd-manager.js"; -import { getMemorySearchManager } from "./search-manager.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./search-manager.js"; // eslint-disable-next-line @typescript-eslint/unbound-method -- mocked static function const createQmdManagerMock = vi.mocked(QmdMemoryManager.create); @@ -119,7 +120,8 @@ async function createFailedQmdSearchHarness(params: { agentId: string; errorMess return { cfg, manager: requireManager(first), firstResult: first }; } -beforeEach(() => { +beforeEach(async () => { + await closeAllMemorySearchManagers(); mockPrimary.search.mockClear(); mockPrimary.readFile.mockClear(); mockPrimary.status.mockClear(); @@ -134,6 +136,7 @@ beforeEach(() => { fallbackManager.probeEmbeddingAvailability.mockClear(); fallbackManager.probeVectorAvailability.mockClear(); fallbackManager.close.mockClear(); + mockCloseAllMemoryIndexManagers.mockClear(); mockMemoryIndexGet.mockClear(); mockMemoryIndexGet.mockResolvedValue(fallbackManager); createQmdManagerMock.mockClear(); @@ -243,4 +246,34 @@ describe("getMemorySearchManager caching", () => { await expect(firstManager.search("hello")).rejects.toThrow("qmd query failed"); }); + + it("closes cached managers on global teardown", async () => { + const cfg = createQmdCfg("teardown-agent"); + const first = await getMemorySearchManager({ cfg, agentId: "teardown-agent" }); + const firstManager = requireManager(first); + + await closeAllMemorySearchManagers(); + + expect(mockPrimary.close).toHaveBeenCalledTimes(1); + expect(mockCloseAllMemoryIndexManagers).toHaveBeenCalledTimes(1); + + const second = await getMemorySearchManager({ cfg, agentId: "teardown-agent" }); + expect(second.manager).toBeTruthy(); + expect(second.manager).not.toBe(firstManager); + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(createQmdManagerMock).toHaveBeenCalledTimes(2); + }); + + it("closes builtin index managers on teardown after runtime is loaded", async () => { + const retryAgentId = "teardown-with-fallback"; + const { manager } = await createFailedQmdSearchHarness({ + agentId: retryAgentId, + errorMessage: "qmd query failed", + }); + await manager.search("hello"); + + await closeAllMemorySearchManagers(); + + expect(mockCloseAllMemoryIndexManagers).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index f4e351fdc1a..ea581b5d6da 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -85,6 +85,22 @@ export async function getMemorySearchManager(params: { } } +export async function closeAllMemorySearchManagers(): Promise { + const managers = Array.from(QMD_MANAGER_CACHE.values()); + QMD_MANAGER_CACHE.clear(); + for (const manager of managers) { + try { + await manager.close?.(); + } catch (err) { + log.warn(`failed to close qmd memory manager: ${String(err)}`); + } + } + if (managerRuntimePromise !== null) { + const { closeAllMemoryIndexManagers } = await loadManagerRuntime(); + await closeAllMemoryIndexManagers(); + } +} + class FallbackMemoryManager implements MemorySearchManager { private fallback: MemorySearchManager | null = null; private primaryFailed = false;