Fix one-shot exit hangs by tearing down cached memory managers (#40389)

Merged via squash.

Prepared head SHA: 0e600e89cf
Co-authored-by: Julbarth <72460857+Julbarth@users.noreply.github.com>
Co-authored-by: frankekn <4488090+frankekn@users.noreply.github.com>
Reviewed-by: @frankekn

CHANGELOG.md

@@ -40,6 +40,7 @@ Docs: https://docs.openclaw.ai
 - Logging/probe observations: suppress structured embedded and model-fallback probe warnings on the console without hiding error or fatal output. (#41338) thanks @altaywtf.
 - Agents/fallback: treat HTTP 499 responses as transient in both raw-text and structured failover paths so Anthropic-style client-closed overload responses trigger model fallback reliably. (#41468) thanks @zeroasterisk.
 - Plugins/context-engine model auth: expose `runtime.modelAuth` and plugin-sdk auth helpers so plugins can resolve provider/model API keys through the normal auth pipeline. (#41090) thanks @xinhuagu.
+- CLI/memory teardown: close cached memory search/index managers in the one-shot CLI shutdown path so watcher-backed memory caches no longer keep completed CLI runs alive after output finishes. (#40389) thanks @Julbarth.
 
 ## 2026.3.8
 

src/cli/run-main.exit.test.ts

@@ -6,6 +6,7 @@ const loadDotEnvMock = vi.hoisted(() => vi.fn());
 const normalizeEnvMock = vi.hoisted(() => vi.fn());
 const ensurePathMock = vi.hoisted(() => vi.fn());
 const assertRuntimeMock = vi.hoisted(() => vi.fn());
+const closeAllMemorySearchManagersMock = vi.hoisted(() => vi.fn(async () => {}));
 
 vi.mock("./route.js", () => ({
   tryRouteCli: tryRouteCliMock,
@@ -27,6 +28,10 @@ vi.mock("../infra/runtime-guard.js", () => ({
   assertSupportedRuntime: assertRuntimeMock,
 }));
 
+vi.mock("../memory/search-manager.js", () => ({
+  closeAllMemorySearchManagers: closeAllMemorySearchManagersMock,
+}));
+
 const { runCli } = await import("./run-main.js");
 
 describe("runCli exit behavior", () => {
@@ -43,6 +48,7 @@ describe("runCli exit behavior", () => {
     await runCli(["node", "openclaw", "status"]);
 
     expect(tryRouteCliMock).toHaveBeenCalledWith(["node", "openclaw", "status"]);
+    expect(closeAllMemorySearchManagersMock).toHaveBeenCalledTimes(1);
     expect(exitSpy).not.toHaveBeenCalled();
     exitSpy.mockRestore();
   });

src/cli/run-main.ts

@@ -13,6 +13,15 @@ import { applyCliProfileEnv, parseCliProfileArgs } from "./profile.js";
 import { tryRouteCli } from "./route.js";
 import { normalizeWindowsArgv } from "./windows-argv.js";
 
+async function closeCliMemoryManagers(): Promise<void> {
+  try {
+    const { closeAllMemorySearchManagers } = await import("../memory/search-manager.js");
+    await closeAllMemorySearchManagers();
+  } catch {
+    // Best-effort teardown for short-lived CLI processes.
+  }
+}
+
 export function rewriteUpdateFlagArgv(argv: string[]): string[] {
   const index = argv.indexOf("--update");
   if (index === -1) {
@@ -82,59 +91,63 @@ export async function runCli(argv: string[] = process.argv) {
   // Enforce the minimum supported runtime before doing any work.
   assertSupportedRuntime();
 
-  if (await tryRouteCli(normalizedArgv)) {
-    return;
-  }
-
-  // Capture all console output into structured logs while keeping stdout/stderr behavior.
-  enableConsoleCapture();
-
-  const { buildProgram } = await import("./program.js");
-  const program = buildProgram();
-
-  // Global error handlers to prevent silent crashes from unhandled rejections/exceptions.
-  // These log the error and exit gracefully instead of crashing without trace.
-  installUnhandledRejectionHandler();
-
-  process.on("uncaughtException", (error) => {
-    console.error("[openclaw] Uncaught exception:", formatUncaughtError(error));
-    process.exit(1);
-  });
-
-  const parseArgv = rewriteUpdateFlagArgv(normalizedArgv);
-  // Register the primary command (builtin or subcli) so help and command parsing
-  // are correct even with lazy command registration.
-  const primary = getPrimaryCommand(parseArgv);
-  if (primary) {
-    const { getProgramContext } = await import("./program/program-context.js");
-    const ctx = getProgramContext(program);
-    if (ctx) {
-      const { registerCoreCliByName } = await import("./program/command-registry.js");
-      await registerCoreCliByName(program, ctx, primary, parseArgv);
+  try {
+    if (await tryRouteCli(normalizedArgv)) {
+      return;
     }
-    const { registerSubCliByName } = await import("./program/register.subclis.js");
-    await registerSubCliByName(program, primary);
-  }
 
-  const hasBuiltinPrimary =
-    primary !== null && program.commands.some((command) => command.name() === primary);
-  const shouldSkipPluginRegistration = shouldSkipPluginCommandRegistration({
-    argv: parseArgv,
-    primary,
-    hasBuiltinPrimary,
-  });
-  if (!shouldSkipPluginRegistration) {
-    // Register plugin CLI commands before parsing
-    const { registerPluginCliCommands } = await import("../plugins/cli.js");
-    const { loadValidatedConfigForPluginRegistration } =
-      await import("./program/register.subclis.js");
-    const config = await loadValidatedConfigForPluginRegistration();
-    if (config) {
-      registerPluginCliCommands(program, config);
+    // Capture all console output into structured logs while keeping stdout/stderr behavior.
+    enableConsoleCapture();
+
+    const { buildProgram } = await import("./program.js");
+    const program = buildProgram();
+
+    // Global error handlers to prevent silent crashes from unhandled rejections/exceptions.
+    // These log the error and exit gracefully instead of crashing without trace.
+    installUnhandledRejectionHandler();
+
+    process.on("uncaughtException", (error) => {
+      console.error("[openclaw] Uncaught exception:", formatUncaughtError(error));
+      process.exit(1);
+    });
+
+    const parseArgv = rewriteUpdateFlagArgv(normalizedArgv);
+    // Register the primary command (builtin or subcli) so help and command parsing
+    // are correct even with lazy command registration.
+    const primary = getPrimaryCommand(parseArgv);
+    if (primary) {
+      const { getProgramContext } = await import("./program/program-context.js");
+      const ctx = getProgramContext(program);
+      if (ctx) {
+        const { registerCoreCliByName } = await import("./program/command-registry.js");
+        await registerCoreCliByName(program, ctx, primary, parseArgv);
+      }
+      const { registerSubCliByName } = await import("./program/register.subclis.js");
+      await registerSubCliByName(program, primary);
     }
-  }
 
-  await program.parseAsync(parseArgv);
+    const hasBuiltinPrimary =
+      primary !== null && program.commands.some((command) => command.name() === primary);
+    const shouldSkipPluginRegistration = shouldSkipPluginCommandRegistration({
+      argv: parseArgv,
+      primary,
+      hasBuiltinPrimary,
+    });
+    if (!shouldSkipPluginRegistration) {
+      // Register plugin CLI commands before parsing
+      const { registerPluginCliCommands } = await import("../plugins/cli.js");
+      const { loadValidatedConfigForPluginRegistration } =
+        await import("./program/register.subclis.js");
+      const config = await loadValidatedConfigForPluginRegistration();
+      if (config) {
+        registerPluginCliCommands(program, config);
+      }
+    }
+
+    await program.parseAsync(parseArgv);
+  } finally {
+    await closeCliMemoryManagers();
+  }
 }
 
 export function isCliMainModule(): boolean {

src/memory/index.ts

@@ -4,4 +4,8 @@ export type {
   MemorySearchManager,
   MemorySearchResult,
 } from "./types.js";
-export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js";
+export {
+  closeAllMemorySearchManagers,
+  getMemorySearchManager,
+  type MemorySearchManagerResult,
+} from "./search-manager.js";

src/memory/manager-runtime.ts

@@ -1 +1 @@
-export { MemoryIndexManager } from "./manager.js";
+export { closeAllMemoryIndexManagers, MemoryIndexManager } from "./manager.js";

src/memory/manager.get-concurrency.test.ts

@@ -4,6 +4,10 @@ import path from "node:path";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import type { OpenClawConfig } from "../config/config.js";
 import { getMemorySearchManager, type MemoryIndexManager } from "./index.js";
+import {
+  closeAllMemoryIndexManagers,
+  MemoryIndexManager as RawMemoryIndexManager,
+} from "./manager.js";
 import "./test-runtime-mocks.js";
 
 const hoisted = vi.hoisted(() => ({
@@ -78,4 +82,37 @@ describe("memory manager cache hydration", () => {
 
     await managers[0].close();
   });
+
+  it("drains in-flight manager creation during global teardown", async () => {
+    const indexPath = path.join(workspaceDir, "index.sqlite");
+    const cfg = {
+      agents: {
+        defaults: {
+          workspace: workspaceDir,
+          memorySearch: {
+            provider: "openai",
+            model: "mock-embed",
+            store: { path: indexPath, vector: { enabled: false } },
+            sync: { watch: false, onSessionStart: false, onSearch: false },
+          },
+        },
+        list: [{ id: "main", default: true }],
+      },
+    } as OpenClawConfig;
+
+    hoisted.providerDelayMs = 100;
+
+    const pendingResult = RawMemoryIndexManager.get({ cfg, agentId: "main" });
+    await closeAllMemoryIndexManagers();
+    const firstManager = await pendingResult;
+
+    const secondManager = await RawMemoryIndexManager.get({ cfg, agentId: "main" });
+
+    expect(firstManager).toBeTruthy();
+    expect(secondManager).toBeTruthy();
+    expect(Object.is(secondManager, firstManager)).toBe(false);
+    expect(hoisted.providerCreateCalls).toBe(2);
+
+    await secondManager?.close?.();
+  });
 });

src/memory/manager.ts

@@ -42,6 +42,22 @@ const log = createSubsystemLogger("memory");
 const INDEX_CACHE = new Map<string, MemoryIndexManager>();
 const INDEX_CACHE_PENDING = new Map<string, Promise<MemoryIndexManager>>();
 
+export async function closeAllMemoryIndexManagers(): Promise<void> {
+  const pending = Array.from(INDEX_CACHE_PENDING.values());
+  if (pending.length > 0) {
+    await Promise.allSettled(pending);
+  }
+  const managers = Array.from(INDEX_CACHE.values());
+  INDEX_CACHE.clear();
+  for (const manager of managers) {
+    try {
+      await manager.close();
+    } catch (err) {
+      log.warn(`failed to close memory index manager: ${String(err)}`);
+    }
+  }
+}
+
 export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements MemorySearchManager {
   private readonly cacheKey: string;
   protected readonly cfg: OpenClawConfig;

src/memory/search-manager.test.ts

@@ -29,53 +29,53 @@ function createManagerStatus(params: {
   };
 }
 
-const qmdManagerStatus = createManagerStatus({
-  backend: "qmd",
-  provider: "qmd",
-  model: "qmd",
-  requestedProvider: "qmd",
-  withMemorySourceCounts: true,
-});
-
-const fallbackManagerStatus = createManagerStatus({
-  backend: "builtin",
-  provider: "openai",
-  model: "text-embedding-3-small",
-  requestedProvider: "openai",
-});
-
-const mockPrimary = {
+const mockPrimary = vi.hoisted(() => ({
   search: vi.fn(async () => []),
   readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })),
-  status: vi.fn(() => qmdManagerStatus),
+  status: vi.fn(() =>
+    createManagerStatus({
+      backend: "qmd",
+      provider: "qmd",
+      model: "qmd",
+      requestedProvider: "qmd",
+      withMemorySourceCounts: true,
+    }),
+  ),
   sync: vi.fn(async () => {}),
   probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })),
   probeVectorAvailability: vi.fn(async () => true),
   close: vi.fn(async () => {}),
-};
+}));
 
-const fallbackSearch = vi.fn(async () => [
-  {
-    path: "MEMORY.md",
-    startLine: 1,
-    endLine: 1,
-    score: 1,
-    snippet: "fallback",
-    source: "memory" as const,
-  },
-]);
-
-const fallbackManager = {
-  search: fallbackSearch,
+const fallbackManager = vi.hoisted(() => ({
+  search: vi.fn(async () => [
+    {
+      path: "MEMORY.md",
+      startLine: 1,
+      endLine: 1,
+      score: 1,
+      snippet: "fallback",
+      source: "memory" as const,
+    },
+  ]),
   readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })),
-  status: vi.fn(() => fallbackManagerStatus),
+  status: vi.fn(() =>
+    createManagerStatus({
+      backend: "builtin",
+      provider: "openai",
+      model: "text-embedding-3-small",
+      requestedProvider: "openai",
+    }),
+  ),
   sync: vi.fn(async () => {}),
   probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })),
   probeVectorAvailability: vi.fn(async () => true),
   close: vi.fn(async () => {}),
-};
+}));
 
-const mockMemoryIndexGet = vi.fn(async () => fallbackManager);
+const fallbackSearch = fallbackManager.search;
+const mockMemoryIndexGet = vi.hoisted(() => vi.fn(async () => fallbackManager));
+const mockCloseAllMemoryIndexManagers = vi.hoisted(() => vi.fn(async () => {}));
 
 vi.mock("./qmd-manager.js", () => ({
   QmdMemoryManager: {
@@ -83,14 +83,15 @@ vi.mock("./qmd-manager.js", () => ({
   },
 }));
 
-vi.mock("./manager.js", () => ({
+vi.mock("./manager-runtime.js", () => ({
   MemoryIndexManager: {
     get: mockMemoryIndexGet,
   },
+  closeAllMemoryIndexManagers: mockCloseAllMemoryIndexManagers,
 }));
 
 import { QmdMemoryManager } from "./qmd-manager.js";
-import { getMemorySearchManager } from "./search-manager.js";
+import { closeAllMemorySearchManagers, getMemorySearchManager } from "./search-manager.js";
 // eslint-disable-next-line @typescript-eslint/unbound-method -- mocked static function
 const createQmdManagerMock = vi.mocked(QmdMemoryManager.create);
 
@@ -119,7 +120,8 @@ async function createFailedQmdSearchHarness(params: { agentId: string; errorMess
   return { cfg, manager: requireManager(first), firstResult: first };
 }
 
-beforeEach(() => {
+beforeEach(async () => {
+  await closeAllMemorySearchManagers();
   mockPrimary.search.mockClear();
   mockPrimary.readFile.mockClear();
   mockPrimary.status.mockClear();
@@ -134,6 +136,7 @@ beforeEach(() => {
   fallbackManager.probeEmbeddingAvailability.mockClear();
   fallbackManager.probeVectorAvailability.mockClear();
   fallbackManager.close.mockClear();
+  mockCloseAllMemoryIndexManagers.mockClear();
   mockMemoryIndexGet.mockClear();
   mockMemoryIndexGet.mockResolvedValue(fallbackManager);
   createQmdManagerMock.mockClear();
@@ -243,4 +246,34 @@ describe("getMemorySearchManager caching", () => {
 
     await expect(firstManager.search("hello")).rejects.toThrow("qmd query failed");
   });
+
+  it("closes cached managers on global teardown", async () => {
+    const cfg = createQmdCfg("teardown-agent");
+    const first = await getMemorySearchManager({ cfg, agentId: "teardown-agent" });
+    const firstManager = requireManager(first);
+
+    await closeAllMemorySearchManagers();
+
+    expect(mockPrimary.close).toHaveBeenCalledTimes(1);
+    expect(mockCloseAllMemoryIndexManagers).toHaveBeenCalledTimes(1);
+
+    const second = await getMemorySearchManager({ cfg, agentId: "teardown-agent" });
+    expect(second.manager).toBeTruthy();
+    expect(second.manager).not.toBe(firstManager);
+    // eslint-disable-next-line @typescript-eslint/unbound-method
+    expect(createQmdManagerMock).toHaveBeenCalledTimes(2);
+  });
+
+  it("closes builtin index managers on teardown after runtime is loaded", async () => {
+    const retryAgentId = "teardown-with-fallback";
+    const { manager } = await createFailedQmdSearchHarness({
+      agentId: retryAgentId,
+      errorMessage: "qmd query failed",
+    });
+    await manager.search("hello");
+
+    await closeAllMemorySearchManagers();
+
+    expect(mockCloseAllMemoryIndexManagers).toHaveBeenCalledTimes(1);
+  });
 });

src/memory/search-manager.ts

@@ -85,6 +85,22 @@ export async function getMemorySearchManager(params: {
   }
 }
 
+export async function closeAllMemorySearchManagers(): Promise<void> {
+  const managers = Array.from(QMD_MANAGER_CACHE.values());
+  QMD_MANAGER_CACHE.clear();
+  for (const manager of managers) {
+    try {
+      await manager.close?.();
+    } catch (err) {
+      log.warn(`failed to close qmd memory manager: ${String(err)}`);
+    }
+  }
+  if (managerRuntimePromise !== null) {
+    const { closeAllMemoryIndexManagers } = await loadManagerRuntime();
+    await closeAllMemoryIndexManagers();
+  }
+}
+
 class FallbackMemoryManager implements MemorySearchManager {
   private fallback: MemorySearchManager | null = null;
   private primaryFailed = false;