diff --git a/src/gateway/server-startup-memory.test.ts b/src/gateway/server-startup-memory.test.ts
index bf1d040e6fb..21ea1896c36 100644
--- a/src/gateway/server-startup-memory.test.ts
+++ b/src/gateway/server-startup-memory.test.ts
@@ -5,13 +5,8 @@ const { getMemorySearchManagerMock } = vi.hoisted(() => ({
   getMemorySearchManagerMock: vi.fn(),
 }));
 
-const { resolveActiveMemoryBackendConfigMock } = vi.hoisted(() => ({
-  resolveActiveMemoryBackendConfigMock: vi.fn(),
-}));
-
 vi.mock("../plugins/memory-runtime.js", () => ({
   getActiveMemorySearchManager: getMemorySearchManagerMock,
-  resolveActiveMemoryBackendConfig: resolveActiveMemoryBackendConfigMock,
 }));
 
 import { startGatewayMemoryBackend } from "./server-startup-memory.js";
@@ -30,11 +25,6 @@ function createGatewayLogMock() {
 describe("startGatewayMemoryBackend", () => {
   beforeEach(() => {
     getMemorySearchManagerMock.mockClear();
-    resolveActiveMemoryBackendConfigMock.mockReset();
-    resolveActiveMemoryBackendConfigMock.mockImplementation(({ cfg }: { cfg: OpenClawConfig }) => ({
-      backend: cfg.memory?.backend === "qmd" ? "qmd" : "builtin",
-      qmd: cfg.memory?.backend === "qmd" ? {} : undefined,
-    }));
   });
 
   it("skips initialization when memory backend is not qmd", async () => {
@@ -51,8 +41,14 @@ describe("startGatewayMemoryBackend", () => {
     expect(log.warn).not.toHaveBeenCalled();
   });
 
-  it("initializes qmd backend for each configured agent", async () => {
-    const cfg = createQmdConfig({ list: [{ id: "ops", default: true }, { id: "main" }] });
+  it("initializes qmd backend for the default and explicitly configured agents", async () => {
+    const cfg = createQmdConfig({
+      list: [
+        { id: "ops", default: true },
+        { id: "main", memorySearch: { enabled: true } },
+        { id: "lazy" },
+      ],
+    });
     const log = createGatewayLogMock();
     getMemorySearchManagerMock.mockResolvedValue({ manager: { search: vi.fn() } });
 
@@ -61,15 +57,41 @@ describe("startGatewayMemoryBackend", () => {
     expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(2);
     expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(1, { cfg, agentId: "ops" });
     expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(2, { cfg, agentId: "main" });
-    expect(log.info).toHaveBeenCalledTimes(1);
     expect(log.info).toHaveBeenCalledWith(
       'qmd memory startup initialization armed for 2 agents: "ops", "main"',
     );
+    expect(log.info).toHaveBeenCalledWith(
+      'qmd memory startup initialization deferred for 1 agent: "lazy"',
+    );
     expect(log.warn).not.toHaveBeenCalled();
   });
 
+  it("initializes all qmd agents when memory search is explicitly enabled in defaults", async () => {
+    const cfg = createQmdConfig({
+      defaults: { memorySearch: { enabled: true } },
+      list: [{ id: "ops", default: true }, { id: "main" }],
+    });
+    const log = createGatewayLogMock();
+    getMemorySearchManagerMock.mockResolvedValue({ manager: { search: vi.fn() } });
+
+    await startGatewayMemoryBackend({ cfg, log });
+
+    expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(2);
+    expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(1, { cfg, agentId: "ops" });
+    expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(2, { cfg, agentId: "main" });
+    expect(log.info).toHaveBeenCalledWith(
+      'qmd memory startup initialization armed for 2 agents: "ops", "main"',
+    );
+    expect(log.info).not.toHaveBeenCalledWith(expect.stringContaining("deferred"));
+  });
+
   it("logs a warning when qmd manager init fails and continues with other agents", async () => {
-    const cfg = createQmdConfig({ list: [{ id: "main", default: true }, { id: "ops" }] });
+    const cfg = createQmdConfig({
+      list: [
+        { id: "main", default: true },
+        { id: "ops", memorySearch: { enabled: true } },
+      ],
+    });
     const log = createGatewayLogMock();
     getMemorySearchManagerMock
       .mockResolvedValueOnce({ manager: null, error: "qmd missing" })
@@ -105,4 +127,23 @@ describe("startGatewayMemoryBackend", () => {
     );
     expect(log.warn).not.toHaveBeenCalled();
   });
+
+  it("does not initialize qmd managers when background work is disabled", async () => {
+    const cfg = {
+      agents: { list: [{ id: "main", default: true }] },
+      memory: {
+        backend: "qmd",
+        qmd: {
+          update: { onBoot: false, interval: "0s", embedInterval: "0s" },
+        },
+      },
+    } as OpenClawConfig;
+    const log = createGatewayLogMock();
+
+    await startGatewayMemoryBackend({ cfg, log });
+
+    expect(getMemorySearchManagerMock).not.toHaveBeenCalled();
+    expect(log.info).not.toHaveBeenCalled();
+    expect(log.warn).not.toHaveBeenCalled();
+  });
 });
diff --git a/src/gateway/server-startup-memory.ts b/src/gateway/server-startup-memory.ts
index ace12fe0113..80d90425c16 100644
--- a/src/gateway/server-startup-memory.ts
+++ b/src/gateway/server-startup-memory.ts
@@ -1,10 +1,39 @@
-import { listAgentIds } from "../agents/agent-scope.js";
+import { listAgentEntries, listAgentIds, resolveDefaultAgentId } from "../agents/agent-scope.js";
 import { resolveMemorySearchConfig } from "../agents/memory-search.js";
 import type { OpenClawConfig } from "../config/types.openclaw.js";
 import {
-  getActiveMemorySearchManager,
-  resolveActiveMemoryBackendConfig,
-} from "../plugins/memory-runtime.js";
+  resolveMemoryBackendConfig,
+  type ResolvedQmdConfig,
+} from "../memory-host-sdk/host/backend-config.js";
+import { getActiveMemorySearchManager } from "../plugins/memory-runtime.js";
+import { normalizeAgentId } from "../routing/session-key.js";
+
+function shouldStartQmdBackgroundWork(qmd: ResolvedQmdConfig): boolean {
+  return qmd.update.onBoot || qmd.update.intervalMs > 0 || qmd.update.embedIntervalMs > 0;
+}
+
+function hasExplicitAgentMemorySearchConfig(cfg: OpenClawConfig, agentId: string): boolean {
+  return listAgentEntries(cfg).some(
+    (entry) => normalizeAgentId(entry.id) === agentId && entry.memorySearch != null,
+  );
+}
+
+function shouldEagerlyStartAgentMemory(params: {
+  cfg: OpenClawConfig;
+  agentId: string;
+  agentCount: number;
+}): boolean {
+  if (params.agentCount <= 1) {
+    return true;
+  }
+  if (params.agentId === resolveDefaultAgentId(params.cfg)) {
+    return true;
+  }
+  if (params.cfg.agents?.defaults?.memorySearch?.enabled === true) {
+    return true;
+  }
+  return hasExplicitAgentMemorySearchConfig(params.cfg, params.agentId);
+}
 
 export async function startGatewayMemoryBackend(params: {
   cfg: OpenClawConfig;
@@ -12,17 +41,31 @@ export async function startGatewayMemoryBackend(params: {
 }): Promise<void> {
   const agentIds = listAgentIds(params.cfg);
   const armedAgentIds: string[] = [];
+  const deferredAgentIds: string[] = [];
   for (const agentId of agentIds) {
     if (!resolveMemorySearchConfig(params.cfg, agentId)) {
       continue;
     }
-    const resolved = resolveActiveMemoryBackendConfig({ cfg: params.cfg, agentId });
+    const resolved = resolveMemoryBackendConfig({ cfg: params.cfg, agentId });
     if (!resolved) {
       continue;
     }
     if (resolved.backend !== "qmd" || !resolved.qmd) {
       continue;
     }
+    if (!shouldStartQmdBackgroundWork(resolved.qmd)) {
+      continue;
+    }
+    if (
+      !shouldEagerlyStartAgentMemory({
+        cfg: params.cfg,
+        agentId,
+        agentCount: agentIds.length,
+      })
+    ) {
+      deferredAgentIds.push(agentId);
+      continue;
+    }
 
     const { manager, error } = await getActiveMemorySearchManager({ cfg: params.cfg, agentId });
     if (!manager) {
@@ -40,6 +83,13 @@ export async function startGatewayMemoryBackend(params: {
         .join(", ")}`,
     );
   }
+  if (deferredAgentIds.length > 0) {
+    params.log.info?.(
+      `qmd memory startup initialization deferred for ${formatAgentCount(deferredAgentIds.length)}: ${deferredAgentIds
+        .map((agentId) => `"${agentId}"`)
+        .join(", ")}`,
+    );
+  }
 }
 
 function formatAgentCount(count: number): string {