fix(memory): group qmd collection searches

2026-05-06 06:50:43 +00:00 · 2026-04-27 14:36:59 +01:00
parent eb1a201060
commit 4ebec8b5dc
5 changed files with 178 additions and 15 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -62,6 +62,7 @@ Docs: https://docs.openclaw.ai
 - Ollama/WSL2: warn when GPU-backed WSL2 installs combine CUDA visibility with an autostarting `ollama.service` using `Restart=always`, and document the systemd, `.wslconfig`, and keep-alive mitigation for crash loops. Carries forward #61022; fixes #61185. Thanks @yhyatt.
 - Ollama/onboarding: de-dupe suggested bare local models against installed `:latest` tags and skip redundant pulls, so setup shows the installed model once and no longer says it is downloading an already available model. Fixes #68952. Thanks @tleyden.
 - Memory-core/doctor: keep `doctor.memory.status` on the cached path by default and only run live embedding pings for explicit deep probes, preventing slow local embedding backends from blocking Gateway status checks. Fixes #71568. Thanks @apex-system.
+- Memory/QMD: group same-source collections into one QMD search invocation when the installed QMD supports multiple `-c` filters, while keeping older QMD builds on the per-collection fallback. Fixes #72484; supersedes #72485 and #69583. Thanks @BsnizND and @zeroaltitude.
 - Memory/QMD: skip QMD vector status probes and embedding maintenance in lexical `searchMode: "search"`, so BM25-only QMD setups on ARM do not trigger llama.cpp/Vulkan builds during status checks or embed cycles. Fixes #59234 and #67113. Thanks @PrinceOfEgypt, @Vksh07, @Snipe76, @NomLom, @t4r3e2q1-commits, and @dmak.
 - Memory/QMD: report the live watcher dirty state in memory status, so changed QMD-backed memory files show as dirty until the queued sync finishes. Fixes #60244. Thanks @xinzf.
 - Compaction: skip oversized pre-compaction checkpoint snapshots and prune duplicate long user turns from compaction input and rotated successor transcripts, preventing retry storms from being preserved across checkpoint cycles. Fixes #72780. Thanks @SweetSophia.
--- a/docs/concepts/memory-qmd.md
+++ b/docs/concepts/memory-qmd.md
@@ -60,6 +60,9 @@ present.
  `vsearch` and `query`). `search` is BM25-only, so OpenClaw skips semantic
  vector readiness probes and embedding maintenance in that mode. If a mode
  fails, OpenClaw retries with `qmd query`.
+- With QMD releases that advertise multi-collection filters, OpenClaw groups
+  same-source collections into one QMD search invocation. Older QMD releases
+  keep the compatible per-collection fallback.
 - If QMD fails entirely, OpenClaw falls back to the builtin SQLite engine.

 <Info>
--- a/docs/reference/memory-config.md
+++ b/docs/reference/memory-config.md
@@ -451,7 +451,7 @@ Set `memory.backend = "qmd"` to enable. All QMD settings live under `memory.qmd`

 `searchMode: "search"` is lexical/BM25-only. OpenClaw does not run semantic vector readiness probes or QMD embedding maintenance for that mode, including during `memory status --deep`; `vsearch` and `query` continue to require QMD vector readiness and embeddings.

-OpenClaw prefers current QMD collection and MCP query shapes, but keeps older QMD releases working by trying compatible collection pattern flags and older MCP tool names when needed.
+OpenClaw prefers current QMD collection and MCP query shapes, but keeps older QMD releases working by trying compatible collection pattern flags and older MCP tool names when needed. When QMD advertises support for multiple collection filters, same-source collections are searched with one QMD process; older QMD builds keep the per-collection compatibility path.

 <Note>
 QMD model overrides stay on the QMD side, not OpenClaw config. If you need to override QMD's models globally, set environment variables such as `QMD_EMBED_MODEL`, `QMD_RERANK_MODEL`, and `QMD_GENERATE_MODEL` in the gateway runtime environment.
--- a/extensions/memory-core/src/memory/qmd-manager.test.ts
+++ b/extensions/memory-core/src/memory/qmd-manager.test.ts
@@ -2134,6 +2134,115 @@ describe("QmdMemoryManager", () => {
    await manager.close();
  });

+  it("groups same-source qmd queries when the installed qmd supports multiple collection filters", async () => {
+    cfg = {
+      ...cfg,
+      memory: {
+        backend: "qmd",
+        qmd: {
+          includeDefaultMemory: false,
+          update: { interval: "0s", debounceMs: 60_000, onBoot: false },
+          paths: [
+            { path: workspaceDir, pattern: "**/*.md", name: "workspace" },
+            { path: path.join(workspaceDir, "notes"), pattern: "**/*.md", name: "notes" },
+          ],
+        },
+      },
+    } as OpenClawConfig;
+
+    spawnMock.mockImplementation((_cmd: string, args: string[]) => {
+      if (args[0] === "--help") {
+        const child = createMockChild({ autoClose: false });
+        emitAndClose(
+          child,
+          "stdout",
+          "-c, --collection <name>    Filter by one or more collections",
+        );
+        return child;
+      }
+      if (args[0] === "search") {
+        const child = createMockChild({ autoClose: false });
+        emitAndClose(child, "stdout", "[]");
+        return child;
+      }
+      return createMockChild();
+    });
+
+    const { manager, resolved } = await createManager();
+
+    await manager.search("test", { sessionKey: "agent:main:slack:dm:u123" });
+    const maxResults = resolved.qmd?.limits.maxResults;
+    if (!maxResults) {
+      throw new Error("qmd maxResults missing");
+    }
+    const searchCalls = spawnMock.mock.calls
+      .map((call: unknown[]) => call[1] as string[])
+      .filter((args: string[]) => args[0] === "search");
+    expect(searchCalls).toEqual([
+      [
+        "search",
+        "test",
+        "--json",
+        "-n",
+        String(maxResults),
+        "-c",
+        "workspace-main",
+        "-c",
+        "notes-main",
+      ],
+    ]);
+    await manager.close();
+  });
+
+  it("keeps mixed-source qmd queries in separate source groups", async () => {
+    cfg = {
+      ...cfg,
+      memory: {
+        backend: "qmd",
+        qmd: {
+          includeDefaultMemory: false,
+          update: { interval: "0s", debounceMs: 60_000, onBoot: false },
+          sessions: { enabled: true },
+          paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
+        },
+      },
+    } as OpenClawConfig;
+
+    spawnMock.mockImplementation((_cmd: string, args: string[]) => {
+      if (args[0] === "--help") {
+        const child = createMockChild({ autoClose: false });
+        emitAndClose(
+          child,
+          "stdout",
+          "-c, --collection <name>    Filter by one or more collections",
+        );
+        return child;
+      }
+      if (args[0] === "search") {
+        const child = createMockChild({ autoClose: false });
+        emitAndClose(child, "stdout", "[]");
+        return child;
+      }
+      return createMockChild();
+    });
+
+    const { manager, resolved } = await createManager();
+
+    await manager.search("test", { sessionKey: "agent:main:slack:dm:u123" });
+    const maxResults = resolved.qmd?.limits.maxResults;
+    if (!maxResults) {
+      throw new Error("qmd maxResults missing");
+    }
+    const searchCalls = spawnMock.mock.calls
+      .map((call: unknown[]) => call[1] as string[])
+      .filter((args: string[]) => args[0] === "search");
+    expect(searchCalls).toEqual([
+      ["search", "test", "--json", "-n", String(maxResults), "-c", "workspace-main"],
+      ["search", "test", "--json", "-n", String(maxResults), "-c", "sessions-main"],
+    ]);
+    await manager.close();
+  });
+
  it("does not query phantom memory-alt collections when MEMORY.md exists", async () => {
    await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# canonical root");
    cfg = {
--- a/extensions/memory-core/src/memory/qmd-manager.ts
+++ b/extensions/memory-core/src/memory/qmd-manager.ts
@@ -338,6 +338,7 @@ export class QmdMemoryManager implements MemorySearchManager {
  private attemptedDuplicateDocumentRepair = false;
  private readonly sessionWarm = new Set<string>();
  private collectionPatternFlag: QmdCollectionPatternFlag | null = "--mask";
+  private multiCollectionFilterSupported: boolean | null = null;

  private constructor(params: {
    agentId: string;
@@ -1150,16 +1151,17 @@ export class QmdMemoryManager implements MemorySearchManager {
            timeoutMs: this.qmd.limits.timeoutMs,
          });
        }
-        if (collectionNames.length > 1) {
-          return await this.runQueryAcrossCollections(
+        const collectionGroups = await this.resolveCollectionSearchGroups(collectionNames);
+        if (collectionGroups.length > 1) {
+          return await this.runQueryAcrossCollectionGroups(
            trimmed,
            limit,
-            collectionNames,
+            collectionGroups,
            qmdSearchCommand,
          );
        }
        const args = this.buildSearchArgs(qmdSearchCommand, trimmed, limit);
-        args.push(...this.buildCollectionFilterArgs(collectionNames));
+        args.push(...this.buildCollectionFilterArgs(collectionGroups[0] ?? collectionNames));
        const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs });
        return parseQmdQueryJson(result.stdout, result.stderr);
      } catch (err) {
@@ -1177,11 +1179,19 @@ export class QmdMemoryManager implements MemorySearchManager {
            `qmd ${qmdSearchCommand} does not support configured flags; retrying search with qmd query`,
          );
          try {
-            if (collectionNames.length > 1) {
-              return await this.runQueryAcrossCollections(trimmed, limit, collectionNames, "query");
+            const collectionGroups = await this.resolveCollectionSearchGroups(collectionNames);
+            if (collectionGroups.length > 1) {
+              return await this.runQueryAcrossCollectionGroups(
+                trimmed,
+                limit,
+                collectionGroups,
+                "query",
+              );
            }
            const fallbackArgs = this.buildSearchArgs("query", trimmed, limit);
-            fallbackArgs.push(...this.buildCollectionFilterArgs(collectionNames));
+            fallbackArgs.push(
+              ...this.buildCollectionFilterArgs(collectionGroups[0] ?? collectionNames),
+            );
            const fallback = await this.runQmd(fallbackArgs, {
              timeoutMs: this.qmd.limits.timeoutMs,
            });
@@ -2884,24 +2894,53 @@ export class QmdMemoryManager implements MemorySearchManager {
    ]);
  }

-  private async runQueryAcrossCollections(
+  private async resolveCollectionSearchGroups(collectionNames: string[]): Promise<string[][]> {
+    if (collectionNames.length <= 1) {
+      return [collectionNames];
+    }
+    if (!(await this.supportsQmdMultiCollectionFilters())) {
+      return collectionNames.map((collectionName) => [collectionName]);
+    }
+    return this.groupCollectionNamesBySource(collectionNames);
+  }
+
+  private async supportsQmdMultiCollectionFilters(): Promise<boolean> {
+    if (this.multiCollectionFilterSupported !== null) {
+      return this.multiCollectionFilterSupported;
+    }
+    try {
+      const result = await this.runQmd(["--help"], {
+        timeoutMs: Math.min(this.qmd.limits.timeoutMs, 5_000),
+      });
+      const helpText = `${result.stdout}\n${result.stderr}`;
+      this.multiCollectionFilterSupported =
+        /\b(?:one or more collections|collection\(s\)|multiple -c flags)\b/i.test(helpText);
+    } catch (err) {
+      this.multiCollectionFilterSupported = false;
+      log.debug(`qmd multi-collection filter probe failed: ${String(err)}`);
+    }
+    return this.multiCollectionFilterSupported;
+  }
+
+  private async runQueryAcrossCollectionGroups(
    query: string,
    limit: number,
-    collectionNames: string[],
+    collectionGroups: string[][],
    command: "query" | "search" | "vsearch",
  ): Promise<QmdQueryResult[]> {
    log.debug(
-      `qmd ${command} multi-collection workaround active (${collectionNames.length} collections)`,
+      `qmd ${command} multi-source collection grouping active (${collectionGroups.length} groups)`,
    );
    const bestByResultKey = new Map<string, QmdQueryResult>();
-    for (const collectionName of collectionNames) {
+    for (const collectionNames of collectionGroups) {
      const args = this.buildSearchArgs(command, query, limit);
-      args.push("-c", collectionName);
+      args.push(...this.buildCollectionFilterArgs(collectionNames));
      const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs });
      const parsed = parseQmdQueryJson(result.stdout, result.stderr);
      for (const entry of parsed) {
+        const defaultCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
        const normalizedHints = this.normalizeDocHints({
-          preferredCollection: entry.collection ?? collectionName,
+          preferredCollection: entry.collection ?? defaultCollection,
          preferredFile: entry.file,
        });
        const normalizedDocId =
@@ -2911,7 +2950,7 @@ export class QmdMemoryManager implements MemorySearchManager {
        const withCollection = {
          ...entry,
          docid: normalizedDocId,
-          collection: normalizedHints.preferredCollection ?? entry.collection ?? collectionName,
+          collection: normalizedHints.preferredCollection ?? entry.collection ?? defaultCollection,
          file: normalizedHints.preferredFile ?? entry.file,
        } satisfies QmdQueryResult;
        const resultKey = this.buildQmdResultKey(withCollection);
@@ -2932,6 +2971,17 @@ export class QmdMemoryManager implements MemorySearchManager {
    return [...bestByResultKey.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0));
  }

+  private groupCollectionNamesBySource(collectionNames: string[]): string[][] {
+    const groups = new Map<string, string[]>();
+    for (const collectionName of collectionNames) {
+      const source = this.collectionRoots.get(collectionName)?.kind ?? collectionName;
+      const group = groups.get(source) ?? [];
+      group.push(collectionName);
+      groups.set(source, group);
+    }
+    return [...groups.values()];
+  }
+
  private buildQmdResultKey(entry: QmdQueryResult): string | null {
    if (typeof entry.docid === "string" && entry.docid.trim().length > 0) {
      return `docid:${entry.docid}`;