From 4ebec8b5dc8318260efab5fc41cd3ab83cdd3db0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 14:36:59 +0100 Subject: [PATCH] fix(memory): group qmd collection searches --- CHANGELOG.md | 1 + docs/concepts/memory-qmd.md | 3 + docs/reference/memory-config.md | 2 +- .../src/memory/qmd-manager.test.ts | 109 ++++++++++++++++++ .../memory-core/src/memory/qmd-manager.ts | 78 ++++++++++--- 5 files changed, 178 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 771abdb4a8a..027717a5668 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ Docs: https://docs.openclaw.ai - Ollama/WSL2: warn when GPU-backed WSL2 installs combine CUDA visibility with an autostarting `ollama.service` using `Restart=always`, and document the systemd, `.wslconfig`, and keep-alive mitigation for crash loops. Carries forward #61022; fixes #61185. Thanks @yhyatt. - Ollama/onboarding: de-dupe suggested bare local models against installed `:latest` tags and skip redundant pulls, so setup shows the installed model once and no longer says it is downloading an already available model. Fixes #68952. Thanks @tleyden. - Memory-core/doctor: keep `doctor.memory.status` on the cached path by default and only run live embedding pings for explicit deep probes, preventing slow local embedding backends from blocking Gateway status checks. Fixes #71568. Thanks @apex-system. +- Memory/QMD: group same-source collections into one QMD search invocation when the installed QMD supports multiple `-c` filters, while keeping older QMD builds on the per-collection fallback. Fixes #72484; supersedes #72485 and #69583. Thanks @BsnizND and @zeroaltitude. - Memory/QMD: skip QMD vector status probes and embedding maintenance in lexical `searchMode: "search"`, so BM25-only QMD setups on ARM do not trigger llama.cpp/Vulkan builds during status checks or embed cycles. Fixes #59234 and #67113. Thanks @PrinceOfEgypt, @Vksh07, @Snipe76, @NomLom, @t4r3e2q1-commits, and @dmak. - Memory/QMD: report the live watcher dirty state in memory status, so changed QMD-backed memory files show as dirty until the queued sync finishes. Fixes #60244. Thanks @xinzf. - Compaction: skip oversized pre-compaction checkpoint snapshots and prune duplicate long user turns from compaction input and rotated successor transcripts, preventing retry storms from being preserved across checkpoint cycles. Fixes #72780. Thanks @SweetSophia. diff --git a/docs/concepts/memory-qmd.md b/docs/concepts/memory-qmd.md index 839f5c7bac8..1d69fe96078 100644 --- a/docs/concepts/memory-qmd.md +++ b/docs/concepts/memory-qmd.md @@ -60,6 +60,9 @@ present. `vsearch` and `query`). `search` is BM25-only, so OpenClaw skips semantic vector readiness probes and embedding maintenance in that mode. If a mode fails, OpenClaw retries with `qmd query`. +- With QMD releases that advertise multi-collection filters, OpenClaw groups + same-source collections into one QMD search invocation. Older QMD releases + keep the compatible per-collection fallback. - If QMD fails entirely, OpenClaw falls back to the builtin SQLite engine. diff --git a/docs/reference/memory-config.md b/docs/reference/memory-config.md index 53f3f5aa522..a14bd4d70d0 100644 --- a/docs/reference/memory-config.md +++ b/docs/reference/memory-config.md @@ -451,7 +451,7 @@ Set `memory.backend = "qmd"` to enable. All QMD settings live under `memory.qmd` `searchMode: "search"` is lexical/BM25-only. OpenClaw does not run semantic vector readiness probes or QMD embedding maintenance for that mode, including during `memory status --deep`; `vsearch` and `query` continue to require QMD vector readiness and embeddings. -OpenClaw prefers current QMD collection and MCP query shapes, but keeps older QMD releases working by trying compatible collection pattern flags and older MCP tool names when needed. +OpenClaw prefers current QMD collection and MCP query shapes, but keeps older QMD releases working by trying compatible collection pattern flags and older MCP tool names when needed. When QMD advertises support for multiple collection filters, same-source collections are searched with one QMD process; older QMD builds keep the per-collection compatibility path. QMD model overrides stay on the QMD side, not OpenClaw config. If you need to override QMD's models globally, set environment variables such as `QMD_EMBED_MODEL`, `QMD_RERANK_MODEL`, and `QMD_GENERATE_MODEL` in the gateway runtime environment. diff --git a/extensions/memory-core/src/memory/qmd-manager.test.ts b/extensions/memory-core/src/memory/qmd-manager.test.ts index 1fc523617dd..87408589f2b 100644 --- a/extensions/memory-core/src/memory/qmd-manager.test.ts +++ b/extensions/memory-core/src/memory/qmd-manager.test.ts @@ -2134,6 +2134,115 @@ describe("QmdMemoryManager", () => { await manager.close(); }); + it("groups same-source qmd queries when the installed qmd supports multiple collection filters", async () => { + cfg = { + ...cfg, + memory: { + backend: "qmd", + qmd: { + includeDefaultMemory: false, + update: { interval: "0s", debounceMs: 60_000, onBoot: false }, + paths: [ + { path: workspaceDir, pattern: "**/*.md", name: "workspace" }, + { path: path.join(workspaceDir, "notes"), pattern: "**/*.md", name: "notes" }, + ], + }, + }, + } as OpenClawConfig; + + spawnMock.mockImplementation((_cmd: string, args: string[]) => { + if (args[0] === "--help") { + const child = createMockChild({ autoClose: false }); + emitAndClose( + child, + "stdout", + "-c, --collection Filter by one or more collections", + ); + return child; + } + if (args[0] === "search") { + const child = createMockChild({ autoClose: false }); + emitAndClose(child, "stdout", "[]"); + return child; + } + return createMockChild(); + }); + + const { manager, resolved } = await createManager(); + + await manager.search("test", { sessionKey: "agent:main:slack:dm:u123" }); + const maxResults = resolved.qmd?.limits.maxResults; + if (!maxResults) { + throw new Error("qmd maxResults missing"); + } + const searchCalls = spawnMock.mock.calls + .map((call: unknown[]) => call[1] as string[]) + .filter((args: string[]) => args[0] === "search"); + expect(searchCalls).toEqual([ + [ + "search", + "test", + "--json", + "-n", + String(maxResults), + "-c", + "workspace-main", + "-c", + "notes-main", + ], + ]); + await manager.close(); + }); + + it("keeps mixed-source qmd queries in separate source groups", async () => { + cfg = { + ...cfg, + memory: { + backend: "qmd", + qmd: { + includeDefaultMemory: false, + update: { interval: "0s", debounceMs: 60_000, onBoot: false }, + sessions: { enabled: true }, + paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }], + }, + }, + } as OpenClawConfig; + + spawnMock.mockImplementation((_cmd: string, args: string[]) => { + if (args[0] === "--help") { + const child = createMockChild({ autoClose: false }); + emitAndClose( + child, + "stdout", + "-c, --collection Filter by one or more collections", + ); + return child; + } + if (args[0] === "search") { + const child = createMockChild({ autoClose: false }); + emitAndClose(child, "stdout", "[]"); + return child; + } + return createMockChild(); + }); + + const { manager, resolved } = await createManager(); + + await manager.search("test", { sessionKey: "agent:main:slack:dm:u123" }); + const maxResults = resolved.qmd?.limits.maxResults; + if (!maxResults) { + throw new Error("qmd maxResults missing"); + } + const searchCalls = spawnMock.mock.calls + .map((call: unknown[]) => call[1] as string[]) + .filter((args: string[]) => args[0] === "search"); + expect(searchCalls).toEqual([ + ["search", "test", "--json", "-n", String(maxResults), "-c", "workspace-main"], + ["search", "test", "--json", "-n", String(maxResults), "-c", "sessions-main"], + ]); + await manager.close(); + }); + it("does not query phantom memory-alt collections when MEMORY.md exists", async () => { await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# canonical root"); cfg = { diff --git a/extensions/memory-core/src/memory/qmd-manager.ts b/extensions/memory-core/src/memory/qmd-manager.ts index 1d92dec35f3..0cfb77b3b27 100644 --- a/extensions/memory-core/src/memory/qmd-manager.ts +++ b/extensions/memory-core/src/memory/qmd-manager.ts @@ -338,6 +338,7 @@ export class QmdMemoryManager implements MemorySearchManager { private attemptedDuplicateDocumentRepair = false; private readonly sessionWarm = new Set(); private collectionPatternFlag: QmdCollectionPatternFlag | null = "--mask"; + private multiCollectionFilterSupported: boolean | null = null; private constructor(params: { agentId: string; @@ -1150,16 +1151,17 @@ export class QmdMemoryManager implements MemorySearchManager { timeoutMs: this.qmd.limits.timeoutMs, }); } - if (collectionNames.length > 1) { - return await this.runQueryAcrossCollections( + const collectionGroups = await this.resolveCollectionSearchGroups(collectionNames); + if (collectionGroups.length > 1) { + return await this.runQueryAcrossCollectionGroups( trimmed, limit, - collectionNames, + collectionGroups, qmdSearchCommand, ); } const args = this.buildSearchArgs(qmdSearchCommand, trimmed, limit); - args.push(...this.buildCollectionFilterArgs(collectionNames)); + args.push(...this.buildCollectionFilterArgs(collectionGroups[0] ?? collectionNames)); const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs }); return parseQmdQueryJson(result.stdout, result.stderr); } catch (err) { @@ -1177,11 +1179,19 @@ export class QmdMemoryManager implements MemorySearchManager { `qmd ${qmdSearchCommand} does not support configured flags; retrying search with qmd query`, ); try { - if (collectionNames.length > 1) { - return await this.runQueryAcrossCollections(trimmed, limit, collectionNames, "query"); + const collectionGroups = await this.resolveCollectionSearchGroups(collectionNames); + if (collectionGroups.length > 1) { + return await this.runQueryAcrossCollectionGroups( + trimmed, + limit, + collectionGroups, + "query", + ); } const fallbackArgs = this.buildSearchArgs("query", trimmed, limit); - fallbackArgs.push(...this.buildCollectionFilterArgs(collectionNames)); + fallbackArgs.push( + ...this.buildCollectionFilterArgs(collectionGroups[0] ?? collectionNames), + ); const fallback = await this.runQmd(fallbackArgs, { timeoutMs: this.qmd.limits.timeoutMs, }); @@ -2884,24 +2894,53 @@ export class QmdMemoryManager implements MemorySearchManager { ]); } - private async runQueryAcrossCollections( + private async resolveCollectionSearchGroups(collectionNames: string[]): Promise { + if (collectionNames.length <= 1) { + return [collectionNames]; + } + if (!(await this.supportsQmdMultiCollectionFilters())) { + return collectionNames.map((collectionName) => [collectionName]); + } + return this.groupCollectionNamesBySource(collectionNames); + } + + private async supportsQmdMultiCollectionFilters(): Promise { + if (this.multiCollectionFilterSupported !== null) { + return this.multiCollectionFilterSupported; + } + try { + const result = await this.runQmd(["--help"], { + timeoutMs: Math.min(this.qmd.limits.timeoutMs, 5_000), + }); + const helpText = `${result.stdout}\n${result.stderr}`; + this.multiCollectionFilterSupported = + /\b(?:one or more collections|collection\(s\)|multiple -c flags)\b/i.test(helpText); + } catch (err) { + this.multiCollectionFilterSupported = false; + log.debug(`qmd multi-collection filter probe failed: ${String(err)}`); + } + return this.multiCollectionFilterSupported; + } + + private async runQueryAcrossCollectionGroups( query: string, limit: number, - collectionNames: string[], + collectionGroups: string[][], command: "query" | "search" | "vsearch", ): Promise { log.debug( - `qmd ${command} multi-collection workaround active (${collectionNames.length} collections)`, + `qmd ${command} multi-source collection grouping active (${collectionGroups.length} groups)`, ); const bestByResultKey = new Map(); - for (const collectionName of collectionNames) { + for (const collectionNames of collectionGroups) { const args = this.buildSearchArgs(command, query, limit); - args.push("-c", collectionName); + args.push(...this.buildCollectionFilterArgs(collectionNames)); const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs }); const parsed = parseQmdQueryJson(result.stdout, result.stderr); for (const entry of parsed) { + const defaultCollection = collectionNames.length === 1 ? collectionNames[0] : undefined; const normalizedHints = this.normalizeDocHints({ - preferredCollection: entry.collection ?? collectionName, + preferredCollection: entry.collection ?? defaultCollection, preferredFile: entry.file, }); const normalizedDocId = @@ -2911,7 +2950,7 @@ export class QmdMemoryManager implements MemorySearchManager { const withCollection = { ...entry, docid: normalizedDocId, - collection: normalizedHints.preferredCollection ?? entry.collection ?? collectionName, + collection: normalizedHints.preferredCollection ?? entry.collection ?? defaultCollection, file: normalizedHints.preferredFile ?? entry.file, } satisfies QmdQueryResult; const resultKey = this.buildQmdResultKey(withCollection); @@ -2932,6 +2971,17 @@ export class QmdMemoryManager implements MemorySearchManager { return [...bestByResultKey.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0)); } + private groupCollectionNamesBySource(collectionNames: string[]): string[][] { + const groups = new Map(); + for (const collectionName of collectionNames) { + const source = this.collectionRoots.get(collectionName)?.kind ?? collectionName; + const group = groups.get(source) ?? []; + group.push(collectionName); + groups.set(source, group); + } + return [...groups.values()]; + } + private buildQmdResultKey(entry: QmdQueryResult): string | null { if (typeof entry.docid === "string" && entry.docid.trim().length > 0) { return `docid:${entry.docid}`;