mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 19:10:58 +00:00
fix(memory): group qmd collection searches
This commit is contained in:
@@ -2134,6 +2134,115 @@ describe("QmdMemoryManager", () => {
|
||||
await manager.close();
|
||||
});
|
||||
|
||||
it("groups same-source qmd queries when the installed qmd supports multiple collection filters", async () => {
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
paths: [
|
||||
{ path: workspaceDir, pattern: "**/*.md", name: "workspace" },
|
||||
{ path: path.join(workspaceDir, "notes"), pattern: "**/*.md", name: "notes" },
|
||||
],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "--help") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(
|
||||
child,
|
||||
"stdout",
|
||||
"-c, --collection <name> Filter by one or more collections",
|
||||
);
|
||||
return child;
|
||||
}
|
||||
if (args[0] === "search") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
|
||||
const { manager, resolved } = await createManager();
|
||||
|
||||
await manager.search("test", { sessionKey: "agent:main:slack:dm:u123" });
|
||||
const maxResults = resolved.qmd?.limits.maxResults;
|
||||
if (!maxResults) {
|
||||
throw new Error("qmd maxResults missing");
|
||||
}
|
||||
const searchCalls = spawnMock.mock.calls
|
||||
.map((call: unknown[]) => call[1] as string[])
|
||||
.filter((args: string[]) => args[0] === "search");
|
||||
expect(searchCalls).toEqual([
|
||||
[
|
||||
"search",
|
||||
"test",
|
||||
"--json",
|
||||
"-n",
|
||||
String(maxResults),
|
||||
"-c",
|
||||
"workspace-main",
|
||||
"-c",
|
||||
"notes-main",
|
||||
],
|
||||
]);
|
||||
await manager.close();
|
||||
});
|
||||
|
||||
it("keeps mixed-source qmd queries in separate source groups", async () => {
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
sessions: { enabled: true },
|
||||
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "--help") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(
|
||||
child,
|
||||
"stdout",
|
||||
"-c, --collection <name> Filter by one or more collections",
|
||||
);
|
||||
return child;
|
||||
}
|
||||
if (args[0] === "search") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
|
||||
const { manager, resolved } = await createManager();
|
||||
|
||||
await manager.search("test", { sessionKey: "agent:main:slack:dm:u123" });
|
||||
const maxResults = resolved.qmd?.limits.maxResults;
|
||||
if (!maxResults) {
|
||||
throw new Error("qmd maxResults missing");
|
||||
}
|
||||
const searchCalls = spawnMock.mock.calls
|
||||
.map((call: unknown[]) => call[1] as string[])
|
||||
.filter((args: string[]) => args[0] === "search");
|
||||
expect(searchCalls).toEqual([
|
||||
["search", "test", "--json", "-n", String(maxResults), "-c", "workspace-main"],
|
||||
["search", "test", "--json", "-n", String(maxResults), "-c", "sessions-main"],
|
||||
]);
|
||||
await manager.close();
|
||||
});
|
||||
|
||||
it("does not query phantom memory-alt collections when MEMORY.md exists", async () => {
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# canonical root");
|
||||
cfg = {
|
||||
|
||||
@@ -338,6 +338,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
private attemptedDuplicateDocumentRepair = false;
|
||||
private readonly sessionWarm = new Set<string>();
|
||||
private collectionPatternFlag: QmdCollectionPatternFlag | null = "--mask";
|
||||
private multiCollectionFilterSupported: boolean | null = null;
|
||||
|
||||
private constructor(params: {
|
||||
agentId: string;
|
||||
@@ -1150,16 +1151,17 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
timeoutMs: this.qmd.limits.timeoutMs,
|
||||
});
|
||||
}
|
||||
if (collectionNames.length > 1) {
|
||||
return await this.runQueryAcrossCollections(
|
||||
const collectionGroups = await this.resolveCollectionSearchGroups(collectionNames);
|
||||
if (collectionGroups.length > 1) {
|
||||
return await this.runQueryAcrossCollectionGroups(
|
||||
trimmed,
|
||||
limit,
|
||||
collectionNames,
|
||||
collectionGroups,
|
||||
qmdSearchCommand,
|
||||
);
|
||||
}
|
||||
const args = this.buildSearchArgs(qmdSearchCommand, trimmed, limit);
|
||||
args.push(...this.buildCollectionFilterArgs(collectionNames));
|
||||
args.push(...this.buildCollectionFilterArgs(collectionGroups[0] ?? collectionNames));
|
||||
const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs });
|
||||
return parseQmdQueryJson(result.stdout, result.stderr);
|
||||
} catch (err) {
|
||||
@@ -1177,11 +1179,19 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
`qmd ${qmdSearchCommand} does not support configured flags; retrying search with qmd query`,
|
||||
);
|
||||
try {
|
||||
if (collectionNames.length > 1) {
|
||||
return await this.runQueryAcrossCollections(trimmed, limit, collectionNames, "query");
|
||||
const collectionGroups = await this.resolveCollectionSearchGroups(collectionNames);
|
||||
if (collectionGroups.length > 1) {
|
||||
return await this.runQueryAcrossCollectionGroups(
|
||||
trimmed,
|
||||
limit,
|
||||
collectionGroups,
|
||||
"query",
|
||||
);
|
||||
}
|
||||
const fallbackArgs = this.buildSearchArgs("query", trimmed, limit);
|
||||
fallbackArgs.push(...this.buildCollectionFilterArgs(collectionNames));
|
||||
fallbackArgs.push(
|
||||
...this.buildCollectionFilterArgs(collectionGroups[0] ?? collectionNames),
|
||||
);
|
||||
const fallback = await this.runQmd(fallbackArgs, {
|
||||
timeoutMs: this.qmd.limits.timeoutMs,
|
||||
});
|
||||
@@ -2884,24 +2894,53 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
]);
|
||||
}
|
||||
|
||||
private async runQueryAcrossCollections(
|
||||
private async resolveCollectionSearchGroups(collectionNames: string[]): Promise<string[][]> {
|
||||
if (collectionNames.length <= 1) {
|
||||
return [collectionNames];
|
||||
}
|
||||
if (!(await this.supportsQmdMultiCollectionFilters())) {
|
||||
return collectionNames.map((collectionName) => [collectionName]);
|
||||
}
|
||||
return this.groupCollectionNamesBySource(collectionNames);
|
||||
}
|
||||
|
||||
private async supportsQmdMultiCollectionFilters(): Promise<boolean> {
|
||||
if (this.multiCollectionFilterSupported !== null) {
|
||||
return this.multiCollectionFilterSupported;
|
||||
}
|
||||
try {
|
||||
const result = await this.runQmd(["--help"], {
|
||||
timeoutMs: Math.min(this.qmd.limits.timeoutMs, 5_000),
|
||||
});
|
||||
const helpText = `${result.stdout}\n${result.stderr}`;
|
||||
this.multiCollectionFilterSupported =
|
||||
/\b(?:one or more collections|collection\(s\)|multiple -c flags)\b/i.test(helpText);
|
||||
} catch (err) {
|
||||
this.multiCollectionFilterSupported = false;
|
||||
log.debug(`qmd multi-collection filter probe failed: ${String(err)}`);
|
||||
}
|
||||
return this.multiCollectionFilterSupported;
|
||||
}
|
||||
|
||||
private async runQueryAcrossCollectionGroups(
|
||||
query: string,
|
||||
limit: number,
|
||||
collectionNames: string[],
|
||||
collectionGroups: string[][],
|
||||
command: "query" | "search" | "vsearch",
|
||||
): Promise<QmdQueryResult[]> {
|
||||
log.debug(
|
||||
`qmd ${command} multi-collection workaround active (${collectionNames.length} collections)`,
|
||||
`qmd ${command} multi-source collection grouping active (${collectionGroups.length} groups)`,
|
||||
);
|
||||
const bestByResultKey = new Map<string, QmdQueryResult>();
|
||||
for (const collectionName of collectionNames) {
|
||||
for (const collectionNames of collectionGroups) {
|
||||
const args = this.buildSearchArgs(command, query, limit);
|
||||
args.push("-c", collectionName);
|
||||
args.push(...this.buildCollectionFilterArgs(collectionNames));
|
||||
const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs });
|
||||
const parsed = parseQmdQueryJson(result.stdout, result.stderr);
|
||||
for (const entry of parsed) {
|
||||
const defaultCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
|
||||
const normalizedHints = this.normalizeDocHints({
|
||||
preferredCollection: entry.collection ?? collectionName,
|
||||
preferredCollection: entry.collection ?? defaultCollection,
|
||||
preferredFile: entry.file,
|
||||
});
|
||||
const normalizedDocId =
|
||||
@@ -2911,7 +2950,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
const withCollection = {
|
||||
...entry,
|
||||
docid: normalizedDocId,
|
||||
collection: normalizedHints.preferredCollection ?? entry.collection ?? collectionName,
|
||||
collection: normalizedHints.preferredCollection ?? entry.collection ?? defaultCollection,
|
||||
file: normalizedHints.preferredFile ?? entry.file,
|
||||
} satisfies QmdQueryResult;
|
||||
const resultKey = this.buildQmdResultKey(withCollection);
|
||||
@@ -2932,6 +2971,17 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
return [...bestByResultKey.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
||||
}
|
||||
|
||||
private groupCollectionNamesBySource(collectionNames: string[]): string[][] {
|
||||
const groups = new Map<string, string[]>();
|
||||
for (const collectionName of collectionNames) {
|
||||
const source = this.collectionRoots.get(collectionName)?.kind ?? collectionName;
|
||||
const group = groups.get(source) ?? [];
|
||||
group.push(collectionName);
|
||||
groups.set(source, group);
|
||||
}
|
||||
return [...groups.values()];
|
||||
}
|
||||
|
||||
private buildQmdResultKey(entry: QmdQueryResult): string | null {
|
||||
if (typeof entry.docid === "string" && entry.docid.trim().length > 0) {
|
||||
return `docid:${entry.docid}`;
|
||||
|
||||
Reference in New Issue
Block a user