feat(memory): allow QMD searches via mcporter keep-alive (openclaw#19617) thanks @vignesh07

Verified:
- pnpm build
- pnpm check
- pnpm test:macmini

Co-authored-by: vignesh07 <1436853+vignesh07@users.noreply.github.com>
Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com>
This commit is contained in:
Vignesh
2026-02-21 16:54:33 -08:00
committed by GitHub
parent 2e8e357bf7
commit 3317b49d3b
10 changed files with 482 additions and 8 deletions

2
.gitignore vendored
View File

@@ -99,3 +99,5 @@ package-lock.json
# Local iOS signing overrides
apps/ios/LocalSigning.xcconfig
# Generated protocol schema (produced via pnpm protocol:gen)
dist/protocol.schema.json

View File

@@ -16,6 +16,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- Memory/QMD: add optional `memory.qmd.mcporter` search routing so QMD `query/search/vsearch` can run through mcporter keep-alive flows (including multi-collection paths) to reduce cold starts, while keeping searches on agent-scoped QMD state for consistent recall. (#19617) Thanks @vignesh07.
- Chat/UI: strip inline reply/audio directive tags (`[[reply_to_current]]`, `[[reply_to:<id>]]`, `[[audio_as_voice]]`) from displayed chat history, live chat event output, and session preview snippets so control tags no longer leak into user-visible surfaces.
- Security/Config: block prototype-key traversal during config merge patch and legacy migration merge helpers (`__proto__`, `constructor`, `prototype`) to prevent prototype pollution during config mutation flows. (#22968) Thanks @Clawborn.
- Security/Shell env: validate login-shell executable paths for shell-env fallback (`/etc/shells` + trusted prefixes) and block `SHELL` in dangerous env override policy paths so untrusted shell-path injection falls back safely to `/bin/sh`. Thanks @athuljayaram for reporting.

View File

@@ -236,6 +236,13 @@ export const FIELD_HELP: Record<string, string> = {
"memory.backend": 'Memory backend ("builtin" for OpenClaw embeddings, "qmd" for QMD sidecar).',
"memory.citations": 'Default citation behavior ("auto", "on", or "off").',
"memory.qmd.command": "Path to the qmd binary (default: resolves from PATH).",
"memory.qmd.mcporter":
"Optional: route QMD searches through mcporter (MCP runtime) instead of spawning `qmd` per query. Intended to avoid per-search cold starts when QMD models are large.",
"memory.qmd.mcporter.enabled": "Enable mcporter-backed QMD searches (default: false).",
"memory.qmd.mcporter.serverName":
"mcporter server name to call (default: qmd). Server should run `qmd mcp` with lifecycle keep-alive.",
"memory.qmd.mcporter.startDaemon":
"Start `mcporter daemon start` automatically when enabled (default: true).",
"memory.qmd.includeDefaultMemory":
"Whether to automatically index MEMORY.md + memory/**/*.md (default: true).",
"memory.qmd.paths":

View File

@@ -12,6 +12,7 @@ export type MemoryConfig = {
export type MemoryQmdConfig = {
command?: string;
mcporter?: MemoryQmdMcporterConfig;
searchMode?: MemoryQmdSearchMode;
includeDefaultMemory?: boolean;
paths?: MemoryQmdIndexPath[];
@@ -21,6 +22,20 @@ export type MemoryQmdConfig = {
scope?: SessionSendPolicyConfig;
};
export type MemoryQmdMcporterConfig = {
/**
* Route QMD searches through mcporter (MCP runtime) instead of spawning `qmd` per query.
* Requires:
* - `mcporter` installed and on PATH
* - A configured mcporter server that runs `qmd mcp` with `lifecycle: keep-alive`
*/
enabled?: boolean;
/** mcporter server name (defaults to "qmd") */
serverName?: string;
/** Start the mcporter daemon automatically (defaults to true when enabled). */
startDaemon?: boolean;
};
export type MemoryQmdIndexPath = {
path: string;
name?: string;

View File

@@ -72,9 +72,18 @@ const MemoryQmdLimitsSchema = z
})
.strict();
const MemoryQmdMcporterSchema = z
.object({
enabled: z.boolean().optional(),
serverName: z.string().optional(),
startDaemon: z.boolean().optional(),
})
.strict();
const MemoryQmdSchema = z
.object({
command: z.string().optional(),
mcporter: MemoryQmdMcporterSchema.optional(),
searchMode: z.union([z.literal("query"), z.literal("search"), z.literal("vsearch")]).optional(),
includeDefaultMemory: z.boolean().optional(),
paths: z.array(MemoryQmdPathSchema).optional(),

View File

@@ -17,7 +17,6 @@ import {
validateSafeBinArgv,
} from "./exec-safe-bin-policy.js";
import { isTrustedSafeBinPath } from "./exec-safe-bin-trust.js";
export function normalizeSafeBins(entries?: string[]): Set<string> {
if (!Array.isArray(entries)) {
return new Set();

View File

@@ -8,6 +8,7 @@ import type {
MemoryCitationsMode,
MemoryQmdConfig,
MemoryQmdIndexPath,
MemoryQmdMcporterConfig,
MemoryQmdSearchMode,
} from "../config/types.memory.js";
import { resolveUserPath } from "../utils.js";
@@ -50,8 +51,15 @@ export type ResolvedQmdSessionConfig = {
retentionDays?: number;
};
export type ResolvedQmdMcporterConfig = {
enabled: boolean;
serverName: string;
startDaemon: boolean;
};
export type ResolvedQmdConfig = {
command: string;
mcporter: ResolvedQmdMcporterConfig;
searchMode: MemoryQmdSearchMode;
collections: ResolvedQmdCollection[];
sessions: ResolvedQmdSessionConfig;
@@ -79,6 +87,12 @@ const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = {
maxInjectedChars: 4_000,
timeoutMs: DEFAULT_QMD_TIMEOUT_MS,
};
const DEFAULT_QMD_MCPORTER: ResolvedQmdMcporterConfig = {
enabled: false,
serverName: "qmd",
startDaemon: true,
};
const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = {
default: "deny",
rules: [
@@ -237,6 +251,27 @@ function resolveCustomPaths(
return collections;
}
function resolveMcporterConfig(raw?: MemoryQmdMcporterConfig): ResolvedQmdMcporterConfig {
const parsed: ResolvedQmdMcporterConfig = { ...DEFAULT_QMD_MCPORTER };
if (!raw) {
return parsed;
}
if (raw.enabled !== undefined) {
parsed.enabled = raw.enabled;
}
if (typeof raw.serverName === "string" && raw.serverName.trim()) {
parsed.serverName = raw.serverName.trim();
}
if (raw.startDaemon !== undefined) {
parsed.startDaemon = raw.startDaemon;
}
// When enabled, default startDaemon to true.
if (parsed.enabled && raw.startDaemon === undefined) {
parsed.startDaemon = true;
}
return parsed;
}
function resolveDefaultCollections(
include: boolean,
workspaceDir: string,
@@ -283,6 +318,7 @@ export function resolveMemoryBackendConfig(params: {
const command = parsedCommand?.[0] || rawCommand.split(/\s+/)[0] || "qmd";
const resolved: ResolvedQmdConfig = {
command,
mcporter: resolveMcporterConfig(qmdCfg?.mcporter),
searchMode: resolveSearchMode(qmdCfg?.searchMode),
collections,
includeDefaultMemory,

View File

@@ -185,7 +185,9 @@ export async function createEmbeddingProvider(
continue;
}
// Non-auth errors (e.g., network) are still fatal
throw new Error(message, { cause: err });
const wrapped = new Error(message) as Error & { cause?: unknown };
wrapped.cause = err;
throw wrapped;
}
}
@@ -228,7 +230,9 @@ export async function createEmbeddingProvider(
};
}
// Non-auth errors are still fatal
throw new Error(combinedReason, { cause: fallbackErr });
const wrapped = new Error(combinedReason) as Error & { cause?: unknown };
wrapped.cause = fallbackErr;
throw wrapped;
}
}
// No fallback configured - check if we should degrade to FTS-only
@@ -239,7 +243,9 @@ export async function createEmbeddingProvider(
providerUnavailableReason: reason,
};
}
throw new Error(reason, { cause: primaryErr });
const wrapped = new Error(reason) as Error & { cause?: unknown };
wrapped.cause = primaryErr;
throw wrapped;
}
}

View File

@@ -148,6 +148,8 @@ describe("QmdMemoryManager", () => {
afterEach(async () => {
vi.useRealTimers();
delete process.env.OPENCLAW_STATE_DIR;
delete (globalThis as Record<string, unknown>).__openclawMcporterDaemonStart;
delete (globalThis as Record<string, unknown>).__openclawMcporterColdStartWarned;
});
it("debounces back-to-back sync calls", async () => {
@@ -910,6 +912,170 @@ describe("QmdMemoryManager", () => {
await manager.close();
});
it("runs qmd searches via mcporter and warns when startDaemon=false", async () => {
cfg = {
...cfg,
memory: {
backend: "qmd",
qmd: {
includeDefaultMemory: false,
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
mcporter: { enabled: true, serverName: "qmd", startDaemon: false },
},
},
} as OpenClawConfig;
spawnMock.mockImplementation((cmd: string, args: string[]) => {
const child = createMockChild({ autoClose: false });
if (cmd === "mcporter" && args[0] === "call") {
emitAndClose(child, "stdout", JSON.stringify({ results: [] }));
return child;
}
emitAndClose(child, "stdout", "[]");
return child;
});
const { manager } = await createManager();
logWarnMock.mockClear();
await expect(
manager.search("hello", { sessionKey: "agent:main:slack:dm:u123" }),
).resolves.toEqual([]);
const mcporterCalls = spawnMock.mock.calls.filter((call: unknown[]) => call[0] === "mcporter");
expect(mcporterCalls.length).toBeGreaterThan(0);
expect(mcporterCalls.some((call: unknown[]) => (call[1] as string[])[0] === "daemon")).toBe(
false,
);
expect(logWarnMock).toHaveBeenCalledWith(expect.stringContaining("cold-start"));
await manager.close();
});
it("passes manager-scoped XDG env to mcporter commands", async () => {
cfg = {
...cfg,
memory: {
backend: "qmd",
qmd: {
includeDefaultMemory: false,
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
mcporter: { enabled: true, serverName: "qmd", startDaemon: false },
},
},
} as OpenClawConfig;
spawnMock.mockImplementation((cmd: string, args: string[]) => {
const child = createMockChild({ autoClose: false });
if (cmd === "mcporter" && args[0] === "call") {
emitAndClose(child, "stdout", JSON.stringify({ results: [] }));
return child;
}
emitAndClose(child, "stdout", "[]");
return child;
});
const { manager } = await createManager();
await manager.search("hello", { sessionKey: "agent:main:slack:dm:u123" });
const mcporterCall = spawnMock.mock.calls.find(
(call: unknown[]) => call[0] === "mcporter" && (call[1] as string[])[0] === "call",
);
expect(mcporterCall).toBeDefined();
const spawnOpts = mcporterCall?.[2] as { env?: NodeJS.ProcessEnv } | undefined;
expect(spawnOpts?.env?.XDG_CONFIG_HOME).toContain("/agents/main/qmd/xdg-config");
expect(spawnOpts?.env?.XDG_CACHE_HOME).toContain("/agents/main/qmd/xdg-cache");
await manager.close();
});
it("retries mcporter daemon start after a failure", async () => {
cfg = {
...cfg,
memory: {
backend: "qmd",
qmd: {
includeDefaultMemory: false,
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
mcporter: { enabled: true, serverName: "qmd", startDaemon: true },
},
},
} as OpenClawConfig;
let daemonAttempts = 0;
spawnMock.mockImplementation((cmd: string, args: string[]) => {
const child = createMockChild({ autoClose: false });
if (cmd === "mcporter" && args[0] === "daemon") {
daemonAttempts += 1;
if (daemonAttempts === 1) {
emitAndClose(child, "stderr", "failed", 1);
} else {
emitAndClose(child, "stdout", "");
}
return child;
}
if (cmd === "mcporter" && args[0] === "call") {
emitAndClose(child, "stdout", JSON.stringify({ results: [] }));
return child;
}
emitAndClose(child, "stdout", "[]");
return child;
});
const { manager } = await createManager();
await manager.search("one", { sessionKey: "agent:main:slack:dm:u123" });
await manager.search("two", { sessionKey: "agent:main:slack:dm:u123" });
expect(daemonAttempts).toBe(2);
await manager.close();
});
it("starts the mcporter daemon only once when enabled", async () => {
cfg = {
...cfg,
memory: {
backend: "qmd",
qmd: {
includeDefaultMemory: false,
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
mcporter: { enabled: true, serverName: "qmd", startDaemon: true },
},
},
} as OpenClawConfig;
spawnMock.mockImplementation((cmd: string, args: string[]) => {
const child = createMockChild({ autoClose: false });
if (cmd === "mcporter" && args[0] === "daemon") {
emitAndClose(child, "stdout", "");
return child;
}
if (cmd === "mcporter" && args[0] === "call") {
emitAndClose(child, "stdout", JSON.stringify({ results: [] }));
return child;
}
emitAndClose(child, "stdout", "[]");
return child;
});
const { manager } = await createManager();
await manager.search("one", { sessionKey: "agent:main:slack:dm:u123" });
await manager.search("two", { sessionKey: "agent:main:slack:dm:u123" });
const daemonStarts = spawnMock.mock.calls.filter(
(call: unknown[]) => call[0] === "mcporter" && (call[1] as string[])[0] === "daemon",
);
expect(daemonStarts).toHaveLength(1);
await manager.close();
});
it("fails closed when no managed collections are configured", async () => {
cfg = {
...cfg,

View File

@@ -25,7 +25,11 @@ import type {
} from "./types.js";
type SqliteDatabase = import("node:sqlite").DatabaseSync;
import type { ResolvedMemoryBackendConfig, ResolvedQmdConfig } from "./backend-config.js";
import type {
ResolvedMemoryBackendConfig,
ResolvedQmdConfig,
ResolvedQmdMcporterConfig,
} from "./backend-config.js";
import { parseQmdQueryJson, type QmdQueryResult } from "./qmd-query-parser.js";
const log = createSubsystemLogger("memory");
@@ -425,9 +429,37 @@ export class QmdMemoryManager implements MemorySearchManager {
return [];
}
const qmdSearchCommand = this.qmd.searchMode;
const mcporterEnabled = this.qmd.mcporter.enabled;
let parsed: QmdQueryResult[];
try {
if (collectionNames.length > 1) {
if (mcporterEnabled) {
const tool: "search" | "vector_search" | "deep_search" =
qmdSearchCommand === "search"
? "search"
: qmdSearchCommand === "vsearch"
? "vector_search"
: "deep_search";
const minScore = opts?.minScore ?? 0;
if (collectionNames.length > 1) {
parsed = await this.runMcporterAcrossCollections({
tool,
query: trimmed,
limit,
minScore,
collectionNames,
});
} else {
parsed = await this.runQmdSearchViaMcporter({
mcporter: this.qmd.mcporter,
tool,
query: trimmed,
limit,
minScore,
collection: collectionNames[0],
timeoutMs: this.qmd.limits.timeoutMs,
});
}
} else if (collectionNames.length > 1) {
parsed = await this.runQueryAcrossCollections(
trimmed,
limit,
@@ -443,7 +475,11 @@ export class QmdMemoryManager implements MemorySearchManager {
parsed = parseQmdQueryJson(result.stdout, result.stderr);
}
} catch (err) {
if (qmdSearchCommand !== "query" && this.isUnsupportedQmdOptionError(err)) {
if (
!mcporterEnabled &&
qmdSearchCommand !== "query" &&
this.isUnsupportedQmdOptionError(err)
) {
log.warn(
`qmd ${qmdSearchCommand} does not support configured flags; retrying search with qmd query`,
);
@@ -463,7 +499,8 @@ export class QmdMemoryManager implements MemorySearchManager {
throw fallbackErr instanceof Error ? fallbackErr : new Error(String(fallbackErr));
}
} else {
log.warn(`qmd ${qmdSearchCommand} failed: ${String(err)}`);
const label = mcporterEnabled ? "mcporter/qmd" : `qmd ${qmdSearchCommand}`;
log.warn(`${label} failed: ${String(err)}`);
throw err instanceof Error ? err : new Error(String(err));
}
}
@@ -859,6 +896,169 @@ export class QmdMemoryManager implements MemorySearchManager {
});
}
private async ensureMcporterDaemonStarted(mcporter: ResolvedQmdMcporterConfig): Promise<void> {
if (!mcporter.enabled) {
return;
}
if (!mcporter.startDaemon) {
type McporterWarnGlobal = typeof globalThis & {
__openclawMcporterColdStartWarned?: boolean;
};
const g: McporterWarnGlobal = globalThis;
if (!g.__openclawMcporterColdStartWarned) {
g.__openclawMcporterColdStartWarned = true;
log.warn(
"mcporter qmd bridge enabled but startDaemon=false; each query may cold-start QMD MCP. Consider setting memory.qmd.mcporter.startDaemon=true to keep it warm.",
);
}
return;
}
type McporterGlobal = typeof globalThis & {
__openclawMcporterDaemonStart?: Promise<void>;
};
const g: McporterGlobal = globalThis;
if (!g.__openclawMcporterDaemonStart) {
g.__openclawMcporterDaemonStart = (async () => {
try {
await this.runMcporter(["daemon", "start"], { timeoutMs: 10_000 });
} catch (err) {
log.warn(`mcporter daemon start failed: ${String(err)}`);
// Allow future searches to retry daemon start on transient failures.
delete g.__openclawMcporterDaemonStart;
}
})();
}
await g.__openclawMcporterDaemonStart;
}
private async runMcporter(
args: string[],
opts?: { timeoutMs?: number },
): Promise<{ stdout: string; stderr: string }> {
return await new Promise((resolve, reject) => {
const child = spawn("mcporter", args, {
// Keep mcporter and direct qmd commands on the same agent-scoped XDG state.
env: this.env,
cwd: this.workspaceDir,
});
let stdout = "";
let stderr = "";
let stdoutTruncated = false;
let stderrTruncated = false;
const timer = opts?.timeoutMs
? setTimeout(() => {
child.kill("SIGKILL");
reject(new Error(`mcporter ${args.join(" ")} timed out after ${opts.timeoutMs}ms`));
}, opts.timeoutMs)
: null;
child.stdout.on("data", (data) => {
const next = appendOutputWithCap(stdout, data.toString("utf8"), this.maxQmdOutputChars);
stdout = next.text;
stdoutTruncated = stdoutTruncated || next.truncated;
});
child.stderr.on("data", (data) => {
const next = appendOutputWithCap(stderr, data.toString("utf8"), this.maxQmdOutputChars);
stderr = next.text;
stderrTruncated = stderrTruncated || next.truncated;
});
child.on("error", (err) => {
if (timer) {
clearTimeout(timer);
}
reject(err);
});
child.on("close", (code) => {
if (timer) {
clearTimeout(timer);
}
if (stdoutTruncated || stderrTruncated) {
reject(
new Error(
`mcporter ${args.join(" ")} produced too much output (limit ${this.maxQmdOutputChars} chars)`,
),
);
return;
}
if (code === 0) {
resolve({ stdout, stderr });
} else {
reject(
new Error(`mcporter ${args.join(" ")} failed (code ${code}): ${stderr || stdout}`),
);
}
});
});
}
private async runQmdSearchViaMcporter(params: {
mcporter: ResolvedQmdMcporterConfig;
tool: "search" | "vector_search" | "deep_search";
query: string;
limit: number;
minScore: number;
collection?: string;
timeoutMs: number;
}): Promise<QmdQueryResult[]> {
await this.ensureMcporterDaemonStarted(params.mcporter);
const selector = `${params.mcporter.serverName}.${params.tool}`;
const callArgs: Record<string, unknown> = {
query: params.query,
limit: params.limit,
minScore: params.minScore,
};
if (params.collection) {
callArgs.collection = params.collection;
}
const result = await this.runMcporter(
[
"call",
selector,
"--args",
JSON.stringify(callArgs),
"--output",
"json",
"--timeout",
String(Math.max(0, params.timeoutMs)),
],
{ timeoutMs: Math.max(params.timeoutMs + 2_000, 5_000) },
);
const parsedUnknown: unknown = JSON.parse(result.stdout);
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const structured =
isRecord(parsedUnknown) && isRecord(parsedUnknown.structuredContent)
? parsedUnknown.structuredContent
: parsedUnknown;
const results: unknown[] =
isRecord(structured) && Array.isArray(structured.results)
? (structured.results as unknown[])
: Array.isArray(structured)
? structured
: [];
const out: QmdQueryResult[] = [];
for (const item of results) {
if (!isRecord(item)) {
continue;
}
const docidRaw = item.docid;
const docid = typeof docidRaw === "string" ? docidRaw.replace(/^#/, "").trim() : "";
if (!docid) {
continue;
}
const scoreRaw = item.score;
const score = typeof scoreRaw === "number" ? scoreRaw : Number(scoreRaw);
const snippet = typeof item.snippet === "string" ? item.snippet : "";
out.push({ docid, score: Number.isFinite(score) ? score : 0, snippet });
}
return out;
}
private async readPartialText(
absPath: string,
from?: number,
@@ -1407,6 +1607,39 @@ export class QmdMemoryManager implements MemorySearchManager {
return [...bestByDocId.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0));
}
private async runMcporterAcrossCollections(params: {
tool: "search" | "vector_search" | "deep_search";
query: string;
limit: number;
minScore: number;
collectionNames: string[];
}): Promise<QmdQueryResult[]> {
const bestByDocId = new Map<string, QmdQueryResult>();
for (const collectionName of params.collectionNames) {
const parsed = await this.runQmdSearchViaMcporter({
mcporter: this.qmd.mcporter,
tool: params.tool,
query: params.query,
limit: params.limit,
minScore: params.minScore,
collection: collectionName,
timeoutMs: this.qmd.limits.timeoutMs,
});
for (const entry of parsed) {
if (typeof entry.docid !== "string" || !entry.docid.trim()) {
continue;
}
const prev = bestByDocId.get(entry.docid);
const prevScore = typeof prev?.score === "number" ? prev.score : Number.NEGATIVE_INFINITY;
const nextScore = typeof entry.score === "number" ? entry.score : Number.NEGATIVE_INFINITY;
if (!prev || nextScore > prevScore) {
bestByDocId.set(entry.docid, entry);
}
}
}
return [...bestByDocId.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0));
}
private listManagedCollectionNames(): string[] {
const seen = new Set<string>();
const names: string[] = [];