diff --git a/CHANGELOG.md b/CHANGELOG.md index 307a9e18004..51b5551cf98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Docs: https://docs.openclaw.ai - CLI/logs: fall back to the configured Gateway file log when implicit loopback Gateway connections close or time out before or during `logs.tail`, so `openclaw logs` still works while diagnosing local-model Gateway disconnects. Refs #74078. Thanks @sakalaboator. - MCP/plugins: stringify non-array plugin tool results with chat-content coercion instead of default object stringification, so MCP callers receive useful JSON/text content from plugin tools. Thanks @vincentkoc. +- Active Memory/QMD: run QMD boot refresh through a one-shot subprocess path, preserve interactive file watching, and align watcher dependency/build ignores with QMD's scanner so gateway startup avoids arming long-lived QMD watchers. Thanks @codexGW. - Channels/Discord: remove Discord-owned queued-run timeout replies through the shared channel lifecycle queue while preserving message ordering and compatibility timeout constants, so long Discord turns stay governed by session/tool/runtime lifecycle instead of channel fallback errors. Thanks @codexGW. - Agents/tools: clamp `process.poll` waits to 30 seconds, advertise that cap in the tool schema, and honor abort signals while waiting, so long command polls cannot pin agent responsiveness after cancellation. Thanks @vincentkoc. - Plugin SDK: add tracked Discord component-message helpers and a Telegram account-resolution compatibility facade, so existing plugins using those subpaths resolve while new plugins stay on generic channel SDK contracts. Thanks @vincentkoc. diff --git a/docs/concepts/memory-qmd.md b/docs/concepts/memory-qmd.md index 502a672e157..860ae1a5de1 100644 --- a/docs/concepts/memory-qmd.md +++ b/docs/concepts/memory-qmd.md @@ -52,9 +52,15 @@ present. - OpenClaw creates collections from your workspace memory files and any configured `memory.qmd.paths`, then runs `qmd update` on boot and - periodically (default every 5 minutes). Semantic modes also run `qmd embed`. + periodically (default every 5 minutes). These refreshes run through QMD + subprocesses, not an in-process filesystem crawl. Semantic modes also run + `qmd embed`. - The default workspace collection tracks `MEMORY.md` plus the `memory/` tree. Lowercase `memory.md` is not indexed as a root memory file. +- QMD's own scanner ignores hidden paths and common dependency/build + directories such as `.git`, `.cache`, `node_modules`, `vendor`, `dist`, and + `build`. Boot refreshes use a one-shot QMD subprocess path instead of + creating the full long-lived in-process watcher during gateway startup. - Boot refresh runs in the background so chat startup is not blocked. - Searches use the configured `searchMode` (default: `search`; also supports `vsearch` and `query`). `search` is BM25-only, so OpenClaw skips semantic diff --git a/docs/reference/memory-config.md b/docs/reference/memory-config.md index 4a985403345..84fa911412e 100644 --- a/docs/reference/memory-config.md +++ b/docs/reference/memory-config.md @@ -497,7 +497,7 @@ QMD model overrides stay on the QMD side, not OpenClaw config. If you need to ov | ------------------------- | --------- | ------- | ------------------------------------- | | `update.interval` | `string` | `5m` | Refresh interval | | `update.debounceMs` | `number` | `15000` | Debounce file changes | - | `update.onBoot` | `boolean` | `true` | Refresh on startup | + | `update.onBoot` | `boolean` | `true` | Refresh on startup in a QMD subprocess | | `update.waitForBootSync` | `boolean` | `false` | Block startup until refresh completes | | `update.embedInterval` | `string` | -- | Separate embed cadence | | `update.commandTimeoutMs` | `number` | -- | Timeout for QMD commands | @@ -545,6 +545,8 @@ QMD model overrides stay on the QMD side, not OpenClaw config. If you need to ov +QMD boot refreshes use a one-shot subprocess path during gateway startup. The long-lived QMD manager still owns the regular file watcher and interval timers when memory search is opened for interactive use. + ### Full QMD example ```json5 diff --git a/extensions/memory-core/src/memory/qmd-manager.test.ts b/extensions/memory-core/src/memory/qmd-manager.test.ts index a72862ce5ae..612c702c38f 100644 --- a/extensions/memory-core/src/memory/qmd-manager.test.ts +++ b/extensions/memory-core/src/memory/qmd-manager.test.ts @@ -432,6 +432,18 @@ describe("QmdMemoryManager", () => { }; const initialUpdateCalls = spawnMock.mock.calls.filter((call) => call[1]?.[0] === "update"); expect(initialUpdateCalls).toHaveLength(0); + const [, watchOptions] = watchMock.mock.calls[0] as unknown as [ + string[], + { ignored?: (watchPath: string) => boolean }, + ]; + expect(watchOptions.ignored?.(path.join(workspaceDir, "node_modules", "pkg", "note.md"))).toBe( + true, + ); + expect(watchOptions.ignored?.(path.join(workspaceDir, ".cache", "qmd", "note.md"))).toBe(true); + expect(watchOptions.ignored?.(path.join(workspaceDir, "vendor", "pkg", "note.md"))).toBe(true); + expect(watchOptions.ignored?.(path.join(workspaceDir, "dist", "note.md"))).toBe(true); + expect(watchOptions.ignored?.(path.join(workspaceDir, "build", "note.md"))).toBe(true); + expect(watchOptions.ignored?.(path.join(workspaceDir, "notes.md"))).toBe(false); watcher.emit("change", path.join(workspaceDir, "notes.md")); expect(manager.status().dirty).toBe(true); diff --git a/extensions/memory-core/src/memory/qmd-manager.ts b/extensions/memory-core/src/memory/qmd-manager.ts index 3a6eb598632..ac5bc0f646e 100644 --- a/extensions/memory-core/src/memory/qmd-manager.ts +++ b/extensions/memory-core/src/memory/qmd-manager.ts @@ -79,7 +79,11 @@ const QMD_EMBED_QUEUE_KEY = Symbol.for("openclaw.qmdEmbedQueueTail"); const QMD_UPDATE_QUEUE_KEY = Symbol.for("openclaw.qmdUpdateQueueState"); const IGNORED_MEMORY_WATCH_DIR_NAMES = new Set([ ".git", + ".cache", "node_modules", + "vendor", + "dist", + "build", ".pnpm-store", ".venv", "venv", @@ -402,6 +406,7 @@ export class QmdMemoryManager implements MemorySearchManager { } private async initialize(mode: QmdManagerMode): Promise { + const startTime = Date.now(); this.bootstrapCollections(); if (mode === "status") { return; @@ -424,10 +429,16 @@ export class QmdMemoryManager implements MemorySearchManager { await this.ensureCollections(); if (mode === "cli") { + log.info( + `qmd manager initialized for agent "${this.agentId}" mode=cli collections=${this.qmd.collections.length} durationMs=${Date.now() - startTime}`, + ); return; } this.ensureWatcher(); + log.info( + `qmd manager initialized for agent "${this.agentId}" mode=full collections=${this.qmd.collections.length} durationMs=${Date.now() - startTime}`, + ); if (this.qmd.update.onBoot) { const bootRun = this.runUpdate("boot", true); @@ -1459,6 +1470,10 @@ export class QmdMemoryManager implements MemorySearchManager { return; } const run = async () => { + const startTime = Date.now(); + log.debug( + `qmd sync started for agent "${this.agentId}" reason=${reason} force=${force === true}`, + ); await this.withQmdUpdateQueue(async () => { if (this.closed) { return; @@ -1492,6 +1507,9 @@ export class QmdMemoryManager implements MemorySearchManager { } this.lastUpdateAt = Date.now(); this.docPathCache.clear(); + log.info( + `qmd sync completed for agent "${this.agentId}" reason=${reason} durationMs=${Date.now() - startTime}`, + ); }; this.pendingUpdate = run().finally(() => { this.pendingUpdate = null; @@ -1513,7 +1531,10 @@ export class QmdMemoryManager implements MemorySearchManager { if (watchPaths.size === 0) { return; } - this.watcher = chokidar.watch(Array.from(watchPaths), { + const watchPathList = Array.from(watchPaths); + const startTime = Date.now(); + log.info(`qmd watcher starting for agent "${this.agentId}" paths=${watchPathList.length}`); + this.watcher = chokidar.watch(watchPathList, { ignoreInitial: true, ignored: (watchPath) => shouldIgnoreMemoryWatchPath(watchPath), awaitWriteFinish: { @@ -1528,6 +1549,11 @@ export class QmdMemoryManager implements MemorySearchManager { this.watcher.on("add", markDirty); this.watcher.on("change", markDirty); this.watcher.on("unlink", markDirty); + this.watcher.once("ready", () => { + log.info( + `qmd watcher ready for agent "${this.agentId}" paths=${watchPathList.length} durationMs=${Date.now() - startTime}`, + ); + }); } private resolveCollectionWatchPath(collection: ManagedCollection): string { diff --git a/src/gateway/server-startup-memory.test.ts b/src/gateway/server-startup-memory.test.ts index 21ea1896c36..e52266dbc97 100644 --- a/src/gateway/server-startup-memory.test.ts +++ b/src/gateway/server-startup-memory.test.ts @@ -22,6 +22,14 @@ function createGatewayLogMock() { return { info: vi.fn(), warn: vi.fn() }; } +function createQmdManagerMock() { + return { + search: vi.fn(), + sync: vi.fn(async () => undefined), + close: vi.fn(async () => undefined), + }; +} + describe("startGatewayMemoryBackend", () => { beforeEach(() => { getMemorySearchManagerMock.mockClear(); @@ -41,7 +49,7 @@ describe("startGatewayMemoryBackend", () => { expect(log.warn).not.toHaveBeenCalled(); }); - it("initializes qmd backend for the default and explicitly configured agents", async () => { + it("runs qmd boot sync for the default and explicitly configured agents", async () => { const cfg = createQmdConfig({ list: [ { id: "ops", default: true }, @@ -50,15 +58,23 @@ describe("startGatewayMemoryBackend", () => { ], }); const log = createGatewayLogMock(); - getMemorySearchManagerMock.mockResolvedValue({ manager: { search: vi.fn() } }); + getMemorySearchManagerMock.mockResolvedValue({ manager: createQmdManagerMock() }); await startGatewayMemoryBackend({ cfg, log }); expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(2); - expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(1, { cfg, agentId: "ops" }); - expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(2, { cfg, agentId: "main" }); + expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(1, { + cfg, + agentId: "ops", + purpose: "cli", + }); + expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(2, { + cfg, + agentId: "main", + purpose: "cli", + }); expect(log.info).toHaveBeenCalledWith( - 'qmd memory startup initialization armed for 2 agents: "ops", "main"', + 'qmd memory startup boot sync completed for 2 agents: "ops", "main"', ); expect(log.info).toHaveBeenCalledWith( 'qmd memory startup initialization deferred for 1 agent: "lazy"', @@ -72,15 +88,23 @@ describe("startGatewayMemoryBackend", () => { list: [{ id: "ops", default: true }, { id: "main" }], }); const log = createGatewayLogMock(); - getMemorySearchManagerMock.mockResolvedValue({ manager: { search: vi.fn() } }); + getMemorySearchManagerMock.mockResolvedValue({ manager: createQmdManagerMock() }); await startGatewayMemoryBackend({ cfg, log }); expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(2); - expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(1, { cfg, agentId: "ops" }); - expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(2, { cfg, agentId: "main" }); + expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(1, { + cfg, + agentId: "ops", + purpose: "cli", + }); + expect(getMemorySearchManagerMock).toHaveBeenNthCalledWith(2, { + cfg, + agentId: "main", + purpose: "cli", + }); expect(log.info).toHaveBeenCalledWith( - 'qmd memory startup initialization armed for 2 agents: "ops", "main"', + 'qmd memory startup boot sync completed for 2 agents: "ops", "main"', ); expect(log.info).not.toHaveBeenCalledWith(expect.stringContaining("deferred")); }); @@ -95,7 +119,7 @@ describe("startGatewayMemoryBackend", () => { const log = createGatewayLogMock(); getMemorySearchManagerMock .mockResolvedValueOnce({ manager: null, error: "qmd missing" }) - .mockResolvedValueOnce({ manager: { search: vi.fn() } }); + .mockResolvedValueOnce({ manager: createQmdManagerMock() }); await startGatewayMemoryBackend({ cfg, log }); @@ -103,7 +127,7 @@ describe("startGatewayMemoryBackend", () => { 'qmd memory startup initialization failed for agent "main": qmd missing', ); expect(log.info).toHaveBeenCalledWith( - 'qmd memory startup initialization armed for 1 agent: "ops"', + 'qmd memory startup boot sync completed for 1 agent: "ops"', ); }); @@ -116,14 +140,18 @@ describe("startGatewayMemoryBackend", () => { ], }); const log = createGatewayLogMock(); - getMemorySearchManagerMock.mockResolvedValue({ manager: { search: vi.fn() } }); + getMemorySearchManagerMock.mockResolvedValue({ manager: createQmdManagerMock() }); await startGatewayMemoryBackend({ cfg, log }); expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(1); - expect(getMemorySearchManagerMock).toHaveBeenCalledWith({ cfg, agentId: "main" }); + expect(getMemorySearchManagerMock).toHaveBeenCalledWith({ + cfg, + agentId: "main", + purpose: "cli", + }); expect(log.info).toHaveBeenCalledWith( - 'qmd memory startup initialization armed for 1 agent: "main"', + 'qmd memory startup boot sync completed for 1 agent: "main"', ); expect(log.warn).not.toHaveBeenCalled(); }); diff --git a/src/gateway/server-startup-memory.ts b/src/gateway/server-startup-memory.ts index 80d90425c16..cbe082eac88 100644 --- a/src/gateway/server-startup-memory.ts +++ b/src/gateway/server-startup-memory.ts @@ -8,8 +8,8 @@ import { import { getActiveMemorySearchManager } from "../plugins/memory-runtime.js"; import { normalizeAgentId } from "../routing/session-key.js"; -function shouldStartQmdBackgroundWork(qmd: ResolvedQmdConfig): boolean { - return qmd.update.onBoot || qmd.update.intervalMs > 0 || qmd.update.embedIntervalMs > 0; +function shouldRunQmdStartupBootSync(qmd: ResolvedQmdConfig): boolean { + return qmd.update.onBoot; } function hasExplicitAgentMemorySearchConfig(cfg: OpenClawConfig, agentId: string): boolean { @@ -53,7 +53,7 @@ export async function startGatewayMemoryBackend(params: { if (resolved.backend !== "qmd" || !resolved.qmd) { continue; } - if (!shouldStartQmdBackgroundWork(resolved.qmd)) { + if (!shouldRunQmdStartupBootSync(resolved.qmd)) { continue; } if ( @@ -67,18 +67,34 @@ export async function startGatewayMemoryBackend(params: { continue; } - const { manager, error } = await getActiveMemorySearchManager({ cfg: params.cfg, agentId }); + const { manager, error } = await getActiveMemorySearchManager({ + cfg: params.cfg, + agentId, + purpose: "cli", + }); if (!manager) { params.log.warn( `qmd memory startup initialization failed for agent "${agentId}": ${error ?? "unknown error"}`, ); continue; } + try { + await manager.sync?.({ reason: "boot", force: true }); + } catch (err) { + params.log.warn(`qmd memory startup boot sync failed for agent "${agentId}": ${String(err)}`); + continue; + } finally { + await manager.close?.().catch((err) => { + params.log.warn( + `qmd memory startup manager close failed for agent "${agentId}": ${String(err)}`, + ); + }); + } armedAgentIds.push(agentId); } if (armedAgentIds.length > 0) { params.log.info?.( - `qmd memory startup initialization armed for ${formatAgentCount(armedAgentIds.length)}: ${armedAgentIds + `qmd memory startup boot sync completed for ${formatAgentCount(armedAgentIds.length)}: ${armedAgentIds .map((agentId) => `"${agentId}"`) .join(", ")}`, ); diff --git a/src/plugins/memory-runtime.ts b/src/plugins/memory-runtime.ts index cc5a6fabf8c..23c428e4f96 100644 --- a/src/plugins/memory-runtime.ts +++ b/src/plugins/memory-runtime.ts @@ -33,7 +33,7 @@ function ensureMemoryRuntime(cfg?: OpenClawConfig) { export async function getActiveMemorySearchManager(params: { cfg: OpenClawConfig; agentId: string; - purpose?: "default" | "status"; + purpose?: "default" | "status" | "cli"; }) { const runtime = ensureMemoryRuntime(params.cfg); if (!runtime) { diff --git a/src/plugins/memory-state.ts b/src/plugins/memory-state.ts index 429c01dedb8..216a13b07bd 100644 --- a/src/plugins/memory-state.ts +++ b/src/plugins/memory-state.ts @@ -98,7 +98,7 @@ export type MemoryPluginRuntime = { getMemorySearchManager(params: { cfg: OpenClawConfig; agentId: string; - purpose?: "default" | "status"; + purpose?: "default" | "status" | "cli"; }): Promise<{ manager: RegisteredMemorySearchManager | null; error?: string;