From 864c4f7ff492f0f514c12557d44f0d6b509231fc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Apr 2026 01:23:40 +0100 Subject: [PATCH] fix(memory-core): bound fallback vector chunk scoring - stream fallback Memory Core vector scoring with SQLite iterate() and a bounded top-K result set - add regression coverage and live-main lint/boundary helper repairs - supersedes #73069 Thanks @parkertoddbrooks. --- CHANGELOG.md | 1 + .../src/memory/manager-search.test.ts | 94 ++++++++++++++++++- .../memory-core/src/memory/manager-search.ts | 82 ++++++++-------- test/extension-test-boundary.test.ts | 2 +- test/scripts/lint-suppressions.test.ts | 3 + 5 files changed, 141 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5807e19a98b..4fdd30ea137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ Docs: https://docs.openclaw.ai - CLI/skills: resolve workspace-backed skills commands from `--agent`, then the current agent workspace, before falling back to the default agent, so multi-agent ClawHub installs, updates, and status checks stay scoped to the active workspace. Fixes #56161; carries forward #72726. Thanks @langbowang and @luyao618. - Plugin SDK: fall back from partial bundled plugin directory overrides to package source public surfaces while preserving `OPENCLAW_DISABLE_BUNDLED_PLUGINS` as a hard disable. (#72817) Thanks @serkonyc. - Agents/ACPX: stop forwarding Codex ACP timeout config controls that Codex rejects while preserving OpenClaw's run-timeout watchdog for ACP subagents. Fixes #73052. Thanks @pfrederiksen and @richa65. +- Memory Core: stream fallback vector search scoring with a bounded top-K result set so large indexes do not materialize every chunk embedding when sqlite-vec is unavailable. (#73069) Thanks @parkertoddbrooks. - Memory/Ollama: add `memorySearch.remote.nonBatchConcurrency` for inline embedding indexing, default Ollama non-batch indexing to one request at a time, and keep batch concurrency separate from non-batch concurrency so local embedding backfills avoid timeout storms on smaller hosts. Carries forward #57733. Thanks @itilys. - Docs/tools: clarify that `tools.profile: "messaging"` is intentionally narrow and that `tools.profile: "full"` is the unrestricted baseline for broader command/control access. Carries forward #39954. Thanks @posigit. - Control UI/Agents: redact tool-call args, partial/final results, derived exec output, and configured custom secret patterns before streaming tool events to the Control UI, so tool output cannot expose provider or channel credentials. Fixes #72283. (#72319) Thanks @volcano303 and @BunsDev. diff --git a/extensions/memory-core/src/memory/manager-search.test.ts b/extensions/memory-core/src/memory/manager-search.test.ts index 24b12eeb4d3..b915848f2ea 100644 --- a/extensions/memory-core/src/memory/manager-search.test.ts +++ b/extensions/memory-core/src/memory/manager-search.test.ts @@ -3,7 +3,7 @@ import { loadSqliteVecExtension, requireNodeSqlite, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { bm25RankToScore, buildFtsQuery } from "./hybrid.js"; import { searchKeyword, searchVector } from "./manager-search.js"; @@ -182,6 +182,98 @@ describe("searchKeyword trigram fallback", () => { describe("searchVector sqlite-vec KNN", () => { const { DatabaseSync } = requireNodeSqlite(); + it("streams fallback chunk scoring without materializing candidates", async () => { + type ChunkRow = { + id: string; + path: string; + start_line: number; + end_line: number; + text: string; + embedding: string; + source: string; + }; + type StatementWithAll = { + all: (...params: unknown[]) => ChunkRow[]; + }; + + const db = new DatabaseSync(":memory:"); + try { + ensureMemoryIndexSchema({ + db, + embeddingCacheTable: "embedding_cache", + cacheEnabled: false, + ftsTable: "chunks_fts", + ftsEnabled: false, + }); + + const insertChunk = db.prepare( + "INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ); + const addChunk = (params: { id: string; model: string; vector: [number, number] }) => { + insertChunk.run( + params.id, + `memory/${params.id}.md`, + "memory", + 1, + 1, + params.id, + params.model, + `chunk ${params.id}`, + JSON.stringify(params.vector), + 1, + ); + }; + addChunk({ id: "target-1", model: "target-model", vector: [1, 0] }); + addChunk({ id: "target-2", model: "target-model", vector: [0.8, 0.2] }); + addChunk({ id: "target-3", model: "target-model", vector: [0, 1] }); + addChunk({ id: "other-1", model: "other-model", vector: [1, 0] }); + + const prepareTarget = db as unknown as { prepare: (sql: string) => unknown }; + const originalPrepare = prepareTarget.prepare.bind(db); + const chunkRows = ( + originalPrepare( + "SELECT id, path, start_line, end_line, text, embedding, source\n" + + " FROM chunks\n" + + " WHERE model = ?", + ) as StatementWithAll + ).all("target-model"); + const prepareSpy = vi.spyOn(prepareTarget, "prepare").mockImplementation((sql: string) => { + if ( + sql.includes("SELECT id, path, start_line, end_line, text, embedding, source") && + sql.includes("FROM chunks") + ) { + return { + all: () => { + throw new Error("fallback vector search must stream rows via iterate()"); + }, + iterate: () => chunkRows[Symbol.iterator](), + }; + } + return originalPrepare(sql); + }); + + try { + const results = await searchVector({ + db, + vectorTable: "chunks_vec", + providerModel: "target-model", + queryVec: [1, 0], + limit: 2, + snippetMaxChars: 200, + ensureVectorReady: async () => false, + sourceFilterVec: { sql: "", params: [] }, + sourceFilterChunks: { sql: "", params: [] }, + }); + + expect(results.map((row) => row.id)).toEqual(["target-1", "target-2"]); + } finally { + prepareSpy.mockRestore(); + } + } finally { + db.close(); + } + }); + it("fills the requested limit after model filters prune nearest KNN candidates", async () => { const db = new DatabaseSync(":memory:", { allowExtension: true }); try { diff --git a/extensions/memory-core/src/memory/manager-search.ts b/extensions/memory-core/src/memory/manager-search.ts index e3e47f69d14..e30318a24e9 100644 --- a/extensions/memory-core/src/memory/manager-search.ts +++ b/extensions/memory-core/src/memory/manager-search.ts @@ -205,51 +205,34 @@ export async function searchVector(params: { })); } - const candidates = listChunks({ + return searchChunksByEmbedding({ db: params.db, providerModel: params.providerModel, sourceFilter: params.sourceFilterChunks, + queryVec: params.queryVec, + limit: params.limit, + snippetMaxChars: params.snippetMaxChars, }); - const scored = candidates - .map((chunk) => ({ - chunk, - score: cosineSimilarity(params.queryVec, chunk.embedding), - })) - .filter((entry) => Number.isFinite(entry.score)); - return scored - .toSorted((a, b) => b.score - a.score) - .slice(0, params.limit) - .map((entry) => ({ - id: entry.chunk.id, - path: entry.chunk.path, - startLine: entry.chunk.startLine, - endLine: entry.chunk.endLine, - score: entry.score, - snippet: truncateUtf16Safe(entry.chunk.text, params.snippetMaxChars), - source: entry.chunk.source, - })); } -export function listChunks(params: { +export function searchChunksByEmbedding(params: { db: DatabaseSync; providerModel: string; sourceFilter: { sql: string; params: SearchSource[] }; -}): Array<{ - id: string; - path: string; - startLine: number; - endLine: number; - text: string; - embedding: number[]; - source: SearchSource; -}> { + queryVec: number[]; + limit: number; + snippetMaxChars: number; +}): SearchRowResult[] { + if (params.limit <= 0) { + return []; + } const rows = params.db .prepare( `SELECT id, path, start_line, end_line, text, embedding, source\n` + ` FROM chunks\n` + ` WHERE model = ?${params.sourceFilter.sql}`, ) - .all(params.providerModel, ...params.sourceFilter.params) as Array<{ + .iterate(params.providerModel, ...params.sourceFilter.params) as IterableIterator<{ id: string; path: string; start_line: number; @@ -259,15 +242,36 @@ export function listChunks(params: { source: SearchSource; }>; - return rows.map((row) => ({ - id: row.id, - path: row.path, - startLine: row.start_line, - endLine: row.end_line, - text: row.text, - embedding: parseEmbedding(row.embedding), - source: row.source, - })); + const topResults: SearchRowResult[] = []; + for (const row of rows) { + const score = cosineSimilarity(params.queryVec, parseEmbedding(row.embedding)); + if (!Number.isFinite(score)) { + continue; + } + const result: SearchRowResult = { + id: row.id, + path: row.path, + startLine: row.start_line, + endLine: row.end_line, + score, + snippet: truncateUtf16Safe(row.text, params.snippetMaxChars), + source: row.source, + }; + if (topResults.length < params.limit) { + topResults.push(result); + if (topResults.length === params.limit) { + topResults.sort((a, b) => b.score - a.score); + } + continue; + } + const lowest = topResults.at(-1); + if (lowest && result.score > lowest.score) { + topResults[topResults.length - 1] = result; + topResults.sort((a, b) => b.score - a.score); + } + } + topResults.sort((a, b) => b.score - a.score); + return topResults; } export async function searchKeyword(params: { diff --git a/test/extension-test-boundary.test.ts b/test/extension-test-boundary.test.ts index 2c9dd653d3f..dd04cc511ed 100644 --- a/test/extension-test-boundary.test.ts +++ b/test/extension-test-boundary.test.ts @@ -1,8 +1,8 @@ import fs from "node:fs"; import path from "node:path"; +import { GUARDED_EXTENSION_PUBLIC_SURFACE_BASENAMES } from "openclaw/plugin-sdk/plugin-test-contracts"; import { describe, expect, it } from "vitest"; import { BUNDLED_PLUGIN_PATH_PREFIX } from "./helpers/bundled-plugin-paths.js"; -import { GUARDED_EXTENSION_PUBLIC_SURFACE_BASENAMES } from "./helpers/plugins/public-artifacts.js"; const repoRoot = path.resolve(import.meta.dirname, ".."); const ALLOWED_EXTENSION_PUBLIC_SURFACE_BASENAMES = new Set( diff --git a/test/scripts/lint-suppressions.test.ts b/test/scripts/lint-suppressions.test.ts index f97ee39279b..bd66cb28ff3 100644 --- a/test/scripts/lint-suppressions.test.ts +++ b/test/scripts/lint-suppressions.test.ts @@ -114,6 +114,9 @@ describe("production lint suppressions", () => { "src/plugin-sdk/facade-loader.ts|typescript/no-unnecessary-type-parameters|1", "src/plugin-sdk/facade-runtime.ts|typescript/no-unnecessary-type-parameters|3", "src/plugin-sdk/qa-runner-runtime.ts|typescript/no-unnecessary-type-parameters|1", + "src/plugin-sdk/test-helpers/package-manifest-contract.ts|typescript/no-unnecessary-type-parameters|1", + "src/plugin-sdk/test-helpers/public-surface-loader.ts|typescript/no-unnecessary-type-parameters|1", + "src/plugin-sdk/test-helpers/subagent-hooks.ts|typescript/no-unnecessary-type-parameters|1", "src/plugins/hooks.ts|typescript/no-unnecessary-type-parameters|1", "src/plugins/host-hook-runtime.ts|typescript/no-unnecessary-type-parameters|2", "src/plugins/host-hooks.ts|typescript/no-unnecessary-type-parameters|1",