mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-22 23:41:07 +00:00
perf: reduce memory startup overhead
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
getMemorySearchManagerMockCalls,
|
||||
getReadAgentMemoryFileMockCalls,
|
||||
resetMemoryToolMockState,
|
||||
setMemoryBackend,
|
||||
setMemoryReadFileImpl,
|
||||
@@ -134,4 +136,18 @@ describe("memory tools", () => {
|
||||
path: "memory/2026-02-19.md",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the builtin direct memory file path for memory_get", async () => {
|
||||
setMemoryBackend("builtin");
|
||||
const tool = createMemoryGetToolOrThrow();
|
||||
|
||||
const result = await tool.execute("call_builtin_fast_path", { path: "memory/2026-02-19.md" });
|
||||
|
||||
expect(result.details).toEqual({
|
||||
text: "",
|
||||
path: "memory/2026-02-19.md",
|
||||
});
|
||||
expect(getReadAgentMemoryFileMockCalls()).toBe(1);
|
||||
expect(getMemorySearchManagerMockCalls()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { OpenClawConfig } from "../../config/config.js";
|
||||
import type { MemoryCitationsMode } from "../../config/types.memory.js";
|
||||
import { resolveMemoryBackendConfig } from "../../memory/backend-config.js";
|
||||
import { getMemorySearchManager } from "../../memory/index.js";
|
||||
import { readAgentMemoryFile } from "../../memory/read-file.js";
|
||||
import type { MemorySearchResult } from "../../memory/types.js";
|
||||
import { parseAgentSessionKey } from "../../routing/session-key.js";
|
||||
import { resolveSessionAgentId } from "../agent-scope.js";
|
||||
@@ -165,6 +166,22 @@ export function createMemoryGetTool(options: {
|
||||
const relPath = readStringParam(params, "path", { required: true });
|
||||
const from = readNumberParam(params, "from", { integer: true });
|
||||
const lines = readNumberParam(params, "lines", { integer: true });
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId });
|
||||
if (resolved.backend === "builtin") {
|
||||
try {
|
||||
const result = await readAgentMemoryFile({
|
||||
cfg,
|
||||
agentId,
|
||||
relPath,
|
||||
from: from ?? undefined,
|
||||
lines: lines ?? undefined,
|
||||
});
|
||||
return jsonResult(result);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return jsonResult({ path: relPath, text: "", disabled: true, error: message });
|
||||
}
|
||||
}
|
||||
const memory = await getMemoryManagerContextWithPurpose({
|
||||
cfg,
|
||||
agentId,
|
||||
|
||||
@@ -28,6 +28,11 @@ export type PluginAutoEnableResult = {
|
||||
changes: string[];
|
||||
};
|
||||
|
||||
const EMPTY_PLUGIN_MANIFEST_REGISTRY: PluginManifestRegistry = {
|
||||
plugins: [],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const PROVIDER_PLUGIN_IDS: Array<{ pluginId: string; providerId: string }> = [
|
||||
{ pluginId: "google", providerId: "google-gemini-cli" },
|
||||
{ pluginId: "qwen-portal-auth", providerId: "qwen-portal" },
|
||||
@@ -330,6 +335,22 @@ function collectCandidateChannelIds(cfg: OpenClawConfig, env: NodeJS.ProcessEnv)
|
||||
return Array.from(channelIds);
|
||||
}
|
||||
|
||||
function configMayNeedPluginManifestRegistry(cfg: OpenClawConfig): boolean {
|
||||
const configuredChannels = cfg.channels as Record<string, unknown> | undefined;
|
||||
if (!configuredChannels || typeof configuredChannels !== "object") {
|
||||
return false;
|
||||
}
|
||||
for (const key of Object.keys(configuredChannels)) {
|
||||
if (key === "defaults" || key === "modelByChannel") {
|
||||
continue;
|
||||
}
|
||||
if (!normalizeChatChannelId(key)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function resolveConfiguredPlugins(
|
||||
cfg: OpenClawConfig,
|
||||
env: NodeJS.ProcessEnv,
|
||||
@@ -478,7 +499,10 @@ export function applyPluginAutoEnable(params: {
|
||||
}): PluginAutoEnableResult {
|
||||
const env = params.env ?? process.env;
|
||||
const registry =
|
||||
params.manifestRegistry ?? loadPluginManifestRegistry({ config: params.config, env });
|
||||
params.manifestRegistry ??
|
||||
(configMayNeedPluginManifestRegistry(params.config)
|
||||
? loadPluginManifestRegistry({ config: params.config, env })
|
||||
: EMPTY_PLUGIN_MANIFEST_REGISTRY);
|
||||
const configured = resolveConfiguredPlugins(params.config, env, registry);
|
||||
if (configured.length === 0) {
|
||||
return { config: params.config, changes: [] };
|
||||
|
||||
@@ -119,7 +119,7 @@ describe("memory manager cache hydration", () => {
|
||||
await secondManager?.close?.();
|
||||
});
|
||||
|
||||
it("does not cache status-only managers when no full manager exists", async () => {
|
||||
it("caches status-only managers separately from full managers", async () => {
|
||||
const indexPath = path.join(workspaceDir, "index.sqlite");
|
||||
const cfg = createMemoryConcurrencyConfig(indexPath);
|
||||
|
||||
@@ -128,10 +128,9 @@ describe("memory manager cache hydration", () => {
|
||||
|
||||
expect(first).toBeTruthy();
|
||||
expect(second).toBeTruthy();
|
||||
expect(Object.is(second, first)).toBe(false);
|
||||
expect(Object.is(second, first)).toBe(true);
|
||||
expect(hoisted.providerCreateCalls).toBe(0);
|
||||
|
||||
await first?.close?.();
|
||||
await second?.close?.();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { resetEmbeddingMocks } from "./embedding.test-mocks.js";
|
||||
import type { MemoryIndexManager } from "./index.js";
|
||||
@@ -32,16 +32,32 @@ function createMemorySearchCfg(options: {
|
||||
describe("MemoryIndexManager.readFile", () => {
|
||||
let workspaceDir: string;
|
||||
let indexPath: string;
|
||||
let memoryDir: string;
|
||||
let manager: MemoryIndexManager | null = null;
|
||||
|
||||
beforeEach(async () => {
|
||||
beforeAll(async () => {
|
||||
resetEmbeddingMocks();
|
||||
workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-read-"));
|
||||
indexPath = path.join(workspaceDir, "index.sqlite");
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
memoryDir = path.join(workspaceDir, "memory");
|
||||
await fs.mkdir(memoryDir, { recursive: true });
|
||||
manager = await getRequiredMemoryIndexManager({
|
||||
cfg: createMemorySearchCfg({ workspaceDir, indexPath }),
|
||||
agentId: "main",
|
||||
purpose: "status",
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
const entries = await fs.readdir(memoryDir).catch(() => []);
|
||||
await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
await fs.rm(path.join(memoryDir, entry), { recursive: true, force: true });
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (manager) {
|
||||
await manager.close();
|
||||
manager = null;
|
||||
@@ -50,14 +66,8 @@ describe("MemoryIndexManager.readFile", () => {
|
||||
});
|
||||
|
||||
it("returns empty text when the requested file does not exist", async () => {
|
||||
manager = await getRequiredMemoryIndexManager({
|
||||
cfg: createMemorySearchCfg({ workspaceDir, indexPath }),
|
||||
agentId: "main",
|
||||
purpose: "status",
|
||||
});
|
||||
|
||||
const relPath = "memory/2099-01-01.md";
|
||||
const result = await manager.readFile({ relPath });
|
||||
const result = await manager!.readFile({ relPath });
|
||||
expect(result).toEqual({ text: "", path: relPath });
|
||||
});
|
||||
|
||||
@@ -67,13 +77,7 @@ describe("MemoryIndexManager.readFile", () => {
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, ["line 1", "line 2", "line 3"].join("\n"), "utf-8");
|
||||
|
||||
manager = await getRequiredMemoryIndexManager({
|
||||
cfg: createMemorySearchCfg({ workspaceDir, indexPath }),
|
||||
agentId: "main",
|
||||
purpose: "status",
|
||||
});
|
||||
|
||||
const result = await manager.readFile({ relPath, from: 2, lines: 1 });
|
||||
const result = await manager!.readFile({ relPath, from: 2, lines: 1 });
|
||||
expect(result).toEqual({ text: "line 2", path: relPath });
|
||||
});
|
||||
|
||||
@@ -83,13 +87,7 @@ describe("MemoryIndexManager.readFile", () => {
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, ["alpha", "beta"].join("\n"), "utf-8");
|
||||
|
||||
manager = await getRequiredMemoryIndexManager({
|
||||
cfg: createMemorySearchCfg({ workspaceDir, indexPath }),
|
||||
agentId: "main",
|
||||
purpose: "status",
|
||||
});
|
||||
|
||||
const result = await manager.readFile({ relPath, from: 10, lines: 5 });
|
||||
const result = await manager!.readFile({ relPath, from: 10, lines: 5 });
|
||||
expect(result).toEqual({ text: "", path: relPath });
|
||||
});
|
||||
|
||||
@@ -99,12 +97,6 @@ describe("MemoryIndexManager.readFile", () => {
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, "first\nsecond", "utf-8");
|
||||
|
||||
manager = await getRequiredMemoryIndexManager({
|
||||
cfg: createMemorySearchCfg({ workspaceDir, indexPath }),
|
||||
agentId: "main",
|
||||
purpose: "status",
|
||||
});
|
||||
|
||||
const realReadFile = fs.readFile;
|
||||
let injected = false;
|
||||
const readSpy = vi
|
||||
@@ -120,9 +112,11 @@ describe("MemoryIndexManager.readFile", () => {
|
||||
return realReadFile(target, options);
|
||||
});
|
||||
|
||||
const result = await manager.readFile({ relPath });
|
||||
expect(result).toEqual({ text: "", path: relPath });
|
||||
|
||||
readSpy.mockRestore();
|
||||
try {
|
||||
const result = await manager!.readFile({ relPath });
|
||||
expect(result).toEqual({ text: "", path: relPath });
|
||||
} finally {
|
||||
readSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,9 +5,34 @@ import type { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { resetEmbeddingMocks } from "./embedding.test-mocks.js";
|
||||
import type { MemoryIndexManager } from "./index.js";
|
||||
import { MemoryIndexManager } from "./manager.js";
|
||||
import { getRequiredMemoryIndexManager } from "./test-manager-helpers.js";
|
||||
|
||||
type ReadonlyRecoveryHarness = {
|
||||
closed: boolean;
|
||||
syncing: Promise<void> | null;
|
||||
queuedSessionFiles: Set<string>;
|
||||
queuedSessionSync: Promise<void> | null;
|
||||
db: DatabaseSync;
|
||||
vectorReady: Promise<boolean> | null;
|
||||
vector: {
|
||||
enabled: boolean;
|
||||
available: boolean | null;
|
||||
loadError?: string;
|
||||
dims?: number;
|
||||
};
|
||||
readonlyRecoveryAttempts: number;
|
||||
readonlyRecoverySuccesses: number;
|
||||
readonlyRecoveryFailures: number;
|
||||
readonlyRecoveryLastError?: string;
|
||||
ensureProviderInitialized: ReturnType<typeof vi.fn>;
|
||||
enqueueTargetedSessionSync: ReturnType<typeof vi.fn>;
|
||||
runSync: ReturnType<typeof vi.fn>;
|
||||
openDatabase: ReturnType<typeof vi.fn>;
|
||||
ensureSchema: ReturnType<typeof vi.fn>;
|
||||
readMeta: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
describe("memory manager readonly recovery", () => {
|
||||
let workspaceDir = "";
|
||||
let indexPath = "";
|
||||
@@ -32,27 +57,69 @@ describe("memory manager readonly recovery", () => {
|
||||
} as OpenClawConfig;
|
||||
}
|
||||
|
||||
async function createManager() {
|
||||
manager = await getRequiredMemoryIndexManager({ cfg: createMemoryConfig(), agentId: "main" });
|
||||
async function createRealManager() {
|
||||
manager = await getRequiredMemoryIndexManager({
|
||||
cfg: createMemoryConfig(),
|
||||
agentId: "main",
|
||||
purpose: "status",
|
||||
});
|
||||
return manager;
|
||||
}
|
||||
|
||||
function createSyncSpies(instance: MemoryIndexManager) {
|
||||
const runSyncSpy = vi.spyOn(
|
||||
instance as unknown as {
|
||||
runSync: (params?: { reason?: string; force?: boolean }) => Promise<void>;
|
||||
function createReadonlyRecoveryHarness() {
|
||||
const reopenedClose = vi.fn();
|
||||
const initialClose = vi.fn();
|
||||
const reopenedDb = { close: reopenedClose } as unknown as DatabaseSync;
|
||||
const initialDb = { close: initialClose } as unknown as DatabaseSync;
|
||||
const harness: ReadonlyRecoveryHarness = {
|
||||
closed: false,
|
||||
syncing: null,
|
||||
queuedSessionFiles: new Set<string>(),
|
||||
queuedSessionSync: null,
|
||||
db: initialDb,
|
||||
vectorReady: null,
|
||||
vector: {
|
||||
enabled: false,
|
||||
available: null,
|
||||
loadError: "stale",
|
||||
dims: 123,
|
||||
},
|
||||
"runSync",
|
||||
);
|
||||
const openDatabaseSpy = vi.spyOn(
|
||||
instance as unknown as { openDatabase: () => DatabaseSync },
|
||||
"openDatabase",
|
||||
);
|
||||
return { runSyncSpy, openDatabaseSpy };
|
||||
readonlyRecoveryAttempts: 0,
|
||||
readonlyRecoverySuccesses: 0,
|
||||
readonlyRecoveryFailures: 0,
|
||||
readonlyRecoveryLastError: undefined,
|
||||
ensureProviderInitialized: vi.fn(async () => {}),
|
||||
enqueueTargetedSessionSync: vi.fn(async () => {}),
|
||||
runSync: vi.fn(),
|
||||
openDatabase: vi.fn(() => reopenedDb),
|
||||
ensureSchema: vi.fn(),
|
||||
readMeta: vi.fn(() => undefined),
|
||||
};
|
||||
Object.setPrototypeOf(harness, MemoryIndexManager.prototype);
|
||||
return {
|
||||
harness,
|
||||
initialDb,
|
||||
initialClose,
|
||||
reopenedDb,
|
||||
reopenedClose,
|
||||
};
|
||||
}
|
||||
|
||||
function expectReadonlyRecoveryStatus(lastError: string) {
|
||||
expect(manager?.status().custom?.readonlyRecovery).toEqual({
|
||||
function expectReadonlyRecoveryStatus(
|
||||
instance: {
|
||||
readonlyRecoveryAttempts: number;
|
||||
readonlyRecoverySuccesses: number;
|
||||
readonlyRecoveryFailures: number;
|
||||
readonlyRecoveryLastError?: string;
|
||||
},
|
||||
lastError: string,
|
||||
) {
|
||||
expect({
|
||||
attempts: instance.readonlyRecoveryAttempts,
|
||||
successes: instance.readonlyRecoverySuccesses,
|
||||
failures: instance.readonlyRecoveryFailures,
|
||||
lastError: instance.readonlyRecoveryLastError,
|
||||
}).toEqual({
|
||||
attempts: 1,
|
||||
successes: 1,
|
||||
failures: 0,
|
||||
@@ -61,15 +128,17 @@ describe("memory manager readonly recovery", () => {
|
||||
}
|
||||
|
||||
async function expectReadonlyRetry(params: { firstError: unknown; expectedLastError: string }) {
|
||||
const currentManager = await createManager();
|
||||
const { runSyncSpy, openDatabaseSpy } = createSyncSpies(currentManager);
|
||||
runSyncSpy.mockRejectedValueOnce(params.firstError).mockResolvedValueOnce(undefined);
|
||||
const { harness, initialClose } = createReadonlyRecoveryHarness();
|
||||
harness.runSync.mockRejectedValueOnce(params.firstError).mockResolvedValueOnce(undefined);
|
||||
|
||||
await currentManager.sync({ reason: "test" });
|
||||
await MemoryIndexManager.prototype.sync.call(harness as unknown as MemoryIndexManager, {
|
||||
reason: "test",
|
||||
});
|
||||
|
||||
expect(runSyncSpy).toHaveBeenCalledTimes(2);
|
||||
expect(openDatabaseSpy).toHaveBeenCalledTimes(1);
|
||||
expectReadonlyRecoveryStatus(params.expectedLastError);
|
||||
expect(harness.runSync).toHaveBeenCalledTimes(2);
|
||||
expect(harness.openDatabase).toHaveBeenCalledTimes(1);
|
||||
expect(initialClose).toHaveBeenCalledTimes(1);
|
||||
expectReadonlyRecoveryStatus(harness, params.expectedLastError);
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -81,6 +150,7 @@ describe("memory manager readonly recovery", () => {
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
if (manager) {
|
||||
await manager.close();
|
||||
manager = null;
|
||||
@@ -103,17 +173,21 @@ describe("memory manager readonly recovery", () => {
|
||||
});
|
||||
|
||||
it("does not retry non-readonly sync errors", async () => {
|
||||
const currentManager = await createManager();
|
||||
const { runSyncSpy, openDatabaseSpy } = createSyncSpies(currentManager);
|
||||
runSyncSpy.mockRejectedValueOnce(new Error("embedding timeout"));
|
||||
const { harness, initialClose } = createReadonlyRecoveryHarness();
|
||||
harness.runSync.mockRejectedValueOnce(new Error("embedding timeout"));
|
||||
|
||||
await expect(currentManager.sync({ reason: "test" })).rejects.toThrow("embedding timeout");
|
||||
expect(runSyncSpy).toHaveBeenCalledTimes(1);
|
||||
expect(openDatabaseSpy).toHaveBeenCalledTimes(0);
|
||||
await expect(
|
||||
MemoryIndexManager.prototype.sync.call(harness as unknown as MemoryIndexManager, {
|
||||
reason: "test",
|
||||
}),
|
||||
).rejects.toThrow("embedding timeout");
|
||||
expect(harness.runSync).toHaveBeenCalledTimes(1);
|
||||
expect(harness.openDatabase).not.toHaveBeenCalled();
|
||||
expect(initialClose).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("sets busy_timeout on memory sqlite connections", async () => {
|
||||
const currentManager = await createManager();
|
||||
const currentManager = await createRealManager();
|
||||
const db = (currentManager as unknown as { db: DatabaseSync }).db;
|
||||
const row = db.prepare("PRAGMA busy_timeout").get() as
|
||||
| { busy_timeout?: number; timeout?: number }
|
||||
|
||||
@@ -58,8 +58,10 @@ describe("memory manager sync failures", () => {
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
manager = await getRequiredMemoryIndexManager({ cfg, agentId: "main" });
|
||||
const syncSpy = vi.spyOn(manager, "sync");
|
||||
manager = await getRequiredMemoryIndexManager({ cfg, agentId: "main", purpose: "status" });
|
||||
const syncSpy = vi
|
||||
.spyOn(manager, "sync")
|
||||
.mockRejectedValueOnce(new Error("openai embeddings failed: 400 bad request"));
|
||||
|
||||
// Call the internal scheduler directly; it uses fire-and-forget sync.
|
||||
(manager as unknown as { scheduleWatchSync: () => void }).scheduleWatchSync();
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { type FSWatcher } from "chokidar";
|
||||
import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
|
||||
@@ -18,12 +16,11 @@ import {
|
||||
type OpenAiEmbeddingClient,
|
||||
type VoyageEmbeddingClient,
|
||||
} from "./embeddings.js";
|
||||
import { isFileMissingError, statRegularFile } from "./fs-utils.js";
|
||||
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
|
||||
import { isMemoryPath, normalizeExtraMemoryPaths } from "./internal.js";
|
||||
import { MemoryManagerEmbeddingOps } from "./manager-embedding-ops.js";
|
||||
import { searchKeyword, searchVector } from "./manager-search.js";
|
||||
import { extractKeywords } from "./query-expansion.js";
|
||||
import { readMemoryFile } from "./read-file.js";
|
||||
import type {
|
||||
MemoryEmbeddingProbeResult,
|
||||
MemoryProviderStatus,
|
||||
@@ -174,7 +171,8 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
return null;
|
||||
}
|
||||
const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
|
||||
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`;
|
||||
const purpose = params.purpose === "status" ? "status" : "default";
|
||||
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}:${purpose}`;
|
||||
const statusOnly = params.purpose === "status";
|
||||
const existing = INDEX_CACHE.get(key);
|
||||
if (existing) {
|
||||
@@ -185,7 +183,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
return pending;
|
||||
}
|
||||
if (statusOnly) {
|
||||
return new MemoryIndexManager({
|
||||
const manager = new MemoryIndexManager({
|
||||
cacheKey: key,
|
||||
cfg,
|
||||
agentId,
|
||||
@@ -193,6 +191,8 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
settings,
|
||||
purpose: params.purpose,
|
||||
});
|
||||
INDEX_CACHE.set(key, manager);
|
||||
return manager;
|
||||
}
|
||||
const createPromise = (async () => {
|
||||
const providerResult = await MemoryIndexManager.loadProviderResult({
|
||||
@@ -667,72 +667,13 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<{ text: string; path: string }> {
|
||||
const rawPath = params.relPath.trim();
|
||||
if (!rawPath) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const absPath = path.isAbsolute(rawPath)
|
||||
? path.resolve(rawPath)
|
||||
: path.resolve(this.workspaceDir, rawPath);
|
||||
const relPath = path.relative(this.workspaceDir, absPath).replace(/\\/g, "/");
|
||||
const inWorkspace =
|
||||
relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath);
|
||||
const allowedWorkspace = inWorkspace && isMemoryPath(relPath);
|
||||
let allowedAdditional = false;
|
||||
if (!allowedWorkspace && this.settings.extraPaths.length > 0) {
|
||||
const additionalPaths = normalizeExtraMemoryPaths(
|
||||
this.workspaceDir,
|
||||
this.settings.extraPaths,
|
||||
);
|
||||
for (const additionalPath of additionalPaths) {
|
||||
try {
|
||||
const stat = await fs.lstat(additionalPath);
|
||||
if (stat.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (stat.isFile()) {
|
||||
if (absPath === additionalPath && absPath.endsWith(".md")) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (!allowedWorkspace && !allowedAdditional) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
if (!absPath.endsWith(".md")) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const statResult = await statRegularFile(absPath);
|
||||
if (statResult.missing) {
|
||||
return { text: "", path: relPath };
|
||||
}
|
||||
let content: string;
|
||||
try {
|
||||
content = await fs.readFile(absPath, "utf-8");
|
||||
} catch (err) {
|
||||
if (isFileMissingError(err)) {
|
||||
return { text: "", path: relPath };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!params.from && !params.lines) {
|
||||
return { text: content, path: relPath };
|
||||
}
|
||||
const lines = content.split("\n");
|
||||
const start = Math.max(1, params.from ?? 1);
|
||||
const count = Math.max(1, params.lines ?? lines.length);
|
||||
const slice = lines.slice(start - 1, start - 1 + count);
|
||||
return { text: slice.join("\n"), path: relPath };
|
||||
return await readMemoryFile({
|
||||
workspaceDir: this.workspaceDir,
|
||||
extraPaths: this.settings.extraPaths,
|
||||
relPath: params.relPath,
|
||||
from: params.from,
|
||||
lines: params.lines,
|
||||
});
|
||||
}
|
||||
|
||||
status(): MemoryProviderStatus {
|
||||
|
||||
96
src/memory/read-file.ts
Normal file
96
src/memory/read-file.ts
Normal file
@@ -0,0 +1,96 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
|
||||
import { resolveMemorySearchConfig } from "../agents/memory-search.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { isFileMissingError, statRegularFile } from "./fs-utils.js";
|
||||
import { isMemoryPath, normalizeExtraMemoryPaths } from "./internal.js";
|
||||
|
||||
export async function readMemoryFile(params: {
|
||||
workspaceDir: string;
|
||||
extraPaths?: string[];
|
||||
relPath: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<{ text: string; path: string }> {
|
||||
const rawPath = params.relPath.trim();
|
||||
if (!rawPath) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const absPath = path.isAbsolute(rawPath)
|
||||
? path.resolve(rawPath)
|
||||
: path.resolve(params.workspaceDir, rawPath);
|
||||
const relPath = path.relative(params.workspaceDir, absPath).replace(/\\/g, "/");
|
||||
const inWorkspace = relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath);
|
||||
const allowedWorkspace = inWorkspace && isMemoryPath(relPath);
|
||||
let allowedAdditional = false;
|
||||
if (!allowedWorkspace && (params.extraPaths?.length ?? 0) > 0) {
|
||||
const additionalPaths = normalizeExtraMemoryPaths(params.workspaceDir, params.extraPaths);
|
||||
for (const additionalPath of additionalPaths) {
|
||||
try {
|
||||
const stat = await fs.lstat(additionalPath);
|
||||
if (stat.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (stat.isFile() && absPath === additionalPath && absPath.endsWith(".md")) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (!allowedWorkspace && !allowedAdditional) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
if (!absPath.endsWith(".md")) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const statResult = await statRegularFile(absPath);
|
||||
if (statResult.missing) {
|
||||
return { text: "", path: relPath };
|
||||
}
|
||||
let content: string;
|
||||
try {
|
||||
content = await fs.readFile(absPath, "utf-8");
|
||||
} catch (err) {
|
||||
if (isFileMissingError(err)) {
|
||||
return { text: "", path: relPath };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!params.from && !params.lines) {
|
||||
return { text: content, path: relPath };
|
||||
}
|
||||
const fileLines = content.split("\n");
|
||||
const start = Math.max(1, params.from ?? 1);
|
||||
const count = Math.max(1, params.lines ?? fileLines.length);
|
||||
const slice = fileLines.slice(start - 1, start - 1 + count);
|
||||
return { text: slice.join("\n"), path: relPath };
|
||||
}
|
||||
|
||||
export async function readAgentMemoryFile(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
relPath: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<{ text: string; path: string }> {
|
||||
const settings = resolveMemorySearchConfig(params.cfg, params.agentId);
|
||||
if (!settings) {
|
||||
throw new Error("memory search disabled");
|
||||
}
|
||||
return await readMemoryFile({
|
||||
workspaceDir: resolveAgentWorkspaceDir(params.cfg, params.agentId),
|
||||
extraPaths: settings.extraPaths,
|
||||
relPath: params.relPath,
|
||||
from: params.from,
|
||||
lines: params.lines,
|
||||
});
|
||||
}
|
||||
@@ -195,7 +195,7 @@ describe("getMemorySearchManager caching", () => {
|
||||
expect(createQmdManagerMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not cache status-only qmd managers", async () => {
|
||||
it("caches status-only qmd managers separately from full managers", async () => {
|
||||
const agentId = "status-agent";
|
||||
const cfg = createQmdCfg(agentId);
|
||||
|
||||
@@ -205,17 +205,13 @@ describe("getMemorySearchManager caching", () => {
|
||||
requireManager(first);
|
||||
requireManager(second);
|
||||
// eslint-disable-next-line @typescript-eslint/unbound-method
|
||||
expect(createQmdManagerMock).toHaveBeenCalledTimes(2);
|
||||
expect(createQmdManagerMock).toHaveBeenCalledTimes(1);
|
||||
// eslint-disable-next-line @typescript-eslint/unbound-method
|
||||
expect(createQmdManagerMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ agentId, mode: "status" }),
|
||||
);
|
||||
// eslint-disable-next-line @typescript-eslint/unbound-method
|
||||
expect(createQmdManagerMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({ agentId, mode: "status" }),
|
||||
);
|
||||
expect(second.manager).toBe(first.manager);
|
||||
});
|
||||
|
||||
it("does not evict a newer cached wrapper when closing an older failed wrapper", async () => {
|
||||
|
||||
@@ -46,13 +46,11 @@ export async function getMemorySearchManager(params: {
|
||||
const resolved = resolveMemoryBackendConfig(params);
|
||||
if (resolved.backend === "qmd" && resolved.qmd) {
|
||||
const statusOnly = params.purpose === "status";
|
||||
let cacheKey: string | undefined;
|
||||
if (!statusOnly) {
|
||||
cacheKey = buildQmdCacheKey(params.agentId, resolved.qmd);
|
||||
const cached = QMD_MANAGER_CACHE.get(cacheKey);
|
||||
if (cached) {
|
||||
return { manager: cached };
|
||||
}
|
||||
const baseCacheKey = buildQmdCacheKey(params.agentId, resolved.qmd);
|
||||
const cacheKey = `${baseCacheKey}:${statusOnly ? "status" : "full"}`;
|
||||
const cached = QMD_MANAGER_CACHE.get(cacheKey);
|
||||
if (cached) {
|
||||
return { manager: cached };
|
||||
}
|
||||
try {
|
||||
const { QmdMemoryManager } = await import("./qmd-manager.js");
|
||||
@@ -64,6 +62,7 @@ export async function getMemorySearchManager(params: {
|
||||
});
|
||||
if (primary) {
|
||||
if (statusOnly) {
|
||||
QMD_MANAGER_CACHE.set(cacheKey, primary);
|
||||
return { manager: primary };
|
||||
}
|
||||
const wrapper = new FallbackMemoryManager(
|
||||
@@ -75,14 +74,10 @@ export async function getMemorySearchManager(params: {
|
||||
},
|
||||
},
|
||||
() => {
|
||||
if (cacheKey) {
|
||||
QMD_MANAGER_CACHE.delete(cacheKey);
|
||||
}
|
||||
QMD_MANAGER_CACHE.delete(cacheKey);
|
||||
},
|
||||
);
|
||||
if (cacheKey) {
|
||||
QMD_MANAGER_CACHE.set(cacheKey, wrapper);
|
||||
}
|
||||
QMD_MANAGER_CACHE.set(cacheKey, wrapper);
|
||||
return { manager: wrapper };
|
||||
}
|
||||
} catch (err) {
|
||||
|
||||
@@ -33,8 +33,17 @@ const stubManager = {
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
const getMemorySearchManagerMock = vi.fn(async () => ({ manager: stubManager }));
|
||||
const readAgentMemoryFileMock = vi.fn(
|
||||
async (params: MemoryReadParams) => await readFileImpl(params),
|
||||
);
|
||||
|
||||
vi.mock("../../src/memory/index.js", () => ({
|
||||
getMemorySearchManager: async () => ({ manager: stubManager }),
|
||||
getMemorySearchManager: getMemorySearchManagerMock,
|
||||
}));
|
||||
|
||||
vi.mock("../../src/memory/read-file.js", () => ({
|
||||
readAgentMemoryFile: readAgentMemoryFileMock,
|
||||
}));
|
||||
|
||||
export function setMemoryBackend(next: MemoryBackend): void {
|
||||
@@ -63,3 +72,11 @@ export function resetMemoryToolMockState(overrides?: {
|
||||
(async (params: MemoryReadParams) => ({ text: "", path: params.relPath }));
|
||||
vi.clearAllMocks();
|
||||
}
|
||||
|
||||
export function getMemorySearchManagerMockCalls(): number {
|
||||
return getMemorySearchManagerMock.mock.calls.length;
|
||||
}
|
||||
|
||||
export function getReadAgentMemoryFileMockCalls(): number {
|
||||
return readAgentMemoryFileMock.mock.calls.length;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user