mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-12 09:41:11 +00:00
test: trim memory manager test startup
This commit is contained in:
@@ -67,6 +67,23 @@ export function installEmbeddingManagerFixture(opts: {
|
||||
return manager as unknown as MemoryIndexManager;
|
||||
};
|
||||
|
||||
const createManager = async (params: {
|
||||
indexPath: string;
|
||||
tokens: number;
|
||||
name: string;
|
||||
}): Promise<MemoryIndexManager> => {
|
||||
const managerResult = await getMemorySearchManager({
|
||||
cfg: opts.createCfg({
|
||||
workspaceDir: requireValue(workspaceDir, "workspaceDir"),
|
||||
indexPath: params.indexPath,
|
||||
tokens: params.tokens,
|
||||
}),
|
||||
agentId: "main",
|
||||
});
|
||||
expect(managerResult.manager).not.toBeNull();
|
||||
return requireIndexManager(managerResult.manager, params.name);
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
vi.resetModules();
|
||||
await import("./embedding.test-mocks.js");
|
||||
@@ -93,31 +110,6 @@ export function installEmbeddingManagerFixture(opts: {
|
||||
workspaceDir = path.join(fixtureRoot, "workspace");
|
||||
memoryDir = path.join(workspaceDir, "memory");
|
||||
await fs.mkdir(memoryDir, { recursive: true });
|
||||
|
||||
const indexPathLarge = path.join(fixtureRoot, "index.large.sqlite");
|
||||
const indexPathSmall = path.join(fixtureRoot, "index.small.sqlite");
|
||||
|
||||
const large = await getMemorySearchManager({
|
||||
cfg: opts.createCfg({
|
||||
workspaceDir,
|
||||
indexPath: indexPathLarge,
|
||||
tokens: opts.largeTokens,
|
||||
}),
|
||||
agentId: "main",
|
||||
});
|
||||
expect(large.manager).not.toBeNull();
|
||||
managerLarge = requireIndexManager(large.manager, "managerLarge");
|
||||
|
||||
const small = await getMemorySearchManager({
|
||||
cfg: opts.createCfg({
|
||||
workspaceDir,
|
||||
indexPath: indexPathSmall,
|
||||
tokens: opts.smallTokens,
|
||||
}),
|
||||
agentId: "main",
|
||||
});
|
||||
expect(small.manager).not.toBeNull();
|
||||
managerSmall = requireIndexManager(small.manager, "managerSmall");
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -149,8 +141,12 @@ export function installEmbeddingManagerFixture(opts: {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
|
||||
if (resetIndexEachTest) {
|
||||
resetManager(requireValue(managerLarge, "managerLarge"));
|
||||
resetManager(requireValue(managerSmall, "managerSmall"));
|
||||
if (managerLarge) {
|
||||
resetManager(managerLarge);
|
||||
}
|
||||
if (managerSmall) {
|
||||
resetManager(managerSmall);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -161,8 +157,22 @@ export function installEmbeddingManagerFixture(opts: {
|
||||
getFixtureRoot: () => requireValue(fixtureRoot, "fixtureRoot"),
|
||||
getWorkspaceDir: () => requireValue(workspaceDir, "workspaceDir"),
|
||||
getMemoryDir: () => requireValue(memoryDir, "memoryDir"),
|
||||
getManagerLarge: () => requireValue(managerLarge, "managerLarge"),
|
||||
getManagerSmall: () => requireValue(managerSmall, "managerSmall"),
|
||||
getManagerLarge: async () => {
|
||||
managerLarge ??= await createManager({
|
||||
indexPath: path.join(requireValue(fixtureRoot, "fixtureRoot"), "index.large.sqlite"),
|
||||
tokens: opts.largeTokens,
|
||||
name: "managerLarge",
|
||||
});
|
||||
return managerLarge;
|
||||
},
|
||||
getManagerSmall: async () => {
|
||||
managerSmall ??= await createManager({
|
||||
indexPath: path.join(requireValue(fixtureRoot, "fixtureRoot"), "index.small.sqlite"),
|
||||
tokens: opts.smallTokens,
|
||||
name: "managerSmall",
|
||||
});
|
||||
return managerSmall;
|
||||
},
|
||||
resetManager,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -111,12 +111,9 @@ describe("memory index", () => {
|
||||
let fixtureRoot = "";
|
||||
let workspaceDir = "";
|
||||
let memoryDir = "";
|
||||
let extraDir = "";
|
||||
let indexVectorPath = "";
|
||||
let indexMainPath = "";
|
||||
let indexExtraPath = "";
|
||||
let indexMultimodalPath = "";
|
||||
let indexFtsOnlyPath = "";
|
||||
|
||||
const managersForCleanup = new Set<MemoryIndexManager>();
|
||||
|
||||
@@ -124,12 +121,9 @@ describe("memory index", () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-"));
|
||||
workspaceDir = path.join(fixtureRoot, "workspace");
|
||||
memoryDir = path.join(workspaceDir, "memory");
|
||||
extraDir = path.join(workspaceDir, "extra");
|
||||
indexMainPath = path.join(workspaceDir, "index-main.sqlite");
|
||||
indexVectorPath = path.join(workspaceDir, "index-vector.sqlite");
|
||||
indexExtraPath = path.join(workspaceDir, "index-extra.sqlite");
|
||||
indexMultimodalPath = path.join(workspaceDir, "index-multimodal.sqlite");
|
||||
indexFtsOnlyPath = path.join(workspaceDir, "index-fts-only.sqlite");
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -575,46 +569,9 @@ describe("memory index", () => {
|
||||
expect(status.vector?.available).toBe(available);
|
||||
});
|
||||
|
||||
it("rejects reading non-memory paths", async () => {
|
||||
const cfg = createCfg({ storePath: indexMainPath });
|
||||
const manager = await getPersistentManager(cfg);
|
||||
await expect(manager.readFile({ relPath: "NOTES.md" })).rejects.toThrow("path required");
|
||||
});
|
||||
|
||||
it("allows reading from additional memory paths and blocks symlinks", async () => {
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "extra.md"), "Extra content.");
|
||||
|
||||
const cfg = createCfg({ storePath: indexExtraPath, extraPaths: [extraDir] });
|
||||
const manager = await getPersistentManager(cfg);
|
||||
await expect(manager.readFile({ relPath: "extra/extra.md" })).resolves.toEqual({
|
||||
path: "extra/extra.md",
|
||||
text: "Extra content.",
|
||||
});
|
||||
|
||||
const linkPath = path.join(extraDir, "linked.md");
|
||||
let symlinkOk = true;
|
||||
try {
|
||||
await fs.symlink(path.join(extraDir, "extra.md"), linkPath, "file");
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "EPERM" || code === "EACCES") {
|
||||
symlinkOk = false;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (symlinkOk) {
|
||||
await expect(manager.readFile({ relPath: "extra/linked.md" })).rejects.toThrow(
|
||||
"path required",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("triggers full reindex and cleans up old-model FTS rows when switching from provider to FTS-only", async () => {
|
||||
const sharedStorePath = path.join(workspaceDir, "index-provider-to-fts-only.sqlite");
|
||||
|
||||
// Phase 1: sync with a real provider — FTS rows stored under model = "mock-embed"
|
||||
const providerCfg = createCfg({ storePath: sharedStorePath, hybrid: { enabled: true } });
|
||||
const providerResult = await getMemorySearchManager({ cfg: providerCfg, agentId: "main" });
|
||||
const providerManager = requireManager(providerResult);
|
||||
@@ -634,7 +591,6 @@ describe("memory index", () => {
|
||||
await providerManager.close();
|
||||
managersForCleanup.delete(providerManager);
|
||||
|
||||
// Phase 2: switch to FTS-only (no provider) — should trigger full reindex
|
||||
forceNoProvider = true;
|
||||
const ftsOnlyCfg = createCfg({ storePath: sharedStorePath, hybrid: { enabled: true } });
|
||||
const ftsOnlyResult = await getMemorySearchManager({ cfg: ftsOnlyCfg, agentId: "main" });
|
||||
@@ -646,14 +602,11 @@ describe("memory index", () => {
|
||||
const db = (
|
||||
ftsOnlyManager as unknown as { db: { prepare: (s: string) => { get: () => { c: number } } } }
|
||||
).db;
|
||||
|
||||
// old provider-model rows should be gone after full reindex
|
||||
const oldRows = db
|
||||
.prepare("SELECT COUNT(*) as c FROM chunks_fts WHERE model = 'mock-embed'")
|
||||
.get();
|
||||
expect(oldRows.c).toBe(0);
|
||||
|
||||
// new fts-only rows should exist
|
||||
const newRows = db
|
||||
.prepare("SELECT COUNT(*) as c FROM chunks_fts WHERE model = 'fts-only'")
|
||||
.get();
|
||||
@@ -664,7 +617,7 @@ describe("memory index", () => {
|
||||
forceNoProvider = true;
|
||||
|
||||
const cfg = createCfg({
|
||||
storePath: indexFtsOnlyPath,
|
||||
storePath: path.join(workspaceDir, "index-fts-only.sqlite"),
|
||||
minScore: 0.35,
|
||||
hybrid: { enabled: true },
|
||||
});
|
||||
@@ -680,16 +633,13 @@ describe("memory index", () => {
|
||||
await manager.sync({ reason: "test" });
|
||||
|
||||
const status = manager.status();
|
||||
// chunks should be indexed via FTS even without a provider
|
||||
expect(status.chunks).toBeGreaterThan(0);
|
||||
expect(embedBatchCalls).toBe(0);
|
||||
|
||||
// keyword search should still return matching results under the default threshold
|
||||
const results = await manager.search("Alpha");
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results[0]?.snippet).toMatch(/Alpha/i);
|
||||
|
||||
// unknown terms should return no results
|
||||
const noResults = await manager.search("nonexistent_xyz_keyword");
|
||||
expect(noResults.length).toBe(0);
|
||||
});
|
||||
|
||||
@@ -56,20 +56,20 @@ describe("memory embedding batches", () => {
|
||||
|
||||
it("splits large files across multiple embedding batches", async () => {
|
||||
const memoryDir = fx.getMemoryDir();
|
||||
const managerLarge = fx.getManagerLarge();
|
||||
const manager = await fx.getManagerLarge();
|
||||
// Keep this small but above the embedding batch byte threshold (8k) so we
|
||||
// exercise multi-batch behavior without generating lots of chunks/DB rows.
|
||||
const line = "a".repeat(4200);
|
||||
const content = [line, line].join("\n");
|
||||
await fs.writeFile(path.join(memoryDir, "2026-01-03.md"), content);
|
||||
const updates: Array<{ completed: number; total: number; label?: string }> = [];
|
||||
await requireSync(managerLarge)({
|
||||
await requireSync(manager)({
|
||||
progress: (update) => {
|
||||
updates.push(update);
|
||||
},
|
||||
});
|
||||
|
||||
const status = managerLarge.status();
|
||||
const status = manager.status();
|
||||
const totalTexts = fx.embedBatch.mock.calls.reduce(
|
||||
(sum: number, call: unknown[]) => sum + ((call[0] as string[] | undefined)?.length ?? 0),
|
||||
0,
|
||||
@@ -89,18 +89,18 @@ describe("memory embedding batches", () => {
|
||||
|
||||
it("keeps small files in a single embedding batch", async () => {
|
||||
const memoryDir = fx.getMemoryDir();
|
||||
const managerSmall = fx.getManagerSmall();
|
||||
const manager = await fx.getManagerLarge();
|
||||
const line = "b".repeat(120);
|
||||
const content = Array.from({ length: 4 }, () => line).join("\n");
|
||||
await fs.writeFile(path.join(memoryDir, "2026-01-04.md"), content);
|
||||
await requireSync(managerSmall)({ reason: "test" });
|
||||
await requireSync(manager)({ reason: "test" });
|
||||
|
||||
expect(fx.embedBatch.mock.calls.length).toBe(1);
|
||||
});
|
||||
|
||||
it("retries embeddings on transient rate limit and 5xx errors", async () => {
|
||||
const memoryDir = fx.getMemoryDir();
|
||||
const managerSmall = fx.getManagerSmall();
|
||||
const manager = await fx.getManagerLarge();
|
||||
const line = "d".repeat(120);
|
||||
const content = Array.from({ length: 4 }, () => line).join("\n");
|
||||
await fs.writeFile(path.join(memoryDir, "2026-01-06.md"), content);
|
||||
@@ -119,14 +119,14 @@ describe("memory embedding batches", () => {
|
||||
return texts.map(() => [0, 1, 0]);
|
||||
});
|
||||
|
||||
await expectSyncWithFastTimeouts(managerSmall);
|
||||
await expectSyncWithFastTimeouts(manager);
|
||||
|
||||
expect(calls).toBe(3);
|
||||
}, 10000);
|
||||
|
||||
it("retries embeddings on too-many-tokens-per-day rate limits", async () => {
|
||||
const memoryDir = fx.getMemoryDir();
|
||||
const managerSmall = fx.getManagerSmall();
|
||||
const manager = await fx.getManagerLarge();
|
||||
const line = "e".repeat(120);
|
||||
const content = Array.from({ length: 4 }, () => line).join("\n");
|
||||
await fs.writeFile(path.join(memoryDir, "2026-01-08.md"), content);
|
||||
@@ -140,16 +140,16 @@ describe("memory embedding batches", () => {
|
||||
return texts.map(() => [0, 1, 0]);
|
||||
});
|
||||
|
||||
await expectSyncWithFastTimeouts(managerSmall);
|
||||
await expectSyncWithFastTimeouts(manager);
|
||||
|
||||
expect(calls).toBe(2);
|
||||
}, 10000);
|
||||
|
||||
it("skips empty chunks so embeddings input stays valid", async () => {
|
||||
const memoryDir = fx.getMemoryDir();
|
||||
const managerSmall = fx.getManagerSmall();
|
||||
const manager = await fx.getManagerLarge();
|
||||
await fs.writeFile(path.join(memoryDir, "2026-01-07.md"), "\n\n\n");
|
||||
await requireSync(managerSmall)({ reason: "test" });
|
||||
await requireSync(manager)({ reason: "test" });
|
||||
|
||||
const inputs = fx.embedBatch.mock.calls.flatMap(
|
||||
(call: unknown[]) => (call[0] as string[]) ?? [],
|
||||
|
||||
@@ -7,18 +7,24 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
|
||||
describe("MemoryIndexManager.readFile", () => {
|
||||
let workspaceDir: string;
|
||||
let memoryDir: string;
|
||||
let extraDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-read-"));
|
||||
memoryDir = path.join(workspaceDir, "memory");
|
||||
extraDir = path.join(workspaceDir, "extra");
|
||||
await fs.mkdir(memoryDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
const entries = await fs.readdir(memoryDir).catch(() => []);
|
||||
await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
await fs.rm(path.join(memoryDir, entry), { recursive: true, force: true });
|
||||
[memoryDir, extraDir].map(async (root) => {
|
||||
const entries = await fs.readdir(root).catch(() => []);
|
||||
await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
await fs.rm(path.join(root, entry), { recursive: true, force: true });
|
||||
}),
|
||||
);
|
||||
}),
|
||||
);
|
||||
});
|
||||
@@ -101,4 +107,52 @@ describe("MemoryIndexManager.readFile", () => {
|
||||
readSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects non-memory paths", async () => {
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath: "NOTES.md",
|
||||
}),
|
||||
).rejects.toThrow("path required");
|
||||
});
|
||||
|
||||
it("allows additional memory paths and blocks symlinks", async () => {
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "extra.md"), "Extra content.");
|
||||
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: "extra/extra.md",
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
path: "extra/extra.md",
|
||||
text: "Extra content.",
|
||||
});
|
||||
|
||||
const linkPath = path.join(extraDir, "linked.md");
|
||||
let symlinkOk = true;
|
||||
try {
|
||||
await fs.symlink(path.join(extraDir, "extra.md"), linkPath, "file");
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "EPERM" || code === "EACCES") {
|
||||
symlinkOk = false;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (symlinkOk) {
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: "extra/linked.md",
|
||||
}),
|
||||
).rejects.toThrow("path required");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user