vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-18 04:31:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

memory: chunk daily dreaming ingestion

Browse Source
This commit is contained in:
Mariano Belinky
2026-04-06 02:54:44 +02:00
parent e02ef0710e
commit 88816a01ef
3 changed files with 337 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -42,6 +42,7 @@ Docs: https://docs.openclaw.ai
- Providers/CLI: remove bundled CLI text-provider backends and the `agents.defaults.cliBackends` surface, while keeping ACP harness sessions and Gemini media understanding on the native bundled providers.
- Matrix/exec approvals: clarify unavailable-approval replies so Matrix no longer claims chat approvals are unsupported when native exec approvals are merely unconfigured. (#61424) Thanks @gumadeiras.
- Docs/IRC: replace public IRC hostname examples with `irc.example.com` and recommend private servers for bot coordination while listing common public networks for intentional use.
- Memory/dreaming: group nearby daily-note lines into short coherent chunks before staging them for dreaming, so one-off context from recent notes reaches REM/deep with better evidence and less line-level noise.
### Fixes

258
extensions/memory-core/src/dreaming-phases.test.ts
View File

@@ -22,7 +22,7 @@ afterEach(async () => {
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
});
function createHarness(config: OpenClawConfig) {
function createHarness(config: OpenClawConfig, workspaceDir?: string) {
let beforeAgentReply:
| ((
event: { cleanedBody: string },
@@ -36,7 +36,18 @@ function createHarness(config: OpenClawConfig) {
};
const api = {
config,
config: workspaceDir
? {
...config,
agents: {
...config.agents,
defaults: {
...config.agents?.defaults,
workspace: workspaceDir,
},
},
}
: config,
pluginConfig: {},
logger,
registerHook: vi.fn(),
@@ -65,18 +76,20 @@ describe("memory-core dreaming phases", () => {
"utf-8",
);
const { beforeAgentReply } = createHarness({
plugins: {
entries: {
"memory-core": {
config: {
dreaming: {
enabled: true,
phases: {
light: {
enabled: true,
limit: 20,
lookbackDays: 2,
const { beforeAgentReply } = createHarness(
{
plugins: {
entries: {
"memory-core": {
config: {
dreaming: {
enabled: true,
phases: {
light: {
enabled: true,
limit: 20,
lookbackDays: 2,
},
},
},
},
@@ -84,7 +97,8 @@ describe("memory-core dreaming phases", () => {
},
},
},
});
workspaceDir,
);
const readSpy = vi.spyOn(fs, "readFile");
try {
@@ -103,7 +117,7 @@ describe("memory-core dreaming phases", () => {
const dailyReadCount = readSpy.mock.calls.filter(
([target]) => String(target) === dailyPath,
).length;
expect(dailyReadCount).toBe(1);
expect(dailyReadCount).toBeLessThanOrEqual(1);
await expect(
fs.access(path.join(workspaceDir, "memory", ".dreams", "daily-ingestion.json")),
).resolves.toBeUndefined();
@@ -129,18 +143,20 @@ describe("memory-core dreaming phases", () => {
});
expect(before).toHaveLength(0);
const { beforeAgentReply } = createHarness({
plugins: {
entries: {
"memory-core": {
config: {
dreaming: {
enabled: true,
phases: {
light: {
enabled: true,
limit: 20,
lookbackDays: 2,
const { beforeAgentReply } = createHarness(
{
plugins: {
entries: {
"memory-core": {
config: {
dreaming: {
enabled: true,
phases: {
light: {
enabled: true,
limit: 20,
lookbackDays: 2,
},
},
},
},
@@ -148,7 +164,8 @@ describe("memory-core dreaming phases", () => {
},
},
},
});
workspaceDir,
);
await beforeAgentReply(
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },
@@ -162,8 +179,146 @@ describe("memory-core dreaming phases", () => {
minUniqueQueries: 0,
nowMs: Date.parse("2026-04-05T10:05:00.000Z"),
});
expect(after.length).toBeGreaterThan(0);
expect(after.some((candidate) => (candidate.dailyCount ?? 0) > 0)).toBe(true);
expect(after).toHaveLength(1);
expect(after[0]?.dailyCount).toBeGreaterThan(0);
expect(after[0]?.startLine).toBe(3);
expect(after[0]?.endLine).toBe(4);
expect(after[0]?.snippet).toContain("Move backups to S3 Glacier.");
expect(after[0]?.snippet).toContain("Keep retention at 365 days.");
});
it("keeps section context when chunking durable daily notes", async () => {
const workspaceDir = await createTempWorkspace();
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-04-05.md"),
[
"# 2026-04-05",
"",
"## Emma Rees",
"- She asked for more space after the last exchange.",
"- Better to keep messages short and low-pressure.",
"- Re-engagement should be time-bounded and optional.",
].join("\n"),
"utf-8",
);
const { beforeAgentReply } = createHarness(
{
plugins: {
entries: {
"memory-core": {
config: {
dreaming: {
enabled: true,
phases: {
light: {
enabled: true,
limit: 20,
lookbackDays: 2,
},
},
},
},
},
},
},
},
workspaceDir,
);
await beforeAgentReply(
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },
{ trigger: "heartbeat", workspaceDir },
);
const after = await rankShortTermPromotionCandidates({
workspaceDir,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
nowMs: Date.parse("2026-04-05T10:05:00.000Z"),
});
expect(after).toHaveLength(1);
expect(after[0]?.startLine).toBe(4);
expect(after[0]?.endLine).toBe(6);
expect(after[0]?.snippet).toContain("Emma Rees:");
expect(after[0]?.snippet).toContain("She asked for more space");
expect(after[0]?.snippet).toContain("messages short and low-pressure");
});
it("splits noisy daily notes into a few coherent chunks instead of one line per item", async () => {
const workspaceDir = await createTempWorkspace();
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-04-05.md"),
[
"# 2026-04-05",
"",
"## Operations",
"- Restarted the gateway after auth drift.",
"- Tokens now line up again.",
"",
"## Bex",
"- She prefers direct plans over open-ended maybes.",
"- Better to offer one concrete time window.",
"",
"11:30",
"",
"## Travel",
"- Flight lands at 08:10.",
].join("\n"),
"utf-8",
);
const { beforeAgentReply } = createHarness(
{
plugins: {
entries: {
"memory-core": {
config: {
dreaming: {
enabled: true,
phases: {
light: {
enabled: true,
limit: 20,
lookbackDays: 2,
},
},
},
},
},
},
},
},
workspaceDir,
);
await beforeAgentReply(
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },
{ trigger: "heartbeat", workspaceDir },
);
const after = await rankShortTermPromotionCandidates({
workspaceDir,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
nowMs: Date.parse("2026-04-05T10:05:00.000Z"),
});
expect(after).toHaveLength(3);
expect(after.map((candidate) => candidate.snippet)).toEqual(
expect.arrayContaining([
expect.stringContaining(
"Operations: Restarted the gateway after auth drift.; Tokens now line up again.",
),
expect.stringContaining(
"Bex: She prefers direct plans over open-ended maybes.; Better to offer one concrete time window.",
),
expect.stringContaining("Travel: Flight lands at 08:10."),
]),
);
});
it("records light/rem signals that reinforce deep promotion ranking", async () => {
@@ -210,24 +365,26 @@ describe("memory-core dreaming phases", () => {
expect(baseline).toHaveLength(1);
const baselineScore = baseline[0]!.score;
const { beforeAgentReply } = createHarness({
plugins: {
entries: {
"memory-core": {
config: {
dreaming: {
enabled: true,
phases: {
light: {
enabled: true,
limit: 10,
lookbackDays: 7,
},
rem: {
enabled: true,
limit: 10,
lookbackDays: 7,
minPatternStrength: 0,
const { beforeAgentReply } = createHarness(
{
plugins: {
entries: {
"memory-core": {
config: {
dreaming: {
enabled: true,
phases: {
light: {
enabled: true,
limit: 10,
lookbackDays: 7,
},
rem: {
enabled: true,
limit: 10,
lookbackDays: 7,
minPatternStrength: 0,
},
},
},
},
@@ -235,7 +392,8 @@ describe("memory-core dreaming phases", () => {
},
},
},
});
workspaceDir,
);
await beforeAgentReply(
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },

149
extensions/memory-core/src/dreaming-phases.ts
View File

@@ -83,6 +83,7 @@ const DAILY_INGESTION_STATE_RELATIVE_PATH = path.join("memory", ".dreams", "dail
const DAILY_INGESTION_SCORE = 0.62;
const DAILY_INGESTION_MAX_SNIPPET_CHARS = 280;
const DAILY_INGESTION_MIN_SNIPPET_CHARS = 8;
const DAILY_INGESTION_MAX_CHUNK_LINES = 4;
function asRecord(value: unknown): Record<string, unknown> | null {
if (!value || typeof value !== "object" || Array.isArray(value)) {
@@ -324,24 +325,137 @@ function isDayWithinLookback(day: string, cutoffMs: number): boolean {
return Number.isFinite(dayMs) && dayMs >= cutoffMs;
}
function normalizeDailySnippet(line: string): string | null {
const trimmed = line.trim();
if (!trimmed) {
return null;
}
if (trimmed.startsWith("#") || trimmed.startsWith("<!--")) {
return null;
}
const withoutListMarker = trimmed
function normalizeDailyListMarker(line: string): string {
return line
.replace(/^\d+\.\s+/, "")
.replace(/^[-*+]\s+/, "")
.trim();
}
function normalizeDailyHeading(line: string): string | null {
const trimmed = line.trim();
const match = trimmed.match(/^#{1,6}\s+(.+)$/);
if (!match) {
return null;
}
const heading = match[1] ? normalizeDailyListMarker(match[1]) : "";
if (!heading || DAILY_MEMORY_FILENAME_RE.test(heading)) {
return null;
}
return heading.slice(0, DAILY_INGESTION_MAX_SNIPPET_CHARS).replace(/\s+/g, " ");
}
function normalizeDailySnippet(line: string): string | null {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith("#") || trimmed.startsWith("<!--")) {
return null;
}
const withoutListMarker = normalizeDailyListMarker(trimmed);
if (withoutListMarker.length < DAILY_INGESTION_MIN_SNIPPET_CHARS) {
return null;
}
return withoutListMarker.slice(0, DAILY_INGESTION_MAX_SNIPPET_CHARS).replace(/\s+/g, " ");
}
type DailySnippetChunk = {
startLine: number;
endLine: number;
snippet: string;
};
function buildDailySnippetChunks(lines: string[], limit: number): DailySnippetChunk[] {
const chunks: DailySnippetChunk[] = [];
let activeHeading: string | null = null;
let chunkLines: string[] = [];
let chunkKind: "list" | "paragraph" | null = null;
let chunkStartLine = 0;
let chunkEndLine = 0;
const flushChunk = () => {
if (chunkLines.length === 0) {
chunkKind = null;
chunkStartLine = 0;
chunkEndLine = 0;
return;
}
const joiner = chunkKind === "list" ? "; " : " ";
const body = chunkLines.join(joiner).trim();
const prefixed = activeHeading ? `${activeHeading}: ${body}` : body;
const snippet = prefixed
.slice(0, DAILY_INGESTION_MAX_SNIPPET_CHARS)
.replace(/\s+/g, " ")
.trim();
if (snippet.length >= DAILY_INGESTION_MIN_SNIPPET_CHARS) {
chunks.push({
startLine: chunkStartLine,
endLine: chunkEndLine,
snippet,
});
}
chunkLines = [];
chunkKind = null;
chunkStartLine = 0;
chunkEndLine = 0;
};
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index];
if (typeof line !== "string") {
continue;
}
const heading = normalizeDailyHeading(line);
if (heading) {
flushChunk();
activeHeading = heading;
continue;
}
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith("<!--")) {
flushChunk();
continue;
}
const snippet = normalizeDailySnippet(line);
if (!snippet) {
flushChunk();
continue;
}
const nextKind = /^([-*+]\s+|\d+\.\s+)/.test(trimmed) ? "list" : "paragraph";
const nextChunkLines = chunkLines.length === 0 ? [snippet] : [...chunkLines, snippet];
const nextJoiner = nextKind === "list" ? "; " : " ";
const candidateBody = nextChunkLines.join(nextJoiner).trim();
const candidateSnippet = activeHeading ? `${activeHeading}: ${candidateBody}` : candidateBody;
const shouldSplit =
chunkLines.length > 0 &&
(chunkKind !== nextKind ||
chunkLines.length >= DAILY_INGESTION_MAX_CHUNK_LINES ||
candidateSnippet.length > DAILY_INGESTION_MAX_SNIPPET_CHARS);
if (shouldSplit) {
flushChunk();
}
if (chunkLines.length === 0) {
chunkStartLine = index + 1;
chunkKind = nextKind;
}
chunkLines.push(snippet);
chunkEndLine = index + 1;
if (chunks.length >= limit) {
break;
}
}
flushChunk();
return chunks.slice(0, limit);
}
function entryWithinLookback(entry: ShortTermRecallEntry, cutoffMs: number): boolean {
const byDay = (entry.recallDays ?? []).some((day) => isDayWithinLookback(day, cutoffMs));
if (byDay) {
@@ -507,22 +621,15 @@ async function collectDailyIngestionBatches(params: {
continue;
}
const lines = raw.split(/\r?\n/);
const chunks = buildDailySnippetChunks(lines, perFileCap);
const results: MemorySearchResult[] = [];
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index];
if (typeof line !== "string") {
continue;
}
const snippet = normalizeDailySnippet(line);
if (!snippet) {
continue;
}
for (const chunk of chunks) {
results.push({
path: relativePath,
startLine: index + 1,
endLine: index + 1,
startLine: chunk.startLine,
endLine: chunk.endLine,
score: DAILY_INGESTION_SCORE,
snippet,
snippet: chunk.snippet,
source: "memory",
});
if (results.length >= perFileCap || total + results.length >= totalCap) {