mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-05 06:10:21 +00:00
Dreaming: simplify sweep flow and add diary surface
This commit is contained in:
Vignesh Natarajan
@@ -66,11 +66,11 @@ function createCommandContext(args?: string): PluginCommandContext {
|
||||
}
|
||||
|
||||
describe("memory-core /dreaming command", () => {
|
||||
it("registers with a phase-oriented description", () => {
|
||||
it("registers with an enable/disable description", () => {
|
||||
const { command } = createHarness();
|
||||
expect(command.name).toBe("dreaming");
|
||||
expect(command.acceptsArgs).toBe(true);
|
||||
expect(command.description).toContain("dreaming phases");
|
||||
expect(command.description).toContain("Enable or disable");
|
||||
});
|
||||
|
||||
it("shows phase explanations when invoked without args", async () => {
|
||||
@@ -79,11 +79,10 @@ describe("memory-core /dreaming command", () => {
|
||||
|
||||
expect(result.text).toContain("Usage: /dreaming status");
|
||||
expect(result.text).toContain("Dreaming status:");
|
||||
expect(result.text).toContain("- light: sorts recent memory traces into DREAMS.md.");
|
||||
expect(result.text).toContain("- implementation detail: each sweep runs light -> REM -> deep.");
|
||||
expect(result.text).toContain(
|
||||
"- deep: promotes durable memories into MEMORY.md and handles recovery when memory is thin.",
|
||||
"- deep is the only stage that writes durable entries to MEMORY.md.",
|
||||
);
|
||||
expect(result.text).toContain("- rem: writes reflection and pattern notes into DREAMS.md.");
|
||||
});
|
||||
|
||||
it("persists global enablement under plugins.entries.memory-core.config.dreaming.enabled", async () => {
|
||||
@@ -98,6 +97,7 @@ describe("memory-core /dreaming command", () => {
|
||||
minScore: 0.9,
|
||||
},
|
||||
},
|
||||
frequency: "0 */6 * * *",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -110,31 +110,11 @@ describe("memory-core /dreaming command", () => {
|
||||
expect(runtime.config.writeConfigFile).toHaveBeenCalledTimes(1);
|
||||
expect(resolveStoredDreaming(getRuntimeConfig())).toMatchObject({
|
||||
enabled: false,
|
||||
phases: {
|
||||
deep: {
|
||||
minScore: 0.9,
|
||||
},
|
||||
},
|
||||
frequency: "0 */6 * * *",
|
||||
});
|
||||
expect(result.text).toContain("Dreaming disabled.");
|
||||
});
|
||||
|
||||
it("persists phase changes under plugins.entries.memory-core.config.dreaming.phases", async () => {
|
||||
const { command, runtime, getRuntimeConfig } = createHarness();
|
||||
|
||||
const result = await command.handler(createCommandContext("disable rem"));
|
||||
|
||||
expect(runtime.config.writeConfigFile).toHaveBeenCalledTimes(1);
|
||||
expect(resolveStoredDreaming(getRuntimeConfig())).toMatchObject({
|
||||
phases: {
|
||||
rem: {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result.text).toContain("REM phase disabled.");
|
||||
});
|
||||
|
||||
it("returns status without mutating config", async () => {
|
||||
const { command, runtime } = createHarness({
|
||||
plugins: {
|
||||
@@ -142,31 +122,25 @@ describe("memory-core /dreaming command", () => {
|
||||
"memory-core": {
|
||||
config: {
|
||||
dreaming: {
|
||||
timezone: "America/Los_Angeles",
|
||||
storage: {
|
||||
mode: "both",
|
||||
separateReports: true,
|
||||
},
|
||||
phases: {
|
||||
deep: {
|
||||
recencyHalfLifeDays: 21,
|
||||
maxAgeDays: 45,
|
||||
},
|
||||
},
|
||||
frequency: "15 */8 * * *",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
userTimezone: "America/Los_Angeles",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await command.handler(createCommandContext("status"));
|
||||
|
||||
expect(result.text).toContain("Dreaming status:");
|
||||
expect(result.text).toContain("- enabled: on (America/Los_Angeles)");
|
||||
expect(result.text).toContain("- storage: both + reports");
|
||||
expect(result.text).toContain("recencyHalfLifeDays=21");
|
||||
expect(result.text).toContain("maxAgeDays=45");
|
||||
expect(result.text).toContain("- sweep cadence: 15 */8 * * *");
|
||||
expect(result.text).toContain("- promotion policy: score>=0.8, recalls>=3, uniqueQueries>=3");
|
||||
expect(runtime.config.writeConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
import type { OpenClawConfig, OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core";
|
||||
import {
|
||||
resolveMemoryLightDreamingConfig,
|
||||
resolveMemoryRemDreamingConfig,
|
||||
resolveMemoryDreamingConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { resolveMemoryDreamingConfig } from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { resolveShortTermPromotionDreamingConfig } from "./dreaming.js";
|
||||
|
||||
type DreamingPhaseName = "light" | "deep" | "rem";
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
@@ -15,17 +9,6 @@ function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function normalizeDreamingPhase(value: unknown): DreamingPhaseName | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "light" || normalized === "deep" || normalized === "rem") {
|
||||
return normalized;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveMemoryCorePluginConfig(cfg: OpenClawConfig): Record<string, unknown> {
|
||||
const entry = asRecord(cfg.plugins?.entries?.["memory-core"]);
|
||||
return asRecord(entry?.config) ?? {};
|
||||
@@ -56,52 +39,15 @@ function updateDreamingEnabledInConfig(cfg: OpenClawConfig, enabled: boolean): O
|
||||
};
|
||||
}
|
||||
|
||||
function updateDreamingPhaseEnabledInConfig(
|
||||
cfg: OpenClawConfig,
|
||||
phase: DreamingPhaseName,
|
||||
enabled: boolean,
|
||||
): OpenClawConfig {
|
||||
const entries = { ...(cfg.plugins?.entries ?? {}) };
|
||||
const existingEntry = asRecord(entries["memory-core"]) ?? {};
|
||||
const existingConfig = asRecord(existingEntry.config) ?? {};
|
||||
const existingSleep = asRecord(existingConfig.dreaming) ?? {};
|
||||
const existingPhases = asRecord(existingSleep.phases) ?? {};
|
||||
const existingPhase = asRecord(existingPhases[phase]) ?? {};
|
||||
entries["memory-core"] = {
|
||||
...existingEntry,
|
||||
config: {
|
||||
...existingConfig,
|
||||
dreaming: {
|
||||
...existingSleep,
|
||||
phases: {
|
||||
...existingPhases,
|
||||
[phase]: {
|
||||
...existingPhase,
|
||||
enabled,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...cfg,
|
||||
plugins: {
|
||||
...cfg.plugins,
|
||||
entries,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatEnabled(value: boolean): string {
|
||||
return value ? "on" : "off";
|
||||
}
|
||||
|
||||
function formatPhaseGuide(): string {
|
||||
return [
|
||||
"- light: sorts recent memory traces into DREAMS.md.",
|
||||
"- deep: promotes durable memories into MEMORY.md and handles recovery when memory is thin.",
|
||||
"- rem: writes reflection and pattern notes into DREAMS.md.",
|
||||
"- implementation detail: each sweep runs light -> REM -> deep.",
|
||||
"- deep is the only stage that writes durable entries to MEMORY.md.",
|
||||
"- DREAMS.md is for human-readable dreaming summaries and diary entries.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -112,18 +58,13 @@ function formatStatus(cfg: OpenClawConfig): string {
|
||||
cfg,
|
||||
});
|
||||
const deep = resolveShortTermPromotionDreamingConfig({ pluginConfig, cfg });
|
||||
const light = resolveMemoryLightDreamingConfig({ pluginConfig, cfg });
|
||||
const rem = resolveMemoryRemDreamingConfig({ pluginConfig, cfg });
|
||||
const timezone = dreaming.timezone ? ` (${dreaming.timezone})` : "";
|
||||
|
||||
return [
|
||||
"Dreaming status:",
|
||||
`- enabled: ${formatEnabled(dreaming.enabled)}${timezone}`,
|
||||
`- storage: ${dreaming.storage.mode}${dreaming.storage.separateReports ? " + reports" : ""}`,
|
||||
`- verboseLogging: ${formatEnabled(dreaming.verboseLogging)}`,
|
||||
`- light: ${formatEnabled(light.enabled)} · cadence=${light.enabled ? light.cron : "disabled"} · lookbackDays=${light.lookbackDays} · limit=${light.limit}`,
|
||||
`- deep: ${formatEnabled(deep.enabled)} · cadence=${deep.enabled ? deep.cron : "disabled"} · limit=${deep.limit} · minScore=${deep.minScore} · minRecallCount=${deep.minRecallCount} · minUniqueQueries=${deep.minUniqueQueries} · recencyHalfLifeDays=${deep.recencyHalfLifeDays} · maxAgeDays=${deep.maxAgeDays ?? "none"}`,
|
||||
`- rem: ${formatEnabled(rem.enabled)} · cadence=${rem.enabled ? rem.cron : "disabled"} · lookbackDays=${rem.lookbackDays} · limit=${rem.limit} · minPatternStrength=${rem.minPatternStrength}`,
|
||||
`- sweep cadence: ${dreaming.frequency}`,
|
||||
`- promotion policy: score>=${deep.minScore}, recalls>=${deep.minRecallCount}, uniqueQueries>=${deep.minUniqueQueries}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -131,8 +72,6 @@ function formatUsage(includeStatus: string): string {
|
||||
return [
|
||||
"Usage: /dreaming status",
|
||||
"Usage: /dreaming on|off",
|
||||
"Usage: /dreaming enable light|deep|rem",
|
||||
"Usage: /dreaming disable light|deep|rem",
|
||||
"",
|
||||
includeStatus,
|
||||
"",
|
||||
@@ -144,11 +83,11 @@ function formatUsage(includeStatus: string): string {
|
||||
export function registerDreamingCommand(api: OpenClawPluginApi): void {
|
||||
api.registerCommand({
|
||||
name: "dreaming",
|
||||
description: "Configure memory dreaming phases and durable promotion behavior.",
|
||||
description: "Enable or disable memory dreaming.",
|
||||
acceptsArgs: true,
|
||||
handler: async (ctx) => {
|
||||
const args = ctx.args?.trim() ?? "";
|
||||
const [firstToken = "", secondToken = ""] = args
|
||||
const [firstToken = ""] = args
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map((token) => token.toLowerCase());
|
||||
@@ -180,20 +119,6 @@ export function registerDreamingCommand(api: OpenClawPluginApi): void {
|
||||
};
|
||||
}
|
||||
|
||||
const phase = normalizeDreamingPhase(secondToken);
|
||||
if ((firstToken === "enable" || firstToken === "disable") && phase) {
|
||||
const enabled = firstToken === "enable";
|
||||
const nextConfig = updateDreamingPhaseEnabledInConfig(currentConfig, phase, enabled);
|
||||
await api.runtime.config.writeConfigFile(nextConfig);
|
||||
return {
|
||||
text: [
|
||||
`${phase.toUpperCase()} phase ${enabled ? "enabled" : "disabled"}.`,
|
||||
"",
|
||||
formatStatus(nextConfig),
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
return { text: formatUsage(formatStatus(currentConfig)) };
|
||||
},
|
||||
});
|
||||
|
||||
@@ -24,8 +24,6 @@ describe("dreaming markdown storage", () => {
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: remember the API key is fake"],
|
||||
nowMs: Date.parse("2026-04-05T10:00:00Z"),
|
||||
timezone: "UTC",
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
@@ -34,7 +32,7 @@ describe("dreaming markdown storage", () => {
|
||||
|
||||
expect(result.inlinePath).toBe(path.join(workspaceDir, "DREAMS.md"));
|
||||
const content = await fs.readFile(result.inlinePath!, "utf-8");
|
||||
expect(content).toContain("## 2026-04-05 - Light Sleep");
|
||||
expect(content).toContain("## Light Sleep");
|
||||
expect(content).toContain("- Candidate: remember the API key is fake");
|
||||
});
|
||||
|
||||
@@ -45,8 +43,6 @@ describe("dreaming markdown storage", () => {
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: first block"],
|
||||
nowMs: Date.parse("2026-04-05T10:00:00Z"),
|
||||
timezone: "UTC",
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
@@ -56,8 +52,6 @@ describe("dreaming markdown storage", () => {
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
bodyLines: ["- Theme: `focus` kept surfacing."],
|
||||
nowMs: Date.parse("2026-04-05T11:00:00Z"),
|
||||
timezone: "UTC",
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
@@ -66,76 +60,31 @@ describe("dreaming markdown storage", () => {
|
||||
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content).toContain("## 2026-04-05 - Light Sleep");
|
||||
expect(content).toContain("## 2026-04-05 - REM Sleep");
|
||||
expect(content).toContain("## Light Sleep");
|
||||
expect(content).toContain("## REM Sleep");
|
||||
expect(content).toContain("- Candidate: first block");
|
||||
expect(content).toContain("- Theme: `focus` kept surfacing.");
|
||||
});
|
||||
|
||||
it("preserves prior days when writing later inline dreaming output", async () => {
|
||||
it("reuses existing lowercase dreams.md when present", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const lowercasePath = path.join(workspaceDir, "dreams.md");
|
||||
await fs.writeFile(lowercasePath, "# Scratch\n\n", "utf-8");
|
||||
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: day one"],
|
||||
nowMs: Date.parse("2026-04-05T10:00:00Z"),
|
||||
timezone: "UTC",
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: day two"],
|
||||
nowMs: Date.parse("2026-04-06T10:00:00Z"),
|
||||
timezone: "UTC",
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("## 2026-04-05 - Light Sleep");
|
||||
expect(content).toContain("## 2026-04-06 - Light Sleep");
|
||||
expect(content).toContain("- Candidate: day one");
|
||||
expect(content).toContain("- Candidate: day two");
|
||||
});
|
||||
|
||||
it("replaces the same day and phase block instead of appending duplicates", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
const result = await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
bodyLines: ["- Theme: initial pass"],
|
||||
nowMs: Date.parse("2026-04-05T10:00:00Z"),
|
||||
timezone: "UTC",
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
bodyLines: ["- Theme: refreshed pass"],
|
||||
nowMs: Date.parse("2026-04-05T14:00:00Z"),
|
||||
timezone: "UTC",
|
||||
bodyLines: ["- Theme: `glacier` kept surfacing."],
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("## 2026-04-05 - REM Sleep");
|
||||
expect(content).toContain("- Theme: refreshed pass");
|
||||
expect(content).not.toContain("- Theme: initial pass");
|
||||
expect(content.match(/## 2026-04-05 - REM Sleep/g)).toHaveLength(1);
|
||||
expect(result.inlinePath).toBe(lowercasePath);
|
||||
const content = await fs.readFile(lowercasePath, "utf-8");
|
||||
expect(content).toContain("## REM Sleep");
|
||||
expect(content).toContain("- Theme: `glacier` kept surfacing.");
|
||||
});
|
||||
|
||||
it("still writes deep reports to the per-phase report directory", async () => {
|
||||
@@ -156,5 +105,10 @@ describe("dreaming markdown storage", () => {
|
||||
const content = await fs.readFile(reportPath!, "utf-8");
|
||||
expect(content).toContain("# Deep Sleep");
|
||||
expect(content).toContain("- Promoted: durable preference");
|
||||
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
const dreamsContent = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(dreamsContent).toContain("## Deep Sleep");
|
||||
expect(dreamsContent).toContain("- Promoted: durable preference");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,24 +6,28 @@ import {
|
||||
type MemoryDreamingStorageConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
|
||||
const DAILY_PHASE_HEADINGS: Record<Exclude<MemoryDreamingPhaseName, "deep">, string> = {
|
||||
light: "## Light Sleep",
|
||||
rem: "## REM Sleep",
|
||||
};
|
||||
const DEEP_PHASE_HEADING = "## Deep Sleep";
|
||||
|
||||
const DAILY_PHASE_LABELS: Record<Exclude<MemoryDreamingPhaseName, "deep">, string> = {
|
||||
light: "light",
|
||||
rem: "rem",
|
||||
};
|
||||
|
||||
const DREAMS_FILENAME = "DREAMS.md";
|
||||
const PRIMARY_DREAMS_FILENAME = "DREAMS.md";
|
||||
const DREAMS_FILENAME_ALIASES = [PRIMARY_DREAMS_FILENAME, "dreams.md"] as const;
|
||||
|
||||
function resolvePhaseMarkers(
|
||||
phase: Exclude<MemoryDreamingPhaseName, "deep">,
|
||||
isoDay: string,
|
||||
): {
|
||||
function resolvePhaseMarkers(phase: Exclude<MemoryDreamingPhaseName, "deep">): {
|
||||
start: string;
|
||||
end: string;
|
||||
} {
|
||||
const label = DAILY_PHASE_LABELS[phase];
|
||||
return {
|
||||
start: `<!-- openclaw:dreaming:${isoDay}:${label}:start -->`,
|
||||
end: `<!-- openclaw:dreaming:${isoDay}:${label}:end -->`,
|
||||
start: `<!-- openclaw:dreaming:${label}:start -->`,
|
||||
end: `<!-- openclaw:dreaming:${label}:end -->`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -57,15 +61,42 @@ function escapeRegex(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
function resolveDreamsPath(workspaceDir: string): string {
|
||||
return path.join(workspaceDir, DREAMS_FILENAME);
|
||||
async function resolveDreamsPath(workspaceDir: string): Promise<string> {
|
||||
for (const candidate of DREAMS_FILENAME_ALIASES) {
|
||||
const target = path.join(workspaceDir, candidate);
|
||||
try {
|
||||
await fs.access(target);
|
||||
return target;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
return path.join(workspaceDir, PRIMARY_DREAMS_FILENAME);
|
||||
}
|
||||
|
||||
function resolveDreamsBlockHeading(
|
||||
phase: Exclude<MemoryDreamingPhaseName, "deep">,
|
||||
isoDay: string,
|
||||
): string {
|
||||
return `## ${isoDay} - ${phase === "light" ? "Light Sleep" : "REM Sleep"}`;
|
||||
async function writeInlineDeepDreamingBlock(params: {
|
||||
workspaceDir: string;
|
||||
body: string;
|
||||
}): Promise<string> {
|
||||
const inlinePath = await resolveDreamsPath(params.workspaceDir);
|
||||
await fs.mkdir(path.dirname(inlinePath), { recursive: true });
|
||||
const original = await fs.readFile(inlinePath, "utf-8").catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
const updated = replaceManagedBlock({
|
||||
original,
|
||||
heading: DEEP_PHASE_HEADING,
|
||||
startMarker: "<!-- openclaw:dreaming:deep:start -->",
|
||||
endMarker: "<!-- openclaw:dreaming:deep:end -->",
|
||||
body: params.body,
|
||||
});
|
||||
await fs.writeFile(inlinePath, withTrailingNewline(updated), "utf-8");
|
||||
return inlinePath;
|
||||
}
|
||||
|
||||
function resolveSeparateReportPath(
|
||||
@@ -95,23 +126,23 @@ export async function writeDailyDreamingPhaseBlock(params: {
|
||||
storage: MemoryDreamingStorageConfig;
|
||||
}): Promise<{ inlinePath?: string; reportPath?: string }> {
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const isoDay = formatMemoryDreamingDay(nowMs, params.timezone);
|
||||
const body = params.bodyLines.length > 0 ? params.bodyLines.join("\n") : "- No notable updates.";
|
||||
let inlinePath: string | undefined;
|
||||
let reportPath: string | undefined;
|
||||
|
||||
if (shouldWriteInline(params.storage)) {
|
||||
inlinePath = resolveDreamsPath(params.workspaceDir);
|
||||
inlinePath = await resolveDreamsPath(params.workspaceDir);
|
||||
await fs.mkdir(path.dirname(inlinePath), { recursive: true });
|
||||
const original = await fs.readFile(inlinePath, "utf-8").catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
const markers = resolvePhaseMarkers(params.phase, isoDay);
|
||||
const markers = resolvePhaseMarkers(params.phase);
|
||||
const updated = replaceManagedBlock({
|
||||
original,
|
||||
heading: resolveDreamsBlockHeading(params.phase, isoDay),
|
||||
heading: DAILY_PHASE_HEADINGS[params.phase],
|
||||
startMarker: markers.start,
|
||||
endMarker: markers.end,
|
||||
body,
|
||||
@@ -149,13 +180,19 @@ export async function writeDeepDreamingReport(params: {
|
||||
timezone?: string;
|
||||
storage: MemoryDreamingStorageConfig;
|
||||
}): Promise<string | undefined> {
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const body = params.bodyLines.length > 0 ? params.bodyLines.join("\n") : "- No durable changes.";
|
||||
await writeInlineDeepDreamingBlock({
|
||||
workspaceDir: params.workspaceDir,
|
||||
body,
|
||||
});
|
||||
|
||||
if (!shouldWriteSeparate(params.storage)) {
|
||||
return undefined;
|
||||
}
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
|
||||
const reportPath = resolveSeparateReportPath(params.workspaceDir, "deep", nowMs, params.timezone);
|
||||
await fs.mkdir(path.dirname(reportPath), { recursive: true });
|
||||
const body = params.bodyLines.length > 0 ? params.bodyLines.join("\n") : "- No durable changes.";
|
||||
await fs.writeFile(reportPath, `# Deep Sleep\n\n${body}\n`, "utf-8");
|
||||
return reportPath;
|
||||
}
|
||||
|
||||
340
extensions/memory-core/src/dreaming-narrative.test.ts
Normal file
340
extensions/memory-core/src/dreaming-narrative.test.ts
Normal file
@@ -0,0 +1,340 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
appendNarrativeEntry,
|
||||
buildDiaryEntry,
|
||||
buildNarrativePrompt,
|
||||
extractNarrativeText,
|
||||
formatNarrativeDate,
|
||||
generateAndAppendDreamNarrative,
|
||||
type NarrativePhaseData,
|
||||
} from "./dreaming-narrative.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
async function createTempWorkspace(): Promise<string> {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-dreaming-narrative-"));
|
||||
tempDirs.push(workspaceDir);
|
||||
return workspaceDir;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
describe("buildNarrativePrompt", () => {
|
||||
it("builds a prompt from snippets only", () => {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "light",
|
||||
snippets: ["user prefers dark mode", "API key rotation scheduled"],
|
||||
};
|
||||
const prompt = buildNarrativePrompt(data);
|
||||
expect(prompt).toContain("user prefers dark mode");
|
||||
expect(prompt).toContain("API key rotation scheduled");
|
||||
expect(prompt).not.toContain("Recurring themes");
|
||||
});
|
||||
|
||||
it("includes themes when provided", () => {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "rem",
|
||||
snippets: ["config migration path"],
|
||||
themes: ["infrastructure", "deployment"],
|
||||
};
|
||||
const prompt = buildNarrativePrompt(data);
|
||||
expect(prompt).toContain("Recurring themes");
|
||||
expect(prompt).toContain("infrastructure");
|
||||
expect(prompt).toContain("deployment");
|
||||
});
|
||||
|
||||
it("includes promotions for deep phase", () => {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "deep",
|
||||
snippets: ["trading bot uses bracket orders"],
|
||||
promotions: ["always use stop-loss on options trades"],
|
||||
};
|
||||
const prompt = buildNarrativePrompt(data);
|
||||
expect(prompt).toContain("crystallized");
|
||||
expect(prompt).toContain("always use stop-loss on options trades");
|
||||
});
|
||||
|
||||
it("caps snippets at 12", () => {
|
||||
const snippets = Array.from({ length: 20 }, (_, i) => `snippet-${i}`);
|
||||
const prompt = buildNarrativePrompt({ phase: "light", snippets });
|
||||
expect(prompt).toContain("snippet-11");
|
||||
expect(prompt).not.toContain("snippet-12");
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractNarrativeText", () => {
|
||||
it("extracts string content from assistant message", () => {
|
||||
const messages = [
|
||||
{ role: "user", content: "hello" },
|
||||
{ role: "assistant", content: "The workspace hummed quietly." },
|
||||
];
|
||||
expect(extractNarrativeText(messages)).toBe("The workspace hummed quietly.");
|
||||
});
|
||||
|
||||
it("extracts from content array with text blocks", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: "First paragraph." },
|
||||
{ type: "text", text: "Second paragraph." },
|
||||
],
|
||||
},
|
||||
];
|
||||
expect(extractNarrativeText(messages)).toBe("First paragraph.\nSecond paragraph.");
|
||||
});
|
||||
|
||||
it("returns null when no assistant message exists", () => {
|
||||
const messages = [{ role: "user", content: "hello" }];
|
||||
expect(extractNarrativeText(messages)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for empty assistant content", () => {
|
||||
const messages = [{ role: "assistant", content: " " }];
|
||||
expect(extractNarrativeText(messages)).toBeNull();
|
||||
});
|
||||
|
||||
it("picks the last assistant message", () => {
|
||||
const messages = [
|
||||
{ role: "assistant", content: "First response." },
|
||||
{ role: "user", content: "more" },
|
||||
{ role: "assistant", content: "Final response." },
|
||||
];
|
||||
expect(extractNarrativeText(messages)).toBe("Final response.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatNarrativeDate", () => {
|
||||
it("formats a UTC date", () => {
|
||||
const date = formatNarrativeDate(Date.parse("2026-04-05T03:00:00Z"), "UTC");
|
||||
expect(date).toContain("April");
|
||||
expect(date).toContain("2026");
|
||||
expect(date).toContain("3:00");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildDiaryEntry", () => {
|
||||
it("formats narrative with date and separators", () => {
|
||||
const entry = buildDiaryEntry("The code drifted gently.", "April 5, 2026, 3:00 AM");
|
||||
expect(entry).toContain("---");
|
||||
expect(entry).toContain("*April 5, 2026, 3:00 AM*");
|
||||
expect(entry).toContain("The code drifted gently.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendNarrativeEntry", () => {
|
||||
it("creates DREAMS.md with diary header on fresh workspace", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const dreamsPath = await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Fragments of authentication logic kept surfacing.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
expect(dreamsPath).toBe(path.join(workspaceDir, "DREAMS.md"));
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content).toContain("# Dream Diary");
|
||||
expect(content).toContain("Fragments of authentication logic kept surfacing.");
|
||||
expect(content).toContain("<!-- openclaw:dreaming:diary:start -->");
|
||||
expect(content).toContain("<!-- openclaw:dreaming:diary:end -->");
|
||||
});
|
||||
|
||||
it("appends a second entry within the diary markers", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "First dream.",
|
||||
nowMs: Date.parse("2026-04-04T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Second dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("First dream.");
|
||||
expect(content).toContain("Second dream.");
|
||||
// Both entries should be between start and end markers.
|
||||
const start = content.indexOf("<!-- openclaw:dreaming:diary:start -->");
|
||||
const end = content.indexOf("<!-- openclaw:dreaming:diary:end -->");
|
||||
const firstIdx = content.indexOf("First dream.");
|
||||
const secondIdx = content.indexOf("Second dream.");
|
||||
expect(firstIdx).toBeGreaterThan(start);
|
||||
expect(secondIdx).toBeGreaterThan(firstIdx);
|
||||
expect(secondIdx).toBeLessThan(end);
|
||||
});
|
||||
|
||||
it("prepends diary before existing managed blocks", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(
|
||||
dreamsPath,
|
||||
"## Light Sleep\n<!-- openclaw:dreaming:light:start -->\n- Candidate: test\n<!-- openclaw:dreaming:light:end -->\n",
|
||||
"utf-8",
|
||||
);
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "The workspace was quiet tonight.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
const diaryIdx = content.indexOf("# Dream Diary");
|
||||
const lightIdx = content.indexOf("## Light Sleep");
|
||||
// Diary should come before the managed block.
|
||||
expect(diaryIdx).toBeLessThan(lightIdx);
|
||||
expect(content).toContain("The workspace was quiet tonight.");
|
||||
});
|
||||
|
||||
it("reuses existing dreams file when present", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(dreamsPath, "# Existing\n", "utf-8");
|
||||
const result = await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Appended dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
expect(result).toBe(dreamsPath);
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content).toContain("Appended dream.");
|
||||
// Original content should still be there, after the diary.
|
||||
expect(content).toContain("# Existing");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateAndAppendDreamNarrative", () => {
|
||||
function createMockSubagent(responseText: string) {
|
||||
return {
|
||||
run: vi.fn().mockResolvedValue({ runId: "run-123" }),
|
||||
waitForRun: vi.fn().mockResolvedValue({ status: "ok" }),
|
||||
getSessionMessages: vi.fn().mockResolvedValue({
|
||||
messages: [
|
||||
{ role: "user", content: "prompt" },
|
||||
{ role: "assistant", content: responseText },
|
||||
],
|
||||
}),
|
||||
deleteSession: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
it("generates narrative and writes diary entry", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const subagent = createMockSubagent("The repository whispered of forgotten endpoints.");
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: {
|
||||
phase: "light",
|
||||
snippets: ["API endpoints need authentication"],
|
||||
},
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.run).toHaveBeenCalledOnce();
|
||||
expect(subagent.run.mock.calls[0][0]).toMatchObject({
|
||||
deliver: false,
|
||||
});
|
||||
expect(subagent.waitForRun).toHaveBeenCalledOnce();
|
||||
expect(subagent.deleteSession).toHaveBeenCalledOnce();
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("The repository whispered of forgotten endpoints.");
|
||||
expect(logger.info).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips narrative when no snippets are available", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const subagent = createMockSubagent("Should not appear.");
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "light", snippets: [] },
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.run).not.toHaveBeenCalled();
|
||||
const exists = await fs
|
||||
.access(path.join(workspaceDir, "DREAMS.md"))
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
expect(exists).toBe(false);
|
||||
});
|
||||
|
||||
it("handles subagent timeout gracefully", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.waitForRun.mockResolvedValue({ status: "timeout" });
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "deep", snippets: ["some memory"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
// Should not throw, should warn.
|
||||
expect(logger.warn).toHaveBeenCalled();
|
||||
const exists = await fs
|
||||
.access(path.join(workspaceDir, "DREAMS.md"))
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
expect(exists).toBe(false);
|
||||
});
|
||||
|
||||
it("handles subagent error gracefully", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.run.mockRejectedValue(new Error("connection failed"));
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "rem", snippets: ["pattern surfaced"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
// Should not throw.
|
||||
expect(logger.warn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cleans up session even on failure", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.getSessionMessages.mockRejectedValue(new Error("fetch failed"));
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "light", snippets: ["memory fragment"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.deleteSession).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
299
extensions/memory-core/src/dreaming-narrative.ts
Normal file
299
extensions/memory-core/src/dreaming-narrative.ts
Normal file
@@ -0,0 +1,299 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
type SubagentSurface = {
|
||||
run: (params: {
|
||||
sessionKey: string;
|
||||
message: string;
|
||||
extraSystemPrompt?: string;
|
||||
deliver?: boolean;
|
||||
}) => Promise<{ runId: string }>;
|
||||
waitForRun: (params: {
|
||||
runId: string;
|
||||
timeoutMs?: number;
|
||||
}) => Promise<{ status: string; error?: string }>;
|
||||
getSessionMessages: (params: {
|
||||
sessionKey: string;
|
||||
limit?: number;
|
||||
}) => Promise<{ messages: unknown[] }>;
|
||||
deleteSession: (params: { sessionKey: string }) => Promise<void>;
|
||||
};
|
||||
|
||||
export type NarrativePhaseData = {
|
||||
phase: "light" | "deep" | "rem";
|
||||
/** Short memory snippets the phase processed. */
|
||||
snippets: string[];
|
||||
/** Concept tags / themes that surfaced (REM and light). */
|
||||
themes?: string[];
|
||||
/** Snippets that were promoted to durable memory (deep). */
|
||||
promotions?: string[];
|
||||
};
|
||||
|
||||
type Logger = {
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
};
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────
|
||||
|
||||
const NARRATIVE_SYSTEM_PROMPT = [
|
||||
"You are keeping a dream diary. Write a single entry in first person.",
|
||||
"",
|
||||
"Voice & tone:",
|
||||
"- You are a curious, gentle, slightly whimsical mind reflecting on the day.",
|
||||
"- Write like a poet who happens to be a programmer — sensory, warm, occasionally funny.",
|
||||
"- Mix the technical and the tender: code and constellations, APIs and afternoon light.",
|
||||
"- Let the fragments surprise you into unexpected connections and small epiphanies.",
|
||||
"",
|
||||
"What you might include (vary each entry, never all at once):",
|
||||
"- A tiny poem or haiku woven naturally into the prose",
|
||||
"- A small sketch described in words — a doodle in the margin of the diary",
|
||||
"- A quiet rumination or philosophical aside",
|
||||
"- Sensory details: the hum of a server, the color of a sunset in hex, rain on a window",
|
||||
"- Gentle humor or playful wordplay",
|
||||
"- An observation that connects two distant memories in an unexpected way",
|
||||
"",
|
||||
"Rules:",
|
||||
"- Draw from the memory fragments provided — weave them into the entry.",
|
||||
'- Never say "I\'m dreaming", "in my dream", "as I dream", or any meta-commentary about dreaming.',
|
||||
'- Never mention "AI", "agent", "LLM", "model", "language model", or any technical self-reference.',
|
||||
"- Do NOT use markdown headers, bullet points, or any formatting — just flowing prose.",
|
||||
"- Keep it between 80-180 words. Quality over quantity.",
|
||||
"- Output ONLY the diary entry. No preamble, no sign-off, no commentary.",
|
||||
].join("\n");
|
||||
|
||||
const NARRATIVE_TIMEOUT_MS = 60_000;
|
||||
const DREAMS_FILENAMES = ["DREAMS.md", "dreams.md"] as const;
|
||||
const DIARY_START_MARKER = "<!-- openclaw:dreaming:diary:start -->";
|
||||
const DIARY_END_MARKER = "<!-- openclaw:dreaming:diary:end -->";
|
||||
|
||||
// ── Prompt building ────────────────────────────────────────────────────
|
||||
|
||||
export function buildNarrativePrompt(data: NarrativePhaseData): string {
|
||||
const lines: string[] = [];
|
||||
lines.push("Write a dream diary entry from these memory fragments:\n");
|
||||
|
||||
for (const snippet of data.snippets.slice(0, 12)) {
|
||||
lines.push(`- ${snippet}`);
|
||||
}
|
||||
|
||||
if (data.themes?.length) {
|
||||
lines.push("\nRecurring themes:");
|
||||
for (const theme of data.themes.slice(0, 6)) {
|
||||
lines.push(`- ${theme}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (data.promotions?.length) {
|
||||
lines.push("\nMemories that crystallized into something lasting:");
|
||||
for (const promo of data.promotions.slice(0, 5)) {
|
||||
lines.push(`- ${promo}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
// ── Message extraction ─────────────────────────────────────────────────
|
||||
|
||||
export function extractNarrativeText(messages: unknown[]): string | null {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const msg = messages[i];
|
||||
if (!msg || typeof msg !== "object" || Array.isArray(msg)) {
|
||||
continue;
|
||||
}
|
||||
const record = msg as Record<string, unknown>;
|
||||
if (record.role !== "assistant") {
|
||||
continue;
|
||||
}
|
||||
const content = record.content;
|
||||
if (typeof content === "string" && content.trim().length > 0) {
|
||||
return content.trim();
|
||||
}
|
||||
if (Array.isArray(content)) {
|
||||
const text = content
|
||||
.filter(
|
||||
(part: unknown) =>
|
||||
part &&
|
||||
typeof part === "object" &&
|
||||
!Array.isArray(part) &&
|
||||
(part as Record<string, unknown>).type === "text" &&
|
||||
typeof (part as Record<string, unknown>).text === "string",
|
||||
)
|
||||
.map((part) => (part as { text: string }).text)
|
||||
.join("\n")
|
||||
.trim();
|
||||
if (text.length > 0) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Date formatting ────────────────────────────────────────────────────
|
||||
|
||||
export function formatNarrativeDate(epochMs: number, timezone?: string): string {
|
||||
const opts: Intl.DateTimeFormatOptions = {
|
||||
timeZone: timezone ?? "UTC",
|
||||
year: "numeric",
|
||||
month: "long",
|
||||
day: "numeric",
|
||||
hour: "numeric",
|
||||
minute: "2-digit",
|
||||
hour12: true,
|
||||
};
|
||||
return new Intl.DateTimeFormat("en-US", opts).format(new Date(epochMs));
|
||||
}
|
||||
|
||||
// ── DREAMS.md file I/O ─────────────────────────────────────────────────
|
||||
|
||||
async function resolveDreamsPath(workspaceDir: string): Promise<string> {
|
||||
for (const name of DREAMS_FILENAMES) {
|
||||
const target = path.join(workspaceDir, name);
|
||||
try {
|
||||
await fs.access(target);
|
||||
return target;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
return path.join(workspaceDir, DREAMS_FILENAMES[0]);
|
||||
}
|
||||
|
||||
export function buildDiaryEntry(narrative: string, dateStr: string): string {
|
||||
return `\n---\n\n*${dateStr}*\n\n${narrative}\n`;
|
||||
}
|
||||
|
||||
export async function appendNarrativeEntry(params: {
|
||||
workspaceDir: string;
|
||||
narrative: string;
|
||||
nowMs: number;
|
||||
timezone?: string;
|
||||
}): Promise<string> {
|
||||
const dreamsPath = await resolveDreamsPath(params.workspaceDir);
|
||||
await fs.mkdir(path.dirname(dreamsPath), { recursive: true });
|
||||
|
||||
const dateStr = formatNarrativeDate(params.nowMs, params.timezone);
|
||||
const entry = buildDiaryEntry(params.narrative, dateStr);
|
||||
|
||||
let existing = "";
|
||||
try {
|
||||
existing = await fs.readFile(dreamsPath, "utf-8");
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
let updated: string;
|
||||
if (existing.includes(DIARY_START_MARKER) && existing.includes(DIARY_END_MARKER)) {
|
||||
// Append entry before end marker.
|
||||
const endIdx = existing.lastIndexOf(DIARY_END_MARKER);
|
||||
updated = existing.slice(0, endIdx) + entry + "\n" + existing.slice(endIdx);
|
||||
} else if (existing.includes(DIARY_START_MARKER)) {
|
||||
// Start marker without end — append entry and add end marker.
|
||||
const startIdx = existing.indexOf(DIARY_START_MARKER) + DIARY_START_MARKER.length;
|
||||
updated =
|
||||
existing.slice(0, startIdx) +
|
||||
entry +
|
||||
"\n" +
|
||||
DIARY_END_MARKER +
|
||||
"\n" +
|
||||
existing.slice(startIdx);
|
||||
} else {
|
||||
// No diary section yet — create one.
|
||||
const diarySection = `# Dream Diary\n\n${DIARY_START_MARKER}${entry}\n${DIARY_END_MARKER}\n`;
|
||||
if (existing.trim().length === 0) {
|
||||
updated = diarySection;
|
||||
} else {
|
||||
// Prepend diary before any existing managed blocks.
|
||||
updated = diarySection + "\n" + existing;
|
||||
}
|
||||
}
|
||||
|
||||
await fs.writeFile(dreamsPath, updated.endsWith("\n") ? updated : `${updated}\n`, "utf-8");
|
||||
return dreamsPath;
|
||||
}
|
||||
|
||||
// ── Orchestrator ───────────────────────────────────────────────────────
|
||||
|
||||
export async function generateAndAppendDreamNarrative(params: {
|
||||
subagent: SubagentSurface;
|
||||
workspaceDir: string;
|
||||
data: NarrativePhaseData;
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
logger: Logger;
|
||||
}): Promise<void> {
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
|
||||
if (params.data.snippets.length === 0 && !params.data.promotions?.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionKey = `dreaming-narrative-${params.data.phase}-${nowMs}`;
|
||||
const message = buildNarrativePrompt(params.data);
|
||||
|
||||
try {
|
||||
const { runId } = await params.subagent.run({
|
||||
sessionKey,
|
||||
message,
|
||||
extraSystemPrompt: NARRATIVE_SYSTEM_PROMPT,
|
||||
deliver: false,
|
||||
});
|
||||
|
||||
const result = await params.subagent.waitForRun({
|
||||
runId,
|
||||
timeoutMs: NARRATIVE_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
if (result.status !== "ok") {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation ended with status=${result.status} for ${params.data.phase} phase.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const { messages } = await params.subagent.getSessionMessages({
|
||||
sessionKey,
|
||||
limit: 5,
|
||||
});
|
||||
|
||||
const narrative = extractNarrativeText(messages);
|
||||
if (!narrative) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation produced no text for ${params.data.phase} phase.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir: params.workspaceDir,
|
||||
narrative,
|
||||
nowMs,
|
||||
timezone: params.timezone,
|
||||
});
|
||||
|
||||
params.logger.info(
|
||||
`memory-core: dream diary entry written for ${params.data.phase} phase [workspace=${params.workspaceDir}].`,
|
||||
);
|
||||
} catch (err) {
|
||||
// Narrative generation is best-effort — never fail the parent phase.
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation failed for ${params.data.phase} phase: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
} finally {
|
||||
// Clean up the transient session.
|
||||
try {
|
||||
await params.subagent.deleteSession({ sessionKey });
|
||||
} catch {
|
||||
// Ignore cleanup failures.
|
||||
}
|
||||
}
|
||||
}
|
||||
268
extensions/memory-core/src/dreaming-phases.test.ts
Normal file
268
extensions/memory-core/src/dreaming-phases.test.ts
Normal file
@@ -0,0 +1,268 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig, OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { registerMemoryDreamingPhases } from "./dreaming-phases.js";
|
||||
import {
|
||||
rankShortTermPromotionCandidates,
|
||||
recordShortTermRecalls,
|
||||
resolveShortTermPhaseSignalStorePath,
|
||||
} from "./short-term-promotion.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
async function createTempWorkspace(): Promise<string> {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-dreaming-phases-"));
|
||||
tempDirs.push(workspaceDir);
|
||||
return workspaceDir;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
function createHarness(config: OpenClawConfig) {
|
||||
let beforeAgentReply:
|
||||
| ((
|
||||
event: { cleanedBody: string },
|
||||
ctx: { trigger?: string; workspaceDir?: string },
|
||||
) => Promise<unknown>)
|
||||
| undefined;
|
||||
const logger = {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
|
||||
const api = {
|
||||
config,
|
||||
pluginConfig: {},
|
||||
logger,
|
||||
registerHook: vi.fn(),
|
||||
on: vi.fn((name: string, handler: unknown) => {
|
||||
if (name === "before_agent_reply") {
|
||||
beforeAgentReply = handler as typeof beforeAgentReply;
|
||||
}
|
||||
}),
|
||||
} as unknown as OpenClawPluginApi;
|
||||
|
||||
registerMemoryDreamingPhases(api);
|
||||
if (!beforeAgentReply) {
|
||||
throw new Error("before_agent_reply hook not registered");
|
||||
}
|
||||
return { beforeAgentReply, logger };
|
||||
}
|
||||
|
||||
describe("memory-core dreaming phases", () => {
|
||||
it("checkpoints daily ingestion and skips unchanged daily files", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
const dailyPath = path.join(workspaceDir, "memory", "2026-04-05.md");
|
||||
await fs.writeFile(
|
||||
dailyPath,
|
||||
["# 2026-04-05", "", "- Move backups to S3 Glacier."].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const { beforeAgentReply } = createHarness({
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-core": {
|
||||
config: {
|
||||
dreaming: {
|
||||
enabled: true,
|
||||
phases: {
|
||||
light: {
|
||||
enabled: true,
|
||||
limit: 20,
|
||||
lookbackDays: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const readSpy = vi.spyOn(fs, "readFile");
|
||||
try {
|
||||
await beforeAgentReply(
|
||||
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },
|
||||
{ trigger: "heartbeat", workspaceDir },
|
||||
);
|
||||
await beforeAgentReply(
|
||||
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },
|
||||
{ trigger: "heartbeat", workspaceDir },
|
||||
);
|
||||
} finally {
|
||||
readSpy.mockRestore();
|
||||
}
|
||||
|
||||
const dailyReadCount = readSpy.mock.calls.filter(
|
||||
([target]) => String(target) === dailyPath,
|
||||
).length;
|
||||
expect(dailyReadCount).toBe(1);
|
||||
await expect(
|
||||
fs.access(path.join(workspaceDir, "memory", ".dreams", "daily-ingestion.json")),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("ingests recent daily memory files even before recall traffic exists", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", "2026-04-05.md"),
|
||||
["# 2026-04-05", "", "- Move backups to S3 Glacier.", "- Keep retention at 365 days."].join(
|
||||
"\n",
|
||||
),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const before = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.parse("2026-04-05T10:00:00.000Z"),
|
||||
});
|
||||
expect(before).toHaveLength(0);
|
||||
|
||||
const { beforeAgentReply } = createHarness({
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-core": {
|
||||
config: {
|
||||
dreaming: {
|
||||
enabled: true,
|
||||
phases: {
|
||||
light: {
|
||||
enabled: true,
|
||||
limit: 20,
|
||||
lookbackDays: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await beforeAgentReply(
|
||||
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },
|
||||
{ trigger: "heartbeat", workspaceDir },
|
||||
);
|
||||
|
||||
const after = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.parse("2026-04-05T10:05:00.000Z"),
|
||||
});
|
||||
expect(after.length).toBeGreaterThan(0);
|
||||
expect(after.some((candidate) => (candidate.dailyCount ?? 0) > 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("records light/rem signals that reinforce deep promotion ranking", async () => {
|
||||
const workspaceDir = await createTempWorkspace();
|
||||
const nowMs = Date.parse("2026-04-05T10:00:00.000Z");
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "glacier backup",
|
||||
nowMs,
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-03.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
score: 0.92,
|
||||
snippet: "Move backups to S3 Glacier.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "cold storage retention",
|
||||
nowMs,
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-03.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
score: 0.9,
|
||||
snippet: "Move backups to S3 Glacier.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const baseline = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs,
|
||||
});
|
||||
expect(baseline).toHaveLength(1);
|
||||
const baselineScore = baseline[0]!.score;
|
||||
|
||||
const { beforeAgentReply } = createHarness({
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-core": {
|
||||
config: {
|
||||
dreaming: {
|
||||
enabled: true,
|
||||
phases: {
|
||||
light: {
|
||||
enabled: true,
|
||||
limit: 10,
|
||||
lookbackDays: 7,
|
||||
},
|
||||
rem: {
|
||||
enabled: true,
|
||||
limit: 10,
|
||||
lookbackDays: 7,
|
||||
minPatternStrength: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await beforeAgentReply(
|
||||
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },
|
||||
{ trigger: "heartbeat", workspaceDir },
|
||||
);
|
||||
await beforeAgentReply(
|
||||
{ cleanedBody: "__openclaw_memory_core_rem_sleep__" },
|
||||
{ trigger: "heartbeat", workspaceDir },
|
||||
);
|
||||
|
||||
const reinforced = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs,
|
||||
});
|
||||
expect(reinforced).toHaveLength(1);
|
||||
expect(reinforced[0]!.score).toBeGreaterThan(baselineScore);
|
||||
|
||||
const phaseSignalPath = resolveShortTermPhaseSignalStorePath(workspaceDir);
|
||||
const phaseSignalStore = JSON.parse(await fs.readFile(phaseSignalPath, "utf-8")) as {
|
||||
entries: Record<string, { lightHits: number; remHits: number }>;
|
||||
};
|
||||
expect(phaseSignalStore.entries[reinforced[0]!.key]).toMatchObject({
|
||||
lightHits: 1,
|
||||
remHits: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,8 @@
|
||||
import type { Dirent } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig, OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core";
|
||||
import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
|
||||
import {
|
||||
resolveMemoryCorePluginConfig,
|
||||
resolveMemoryLightDreamingConfig,
|
||||
@@ -9,7 +13,13 @@ import {
|
||||
type MemoryDreamingPhaseName,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js";
|
||||
import { readShortTermRecallEntries, type ShortTermRecallEntry } from "./short-term-promotion.js";
|
||||
import { generateAndAppendDreamNarrative, type NarrativePhaseData } from "./dreaming-narrative.js";
|
||||
import {
|
||||
readShortTermRecallEntries,
|
||||
recordShortTermRecalls,
|
||||
recordDreamingPhaseSignals,
|
||||
type ShortTermRecallEntry,
|
||||
} from "./short-term-promotion.js";
|
||||
|
||||
type Logger = Pick<OpenClawPluginApi["logger"], "info" | "warn" | "error">;
|
||||
|
||||
@@ -68,6 +78,11 @@ const LIGHT_SLEEP_EVENT_TEXT = "__openclaw_memory_core_light_sleep__";
|
||||
const REM_SLEEP_CRON_NAME = "Memory REM Dreaming";
|
||||
const REM_SLEEP_CRON_TAG = "[managed-by=memory-core.dreaming.rem]";
|
||||
const REM_SLEEP_EVENT_TEXT = "__openclaw_memory_core_rem_sleep__";
|
||||
const DAILY_MEMORY_FILENAME_RE = /^(\d{4}-\d{2}-\d{2})\.md$/;
|
||||
const DAILY_INGESTION_STATE_RELATIVE_PATH = path.join("memory", ".dreams", "daily-ingestion.json");
|
||||
const DAILY_INGESTION_SCORE = 0.62;
|
||||
const DAILY_INGESTION_MAX_SNIPPET_CHARS = 280;
|
||||
const DAILY_INGESTION_MIN_SNIPPET_CHARS = 8;
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
@@ -304,6 +319,279 @@ function calculateLookbackCutoffMs(nowMs: number, lookbackDays: number): number
|
||||
return nowMs - Math.max(0, lookbackDays) * 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
function isDayWithinLookback(day: string, cutoffMs: number): boolean {
|
||||
const dayMs = Date.parse(`${day}T23:59:59.999Z`);
|
||||
return Number.isFinite(dayMs) && dayMs >= cutoffMs;
|
||||
}
|
||||
|
||||
function normalizeDailySnippet(line: string): string | null {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
if (trimmed.startsWith("#") || trimmed.startsWith("<!--")) {
|
||||
return null;
|
||||
}
|
||||
const withoutListMarker = trimmed
|
||||
.replace(/^\d+\.\s+/, "")
|
||||
.replace(/^[-*+]\s+/, "")
|
||||
.trim();
|
||||
if (withoutListMarker.length < DAILY_INGESTION_MIN_SNIPPET_CHARS) {
|
||||
return null;
|
||||
}
|
||||
return withoutListMarker.slice(0, DAILY_INGESTION_MAX_SNIPPET_CHARS).replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
function entryWithinLookback(entry: ShortTermRecallEntry, cutoffMs: number): boolean {
|
||||
const byDay = (entry.recallDays ?? []).some((day) => isDayWithinLookback(day, cutoffMs));
|
||||
if (byDay) {
|
||||
return true;
|
||||
}
|
||||
const lastRecalledAtMs = Date.parse(entry.lastRecalledAt);
|
||||
return Number.isFinite(lastRecalledAtMs) && lastRecalledAtMs >= cutoffMs;
|
||||
}
|
||||
|
||||
type DailyIngestionBatch = {
|
||||
day: string;
|
||||
results: MemorySearchResult[];
|
||||
};
|
||||
|
||||
type DailyIngestionFileState = {
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
};
|
||||
|
||||
type DailyIngestionState = {
|
||||
version: 1;
|
||||
files: Record<string, DailyIngestionFileState>;
|
||||
};
|
||||
|
||||
function resolveDailyIngestionStatePath(workspaceDir: string): string {
|
||||
return path.join(workspaceDir, DAILY_INGESTION_STATE_RELATIVE_PATH);
|
||||
}
|
||||
|
||||
function normalizeDailyIngestionState(raw: unknown): DailyIngestionState {
|
||||
const record = asRecord(raw);
|
||||
const filesRaw = asRecord(record?.files);
|
||||
if (!filesRaw) {
|
||||
return {
|
||||
version: 1,
|
||||
files: {},
|
||||
};
|
||||
}
|
||||
const files: Record<string, DailyIngestionFileState> = {};
|
||||
for (const [key, value] of Object.entries(filesRaw)) {
|
||||
const file = asRecord(value);
|
||||
if (!file || typeof key !== "string" || key.trim().length === 0) {
|
||||
continue;
|
||||
}
|
||||
const mtimeMs = Number(file.mtimeMs);
|
||||
const size = Number(file.size);
|
||||
if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) {
|
||||
continue;
|
||||
}
|
||||
files[key] = {
|
||||
mtimeMs: Math.floor(mtimeMs),
|
||||
size: Math.floor(size),
|
||||
};
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
files,
|
||||
};
|
||||
}
|
||||
|
||||
async function readDailyIngestionState(workspaceDir: string): Promise<DailyIngestionState> {
|
||||
const statePath = resolveDailyIngestionStatePath(workspaceDir);
|
||||
try {
|
||||
const raw = await fs.readFile(statePath, "utf-8");
|
||||
return normalizeDailyIngestionState(JSON.parse(raw) as unknown);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code === "ENOENT" || err instanceof SyntaxError) {
|
||||
return { version: 1, files: {} };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeDailyIngestionState(
|
||||
workspaceDir: string,
|
||||
state: DailyIngestionState,
|
||||
): Promise<void> {
|
||||
const statePath = resolveDailyIngestionStatePath(workspaceDir);
|
||||
await fs.mkdir(path.dirname(statePath), { recursive: true });
|
||||
const tmpPath = `${statePath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, "utf-8");
|
||||
await fs.rename(tmpPath, statePath);
|
||||
}
|
||||
|
||||
type DailyIngestionCollectionResult = {
|
||||
batches: DailyIngestionBatch[];
|
||||
nextState: DailyIngestionState;
|
||||
changed: boolean;
|
||||
};
|
||||
|
||||
async function collectDailyIngestionBatches(params: {
|
||||
workspaceDir: string;
|
||||
lookbackDays: number;
|
||||
limit: number;
|
||||
nowMs: number;
|
||||
state: DailyIngestionState;
|
||||
}): Promise<DailyIngestionCollectionResult> {
|
||||
const memoryDir = path.join(params.workspaceDir, "memory");
|
||||
const cutoffMs = calculateLookbackCutoffMs(params.nowMs, params.lookbackDays);
|
||||
const entries = await fs.readdir(memoryDir, { withFileTypes: true }).catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return [] as Dirent[];
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
const files = entries
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => {
|
||||
const match = entry.name.match(DAILY_MEMORY_FILENAME_RE);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const day = match[1]!;
|
||||
if (!isDayWithinLookback(day, cutoffMs)) {
|
||||
return null;
|
||||
}
|
||||
return { fileName: entry.name, day };
|
||||
})
|
||||
.filter((entry): entry is { fileName: string; day: string } => entry !== null)
|
||||
.toSorted((a, b) => b.day.localeCompare(a.day));
|
||||
|
||||
const batches: DailyIngestionBatch[] = [];
|
||||
const nextFiles: Record<string, DailyIngestionFileState> = {};
|
||||
let changed = false;
|
||||
const totalCap = Math.max(20, params.limit * 4);
|
||||
const perFileCap = Math.max(6, Math.ceil(totalCap / Math.max(1, Math.max(files.length, 1))));
|
||||
let total = 0;
|
||||
for (const file of files) {
|
||||
const relativePath = `memory/${file.fileName}`;
|
||||
const filePath = path.join(memoryDir, file.fileName);
|
||||
const stat = await fs.stat(filePath).catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (!stat) {
|
||||
continue;
|
||||
}
|
||||
const fingerprint: DailyIngestionFileState = {
|
||||
mtimeMs: Math.floor(Math.max(0, stat.mtimeMs)),
|
||||
size: Math.floor(Math.max(0, stat.size)),
|
||||
};
|
||||
nextFiles[relativePath] = fingerprint;
|
||||
const previous = params.state.files[relativePath];
|
||||
const unchanged =
|
||||
previous !== undefined &&
|
||||
previous.mtimeMs === fingerprint.mtimeMs &&
|
||||
previous.size === fingerprint.size;
|
||||
if (!unchanged) {
|
||||
changed = true;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
|
||||
const raw = await fs.readFile(filePath, "utf-8").catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (!raw) {
|
||||
continue;
|
||||
}
|
||||
const lines = raw.split(/\r?\n/);
|
||||
const results: MemorySearchResult[] = [];
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const line = lines[index];
|
||||
if (typeof line !== "string") {
|
||||
continue;
|
||||
}
|
||||
const snippet = normalizeDailySnippet(line);
|
||||
if (!snippet) {
|
||||
continue;
|
||||
}
|
||||
results.push({
|
||||
path: relativePath,
|
||||
startLine: index + 1,
|
||||
endLine: index + 1,
|
||||
score: DAILY_INGESTION_SCORE,
|
||||
snippet,
|
||||
source: "memory",
|
||||
});
|
||||
if (results.length >= perFileCap || total + results.length >= totalCap) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (results.length === 0) {
|
||||
continue;
|
||||
}
|
||||
batches.push({ day: file.day, results });
|
||||
total += results.length;
|
||||
if (total >= totalCap) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!changed) {
|
||||
const previousKeys = Object.keys(params.state.files);
|
||||
const nextKeys = Object.keys(nextFiles);
|
||||
if (
|
||||
previousKeys.length !== nextKeys.length ||
|
||||
previousKeys.some((key) => !Object.hasOwn(nextFiles, key))
|
||||
) {
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
batches,
|
||||
nextState: {
|
||||
version: 1,
|
||||
files: nextFiles,
|
||||
},
|
||||
changed,
|
||||
};
|
||||
}
|
||||
|
||||
async function ingestDailyMemorySignals(params: {
|
||||
workspaceDir: string;
|
||||
lookbackDays: number;
|
||||
limit: number;
|
||||
nowMs: number;
|
||||
timezone?: string;
|
||||
}): Promise<void> {
|
||||
const state = await readDailyIngestionState(params.workspaceDir);
|
||||
const collected = await collectDailyIngestionBatches({
|
||||
workspaceDir: params.workspaceDir,
|
||||
lookbackDays: params.lookbackDays,
|
||||
limit: params.limit,
|
||||
nowMs: params.nowMs,
|
||||
state,
|
||||
});
|
||||
for (const batch of collected.batches) {
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir: params.workspaceDir,
|
||||
query: `__dreaming_daily__:${batch.day}`,
|
||||
results: batch.results,
|
||||
signalType: "daily",
|
||||
dedupeByQueryPerDay: true,
|
||||
dayBucket: batch.day,
|
||||
nowMs: params.nowMs,
|
||||
timezone: params.timezone,
|
||||
});
|
||||
}
|
||||
if (collected.changed) {
|
||||
await writeDailyIngestionState(params.workspaceDir, collected.nextState);
|
||||
}
|
||||
}
|
||||
|
||||
function entryAverageScore(entry: ShortTermRecallEntry): number {
|
||||
return entry.recallCount > 0 ? Math.max(0, Math.min(1, entry.totalScore / entry.recallCount)) : 0;
|
||||
}
|
||||
@@ -380,16 +668,20 @@ function buildLightDreamingBody(entries: ShortTermRecallEntry[]): string[] {
|
||||
return lines;
|
||||
}
|
||||
|
||||
type RemTruthCandidate = {
|
||||
type RemTruthSelection = {
|
||||
key: string;
|
||||
snippet: string;
|
||||
confidence: number;
|
||||
evidence: string;
|
||||
};
|
||||
|
||||
type RemTruthCandidate = Omit<RemTruthSelection, "key">;
|
||||
|
||||
export type RemDreamingPreview = {
|
||||
sourceEntryCount: number;
|
||||
reflections: string[];
|
||||
candidateTruths: RemTruthCandidate[];
|
||||
candidateKeys: string[];
|
||||
bodyLines: string[];
|
||||
};
|
||||
|
||||
@@ -410,7 +702,7 @@ function calculateCandidateTruthConfidence(entry: ShortTermRecallEntry): number
|
||||
function selectRemCandidateTruths(
|
||||
entries: ShortTermRecallEntry[],
|
||||
limit: number,
|
||||
): RemTruthCandidate[] {
|
||||
): RemTruthSelection[] {
|
||||
if (limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
@@ -419,6 +711,7 @@ function selectRemCandidateTruths(
|
||||
0.88,
|
||||
)
|
||||
.map((entry) => ({
|
||||
key: entry.key,
|
||||
snippet: entry.snippet || "(no snippet captured)",
|
||||
confidence: calculateCandidateTruthConfidence(entry),
|
||||
evidence: `${entry.path}:${entry.startLine}-${entry.endLine}`,
|
||||
@@ -478,10 +771,16 @@ export function previewRemDreaming(params: {
|
||||
minPatternStrength: number;
|
||||
}): RemDreamingPreview {
|
||||
const reflections = buildRemReflections(params.entries, params.limit, params.minPatternStrength);
|
||||
const candidateTruths = selectRemCandidateTruths(
|
||||
const candidateSelections = selectRemCandidateTruths(
|
||||
params.entries,
|
||||
Math.max(1, Math.min(3, params.limit)),
|
||||
);
|
||||
const candidateTruths = candidateSelections.map((entry) => ({
|
||||
snippet: entry.snippet,
|
||||
confidence: entry.confidence,
|
||||
evidence: entry.evidence,
|
||||
}));
|
||||
const candidateKeys = [...new Set(candidateSelections.map((entry) => entry.key))];
|
||||
const bodyLines = [
|
||||
"### Reflections",
|
||||
...reflections,
|
||||
@@ -498,6 +797,7 @@ export function previewRemDreaming(params: {
|
||||
sourceEntryCount: params.entries.length,
|
||||
reflections,
|
||||
candidateTruths,
|
||||
candidateKeys,
|
||||
bodyLines,
|
||||
};
|
||||
}
|
||||
@@ -509,13 +809,21 @@ async function runLightDreaming(params: {
|
||||
storage: { mode: "inline" | "separate" | "both"; separateReports: boolean };
|
||||
};
|
||||
logger: Logger;
|
||||
subagent?: Parameters<typeof generateAndAppendDreamNarrative>[0]["subagent"];
|
||||
nowMs?: number;
|
||||
}): Promise<void> {
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const cutoffMs = calculateLookbackCutoffMs(nowMs, params.config.lookbackDays);
|
||||
await ingestDailyMemorySignals({
|
||||
workspaceDir: params.workspaceDir,
|
||||
lookbackDays: params.config.lookbackDays,
|
||||
limit: params.config.limit,
|
||||
nowMs,
|
||||
timezone: params.config.timezone,
|
||||
});
|
||||
const entries = dedupeEntries(
|
||||
(await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }))
|
||||
.filter((entry) => Date.parse(entry.lastRecalledAt) >= cutoffMs)
|
||||
.filter((entry) => entryWithinLookback(entry, cutoffMs))
|
||||
.toSorted((a, b) => {
|
||||
const byTime = Date.parse(b.lastRecalledAt) - Date.parse(a.lastRecalledAt);
|
||||
if (byTime !== 0) {
|
||||
@@ -526,7 +834,8 @@ async function runLightDreaming(params: {
|
||||
.slice(0, params.config.limit),
|
||||
params.config.dedupeSimilarity,
|
||||
);
|
||||
const bodyLines = buildLightDreamingBody(entries.slice(0, params.config.limit));
|
||||
const capped = entries.slice(0, params.config.limit);
|
||||
const bodyLines = buildLightDreamingBody(capped);
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir: params.workspaceDir,
|
||||
phase: "light",
|
||||
@@ -535,11 +844,34 @@ async function runLightDreaming(params: {
|
||||
timezone: params.config.timezone,
|
||||
storage: params.config.storage,
|
||||
});
|
||||
await recordDreamingPhaseSignals({
|
||||
workspaceDir: params.workspaceDir,
|
||||
phase: "light",
|
||||
keys: capped.map((entry) => entry.key),
|
||||
nowMs,
|
||||
});
|
||||
if (params.config.enabled && entries.length > 0 && params.config.storage.mode !== "separate") {
|
||||
params.logger.info(
|
||||
`memory-core: light dreaming staged ${Math.min(entries.length, params.config.limit)} candidate(s) [workspace=${params.workspaceDir}].`,
|
||||
);
|
||||
}
|
||||
// Generate dream diary narrative from the staged entries.
|
||||
if (params.subagent && capped.length > 0) {
|
||||
const themes = [...new Set(capped.flatMap((e) => e.conceptTags).filter(Boolean))];
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "light",
|
||||
snippets: capped.map((e) => e.snippet).filter(Boolean),
|
||||
...(themes.length > 0 ? { themes } : {}),
|
||||
};
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent: params.subagent,
|
||||
workspaceDir: params.workspaceDir,
|
||||
data,
|
||||
nowMs,
|
||||
timezone: params.config.timezone,
|
||||
logger: params.logger,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function runRemDreaming(params: {
|
||||
@@ -549,13 +881,21 @@ async function runRemDreaming(params: {
|
||||
storage: { mode: "inline" | "separate" | "both"; separateReports: boolean };
|
||||
};
|
||||
logger: Logger;
|
||||
subagent?: Parameters<typeof generateAndAppendDreamNarrative>[0]["subagent"];
|
||||
nowMs?: number;
|
||||
}): Promise<void> {
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const cutoffMs = calculateLookbackCutoffMs(nowMs, params.config.lookbackDays);
|
||||
await ingestDailyMemorySignals({
|
||||
workspaceDir: params.workspaceDir,
|
||||
lookbackDays: params.config.lookbackDays,
|
||||
limit: params.config.limit,
|
||||
nowMs,
|
||||
timezone: params.config.timezone,
|
||||
});
|
||||
const entries = (
|
||||
await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs })
|
||||
).filter((entry) => Date.parse(entry.lastRecalledAt) >= cutoffMs);
|
||||
).filter((entry) => entryWithinLookback(entry, cutoffMs));
|
||||
const preview = previewRemDreaming({
|
||||
entries,
|
||||
limit: params.config.limit,
|
||||
@@ -569,11 +909,80 @@ async function runRemDreaming(params: {
|
||||
timezone: params.config.timezone,
|
||||
storage: params.config.storage,
|
||||
});
|
||||
await recordDreamingPhaseSignals({
|
||||
workspaceDir: params.workspaceDir,
|
||||
phase: "rem",
|
||||
keys: preview.candidateKeys,
|
||||
nowMs,
|
||||
});
|
||||
if (params.config.enabled && entries.length > 0 && params.config.storage.mode !== "separate") {
|
||||
params.logger.info(
|
||||
`memory-core: REM dreaming wrote reflections from ${entries.length} recent memory trace(s) [workspace=${params.workspaceDir}].`,
|
||||
);
|
||||
}
|
||||
// Generate dream diary narrative from REM reflections.
|
||||
if (params.subagent && entries.length > 0) {
|
||||
const snippets = preview.candidateTruths.map((t) => t.snippet).filter(Boolean);
|
||||
const themes = preview.reflections.filter(
|
||||
(r) => !r.startsWith("- No strong") && !r.startsWith(" -"),
|
||||
);
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "rem",
|
||||
snippets:
|
||||
snippets.length > 0
|
||||
? snippets
|
||||
: entries
|
||||
.slice(0, 8)
|
||||
.map((e) => e.snippet)
|
||||
.filter(Boolean),
|
||||
...(themes.length > 0 ? { themes } : {}),
|
||||
};
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent: params.subagent,
|
||||
workspaceDir: params.workspaceDir,
|
||||
data,
|
||||
nowMs,
|
||||
timezone: params.config.timezone,
|
||||
logger: params.logger,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function runDreamingSweepPhases(params: {
|
||||
workspaceDir: string;
|
||||
pluginConfig?: Record<string, unknown>;
|
||||
cfg?: OpenClawConfig;
|
||||
logger: Logger;
|
||||
subagent?: Parameters<typeof generateAndAppendDreamNarrative>[0]["subagent"];
|
||||
nowMs?: number;
|
||||
}): Promise<void> {
|
||||
const light = resolveMemoryLightDreamingConfig({
|
||||
pluginConfig: params.pluginConfig,
|
||||
cfg: params.cfg,
|
||||
});
|
||||
if (light.enabled && light.limit > 0) {
|
||||
await runLightDreaming({
|
||||
workspaceDir: params.workspaceDir,
|
||||
config: light,
|
||||
logger: params.logger,
|
||||
subagent: params.subagent,
|
||||
nowMs: params.nowMs,
|
||||
});
|
||||
}
|
||||
|
||||
const rem = resolveMemoryRemDreamingConfig({
|
||||
pluginConfig: params.pluginConfig,
|
||||
cfg: params.cfg,
|
||||
});
|
||||
if (rem.enabled && rem.limit > 0) {
|
||||
await runRemDreaming({
|
||||
workspaceDir: params.workspaceDir,
|
||||
config: rem,
|
||||
logger: params.logger,
|
||||
subagent: params.subagent,
|
||||
nowMs: params.nowMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function runPhaseIfTriggered(params: {
|
||||
@@ -582,6 +991,7 @@ async function runPhaseIfTriggered(params: {
|
||||
workspaceDir?: string;
|
||||
cfg?: OpenClawConfig;
|
||||
logger: Logger;
|
||||
subagent?: Parameters<typeof generateAndAppendDreamNarrative>[0]["subagent"];
|
||||
phase: "light" | "rem";
|
||||
eventText: string;
|
||||
config:
|
||||
@@ -624,6 +1034,7 @@ async function runPhaseIfTriggered(params: {
|
||||
storage: { mode: "inline" | "separate" | "both"; separateReports: boolean };
|
||||
},
|
||||
logger: params.logger,
|
||||
subagent: params.subagent,
|
||||
});
|
||||
} else {
|
||||
await runRemDreaming({
|
||||
@@ -633,6 +1044,7 @@ async function runPhaseIfTriggered(params: {
|
||||
storage: { mode: "inline" | "separate" | "both"; separateReports: boolean };
|
||||
},
|
||||
logger: params.logger,
|
||||
subagent: params.subagent,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -713,6 +1125,7 @@ export function registerMemoryDreamingPhases(api: OpenClawPluginApi): void {
|
||||
workspaceDir: ctx.workspaceDir,
|
||||
cfg: api.config,
|
||||
logger: api.logger,
|
||||
subagent: light.enabled ? api.runtime?.subagent : undefined,
|
||||
phase: "light",
|
||||
eventText: LIGHT_SLEEP_EVENT_TEXT,
|
||||
config: light,
|
||||
@@ -727,6 +1140,7 @@ export function registerMemoryDreamingPhases(api: OpenClawPluginApi): void {
|
||||
workspaceDir: ctx.workspaceDir,
|
||||
cfg: api.config,
|
||||
logger: api.logger,
|
||||
subagent: rem.enabled ? api.runtime?.subagent : undefined,
|
||||
phase: "rem",
|
||||
eventText: REM_SLEEP_EVENT_TEXT,
|
||||
config: rem,
|
||||
|
||||
@@ -141,12 +141,12 @@ describe("short-term dreaming config", () => {
|
||||
const resolved = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig: {
|
||||
dreaming: {
|
||||
mode: "core",
|
||||
enabled: true,
|
||||
timezone: "UTC",
|
||||
verboseLogging: true,
|
||||
frequency: "5 1 * * *",
|
||||
phases: {
|
||||
deep: {
|
||||
cron: "15 2 * * *",
|
||||
limit: 7,
|
||||
minScore: 0.4,
|
||||
minRecallCount: 2,
|
||||
@@ -160,7 +160,7 @@ describe("short-term dreaming config", () => {
|
||||
});
|
||||
expect(resolved).toEqual({
|
||||
enabled: true,
|
||||
cron: "15 2 * * *",
|
||||
cron: "5 1 * * *",
|
||||
timezone: "UTC",
|
||||
limit: 7,
|
||||
minScore: 0.4,
|
||||
@@ -176,14 +176,14 @@ describe("short-term dreaming config", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts cron alias and numeric string thresholds", () => {
|
||||
it("accepts top-level frequency and numeric string thresholds", () => {
|
||||
const resolved = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig: {
|
||||
dreaming: {
|
||||
mode: "core",
|
||||
enabled: true,
|
||||
frequency: "5 1 * * *",
|
||||
phases: {
|
||||
deep: {
|
||||
cron: "5 1 * * *",
|
||||
limit: "4",
|
||||
minScore: "0.6",
|
||||
minRecallCount: "2",
|
||||
@@ -216,7 +216,7 @@ describe("short-term dreaming config", () => {
|
||||
const resolved = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig: {
|
||||
dreaming: {
|
||||
mode: "core",
|
||||
enabled: true,
|
||||
phases: {
|
||||
deep: {
|
||||
limit: " ",
|
||||
@@ -251,7 +251,7 @@ describe("short-term dreaming config", () => {
|
||||
const resolved = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig: {
|
||||
dreaming: {
|
||||
mode: "core",
|
||||
enabled: true,
|
||||
phases: {
|
||||
deep: {
|
||||
limit: 0,
|
||||
@@ -267,7 +267,6 @@ describe("short-term dreaming config", () => {
|
||||
const enabled = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig: {
|
||||
dreaming: {
|
||||
mode: "core",
|
||||
verboseLogging: true,
|
||||
},
|
||||
},
|
||||
@@ -275,7 +274,6 @@ describe("short-term dreaming config", () => {
|
||||
const disabled = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig: {
|
||||
dreaming: {
|
||||
mode: "core",
|
||||
verboseLogging: "false",
|
||||
},
|
||||
},
|
||||
@@ -289,7 +287,7 @@ describe("short-term dreaming config", () => {
|
||||
const resolved = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig: {
|
||||
dreaming: {
|
||||
mode: "core",
|
||||
enabled: true,
|
||||
phases: {
|
||||
deep: {
|
||||
minScore: -0.2,
|
||||
@@ -312,11 +310,15 @@ describe("short-term dreaming config", () => {
|
||||
expect(resolved.maxAgeDays).toBe(30);
|
||||
});
|
||||
|
||||
it("keeps deep sleep disabled when mode is off", () => {
|
||||
it("keeps deep sleep disabled when the phase is off", () => {
|
||||
const resolved = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig: {
|
||||
dreaming: {
|
||||
mode: "off",
|
||||
phases: {
|
||||
deep: {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -490,6 +492,63 @@ describe("short-term dreaming cron reconciliation", () => {
|
||||
expect(harness.jobs.map((entry) => entry.id)).toEqual(["job-other"]);
|
||||
});
|
||||
|
||||
it("prunes legacy light/rem dreaming cron jobs during reconciliation", async () => {
|
||||
const deepManagedJob: CronJobLike = {
|
||||
id: "job-deep",
|
||||
name: constants.MANAGED_DREAMING_CRON_NAME,
|
||||
description: `${constants.MANAGED_DREAMING_CRON_TAG} test`,
|
||||
enabled: true,
|
||||
schedule: { kind: "cron", expr: "0 3 * * *" },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: constants.DREAMING_SYSTEM_EVENT_TEXT },
|
||||
createdAtMs: 10,
|
||||
};
|
||||
const legacyLightJob: CronJobLike = {
|
||||
id: "job-light",
|
||||
name: "Memory Light Dreaming",
|
||||
description: "[managed-by=memory-core.dreaming.light] legacy",
|
||||
enabled: true,
|
||||
schedule: { kind: "cron", expr: "0 */6 * * *" },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "__openclaw_memory_core_light_sleep__" },
|
||||
createdAtMs: 8,
|
||||
};
|
||||
const legacyRemJob: CronJobLike = {
|
||||
id: "job-rem",
|
||||
name: "Memory REM Dreaming",
|
||||
description: "[managed-by=memory-core.dreaming.rem] legacy",
|
||||
enabled: true,
|
||||
schedule: { kind: "cron", expr: "0 5 * * 0" },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "__openclaw_memory_core_rem_sleep__" },
|
||||
createdAtMs: 9,
|
||||
};
|
||||
const harness = createCronHarness([legacyLightJob, legacyRemJob, deepManagedJob]);
|
||||
const logger = createLogger();
|
||||
|
||||
const result = await reconcileShortTermDreamingCronJob({
|
||||
cron: harness.cron,
|
||||
config: {
|
||||
enabled: true,
|
||||
cron: constants.DEFAULT_DREAMING_CRON_EXPR,
|
||||
limit: constants.DEFAULT_DREAMING_LIMIT,
|
||||
minScore: constants.DEFAULT_DREAMING_MIN_SCORE,
|
||||
minRecallCount: constants.DEFAULT_DREAMING_MIN_RECALL_COUNT,
|
||||
minUniqueQueries: constants.DEFAULT_DREAMING_MIN_UNIQUE_QUERIES,
|
||||
recencyHalfLifeDays: constants.DEFAULT_DREAMING_RECENCY_HALF_LIFE_DAYS,
|
||||
verboseLogging: false,
|
||||
},
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(result.status).toBe("noop");
|
||||
expect(result.removed).toBe(2);
|
||||
expect(harness.removeCalls).toEqual(["job-light", "job-rem"]);
|
||||
});
|
||||
|
||||
it("does not overcount removed jobs when cron remove result is unknown", async () => {
|
||||
const managedJob: CronJobLike = {
|
||||
id: "job-managed",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { OpenClawConfig, OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core";
|
||||
import {
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_CRON_EXPR as DEFAULT_MEMORY_DREAMING_CRON_EXPR,
|
||||
DEFAULT_MEMORY_DREAMING_FREQUENCY as DEFAULT_MEMORY_DREAMING_CRON_EXPR,
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_LIMIT as DEFAULT_MEMORY_DREAMING_LIMIT,
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_MIN_RECALL_COUNT as DEFAULT_MEMORY_DREAMING_MIN_RECALL_COUNT,
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_MIN_SCORE as DEFAULT_MEMORY_DREAMING_MIN_SCORE,
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
resolveMemoryDreamingWorkspaces,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { writeDeepDreamingReport } from "./dreaming-markdown.js";
|
||||
import { generateAndAppendDreamNarrative, type NarrativePhaseData } from "./dreaming-narrative.js";
|
||||
import { runDreamingSweepPhases } from "./dreaming-phases.js";
|
||||
import {
|
||||
applyShortTermPromotions,
|
||||
repairShortTermPromotionArtifacts,
|
||||
@@ -20,6 +22,12 @@ import {
|
||||
const MANAGED_DREAMING_CRON_NAME = "Memory Dreaming Promotion";
|
||||
const MANAGED_DREAMING_CRON_TAG = "[managed-by=memory-core.short-term-promotion]";
|
||||
const DREAMING_SYSTEM_EVENT_TEXT = "__openclaw_memory_core_short_term_promotion_dream__";
|
||||
const LEGACY_LIGHT_SLEEP_CRON_NAME = "Memory Light Dreaming";
|
||||
const LEGACY_LIGHT_SLEEP_CRON_TAG = "[managed-by=memory-core.dreaming.light]";
|
||||
const LEGACY_LIGHT_SLEEP_EVENT_TEXT = "__openclaw_memory_core_light_sleep__";
|
||||
const LEGACY_REM_SLEEP_CRON_NAME = "Memory REM Dreaming";
|
||||
const LEGACY_REM_SLEEP_CRON_TAG = "[managed-by=memory-core.dreaming.rem]";
|
||||
const LEGACY_REM_SLEEP_EVENT_TEXT = "__openclaw_memory_core_rem_sleep__";
|
||||
|
||||
type Logger = Pick<OpenClawPluginApi["logger"], "info" | "warn" | "error">;
|
||||
|
||||
@@ -171,6 +179,22 @@ function isManagedDreamingJob(job: ManagedCronJobLike): boolean {
|
||||
return name === MANAGED_DREAMING_CRON_NAME && payloadText === DREAMING_SYSTEM_EVENT_TEXT;
|
||||
}
|
||||
|
||||
function isLegacyPhaseDreamingJob(job: ManagedCronJobLike): boolean {
|
||||
const description = normalizeTrimmedString(job.description);
|
||||
if (
|
||||
description?.includes(LEGACY_LIGHT_SLEEP_CRON_TAG) ||
|
||||
description?.includes(LEGACY_REM_SLEEP_CRON_TAG)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const name = normalizeTrimmedString(job.name);
|
||||
const payloadText = normalizeTrimmedString(job.payload?.text);
|
||||
if (name === LEGACY_LIGHT_SLEEP_CRON_NAME && payloadText === LEGACY_LIGHT_SLEEP_EVENT_TEXT) {
|
||||
return true;
|
||||
}
|
||||
return name === LEGACY_REM_SLEEP_CRON_NAME && payloadText === LEGACY_REM_SLEEP_EVENT_TEXT;
|
||||
}
|
||||
|
||||
function compareOptionalStrings(a: string | undefined, b: string | undefined): boolean {
|
||||
return a === b;
|
||||
}
|
||||
@@ -295,9 +319,27 @@ export async function reconcileShortTermDreamingCronJob(params: {
|
||||
|
||||
const allJobs = await cron.list({ includeDisabled: true });
|
||||
const managed = allJobs.filter(isManagedDreamingJob);
|
||||
const legacyPhaseJobs = allJobs.filter(isLegacyPhaseDreamingJob);
|
||||
|
||||
let removedLegacy = 0;
|
||||
for (const job of legacyPhaseJobs) {
|
||||
try {
|
||||
const result = await cron.remove(job.id);
|
||||
if (result.removed === true) {
|
||||
removedLegacy += 1;
|
||||
}
|
||||
} catch (err) {
|
||||
params.logger.warn(
|
||||
`memory-core: failed to remove legacy managed dreaming cron job ${job.id}: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (removedLegacy > 0) {
|
||||
params.logger.info(`memory-core: removed ${removedLegacy} legacy phase dreaming cron job(s).`);
|
||||
}
|
||||
|
||||
if (!params.config.enabled) {
|
||||
let removed = 0;
|
||||
let removed = removedLegacy;
|
||||
for (const job of managed) {
|
||||
try {
|
||||
const result = await cron.remove(job.id);
|
||||
@@ -320,11 +362,11 @@ export async function reconcileShortTermDreamingCronJob(params: {
|
||||
if (managed.length === 0) {
|
||||
await cron.add(desired);
|
||||
params.logger.info("memory-core: created managed dreaming cron job.");
|
||||
return { status: "added", removed: 0 };
|
||||
return { status: "added", removed: removedLegacy };
|
||||
}
|
||||
|
||||
const [primary, ...duplicates] = sortManagedJobs(managed);
|
||||
let removed = 0;
|
||||
let removed = removedLegacy;
|
||||
for (const duplicate of duplicates) {
|
||||
try {
|
||||
const result = await cron.remove(duplicate.id);
|
||||
@@ -358,6 +400,7 @@ export async function runShortTermDreamingPromotionIfTriggered(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
config: ShortTermPromotionDreamingConfig;
|
||||
logger: Logger;
|
||||
subagent?: Parameters<typeof generateAndAppendDreamNarrative>[0]["subagent"];
|
||||
}): Promise<{ handled: true; reason: string } | undefined> {
|
||||
if (params.trigger !== "heartbeat") {
|
||||
return undefined;
|
||||
@@ -406,8 +449,19 @@ export async function runShortTermDreamingPromotionIfTriggered(params: {
|
||||
let totalCandidates = 0;
|
||||
let totalApplied = 0;
|
||||
let failedWorkspaces = 0;
|
||||
const pluginConfig = params.cfg ? resolveMemoryCorePluginConfig(params.cfg) : undefined;
|
||||
for (const workspaceDir of workspaces) {
|
||||
try {
|
||||
const sweepNowMs = Date.now();
|
||||
await runDreamingSweepPhases({
|
||||
workspaceDir,
|
||||
pluginConfig,
|
||||
cfg: params.cfg,
|
||||
logger: params.logger,
|
||||
subagent: params.subagent,
|
||||
nowMs: sweepNowMs,
|
||||
});
|
||||
|
||||
const reportLines: string[] = [];
|
||||
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
|
||||
if (repair.changed) {
|
||||
@@ -424,6 +478,7 @@ export async function runShortTermDreamingPromotionIfTriggered(params: {
|
||||
minUniqueQueries: params.config.minUniqueQueries,
|
||||
recencyHalfLifeDays,
|
||||
maxAgeDays: params.config.maxAgeDays,
|
||||
nowMs: sweepNowMs,
|
||||
});
|
||||
totalCandidates += candidates.length;
|
||||
reportLines.push(`- Ranked ${candidates.length} candidate(s) for durable promotion.`);
|
||||
@@ -450,6 +505,7 @@ export async function runShortTermDreamingPromotionIfTriggered(params: {
|
||||
minUniqueQueries: params.config.minUniqueQueries,
|
||||
maxAgeDays: params.config.maxAgeDays,
|
||||
timezone: params.config.timezone,
|
||||
nowMs: sweepNowMs,
|
||||
});
|
||||
totalApplied += applied.applied;
|
||||
reportLines.push(`- Promoted ${applied.applied} candidate(s) into MEMORY.md.`);
|
||||
@@ -470,9 +526,26 @@ export async function runShortTermDreamingPromotionIfTriggered(params: {
|
||||
await writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: reportLines,
|
||||
nowMs: sweepNowMs,
|
||||
timezone: params.config.timezone,
|
||||
storage: params.config.storage ?? { mode: "inline", separateReports: false },
|
||||
});
|
||||
// Generate dream diary narrative from promoted memories.
|
||||
if (params.subagent && (candidates.length > 0 || applied.applied > 0)) {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "deep",
|
||||
snippets: candidates.map((c) => c.snippet).filter(Boolean),
|
||||
promotions: applied.appliedCandidates.map((c) => c.snippet).filter(Boolean),
|
||||
};
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent: params.subagent,
|
||||
workspaceDir,
|
||||
data,
|
||||
nowMs: sweepNowMs,
|
||||
timezone: params.config.timezone,
|
||||
logger: params.logger,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
failedWorkspaces += 1;
|
||||
params.logger.error(
|
||||
@@ -529,6 +602,7 @@ export function registerShortTermPromotionDreaming(api: OpenClawPluginApi): void
|
||||
cfg: api.config,
|
||||
config,
|
||||
logger: api.logger,
|
||||
subagent: config.enabled ? api.runtime?.subagent : undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
api.logger.error(`memory-core: dreaming trigger failed: ${formatErrorMessage(err)}`);
|
||||
|
||||
@@ -7,9 +7,11 @@ import {
|
||||
auditShortTermPromotionArtifacts,
|
||||
isShortTermMemoryPath,
|
||||
rankShortTermPromotionCandidates,
|
||||
recordDreamingPhaseSignals,
|
||||
recordShortTermRecalls,
|
||||
repairShortTermPromotionArtifacts,
|
||||
resolveShortTermRecallLockPath,
|
||||
resolveShortTermPhaseSignalStorePath,
|
||||
resolveShortTermRecallStorePath,
|
||||
__testing,
|
||||
} from "./short-term-promotion.js";
|
||||
@@ -249,6 +251,178 @@ describe("short-term promotion", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("boosts deep ranking when light/rem phase signals reinforce a candidate", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const nowMs = Date.parse("2026-04-05T10:00:00.000Z");
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "router setup",
|
||||
nowMs,
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-01.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.75,
|
||||
snippet: "Router VLAN baseline noted.",
|
||||
source: "memory",
|
||||
},
|
||||
{
|
||||
path: "memory/2026-04-02.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.75,
|
||||
snippet: "Backup policy for router snapshots.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "router backup",
|
||||
nowMs,
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-01.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.75,
|
||||
snippet: "Router VLAN baseline noted.",
|
||||
source: "memory",
|
||||
},
|
||||
{
|
||||
path: "memory/2026-04-02.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.75,
|
||||
snippet: "Backup policy for router snapshots.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const baseline = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs,
|
||||
});
|
||||
expect(baseline).toHaveLength(2);
|
||||
expect(baseline[0]?.path).toBe("memory/2026-04-01.md");
|
||||
|
||||
const boostedKey = baseline.find((entry) => entry.path === "memory/2026-04-02.md")?.key;
|
||||
expect(boostedKey).toBeTruthy();
|
||||
await recordDreamingPhaseSignals({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
keys: [boostedKey!],
|
||||
nowMs,
|
||||
});
|
||||
await recordDreamingPhaseSignals({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
keys: [boostedKey!],
|
||||
nowMs,
|
||||
});
|
||||
|
||||
const ranked = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs,
|
||||
});
|
||||
expect(ranked[0]?.path).toBe("memory/2026-04-02.md");
|
||||
expect(ranked[0]!.score).toBeGreaterThan(ranked[1]!.score);
|
||||
|
||||
const phaseStorePath = resolveShortTermPhaseSignalStorePath(workspaceDir);
|
||||
const phaseStore = JSON.parse(await fs.readFile(phaseStorePath, "utf-8")) as {
|
||||
entries: Record<string, { lightHits: number; remHits: number }>;
|
||||
};
|
||||
expect(phaseStore.entries[boostedKey!]).toMatchObject({
|
||||
lightHits: 1,
|
||||
remHits: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("weights fresh phase signals more than stale ones", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "glacier cadence",
|
||||
nowMs: Date.parse("2026-04-01T10:00:00.000Z"),
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-01.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.9,
|
||||
snippet: "Move backups to S3 Glacier.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "backup lifecycle",
|
||||
nowMs: Date.parse("2026-04-01T12:00:00.000Z"),
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-01.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.9,
|
||||
snippet: "Move backups to S3 Glacier.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const rankedBaseline = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.parse("2026-04-05T10:00:00.000Z"),
|
||||
});
|
||||
const key = rankedBaseline[0]?.key;
|
||||
expect(key).toBeTruthy();
|
||||
|
||||
await recordDreamingPhaseSignals({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
keys: [key!],
|
||||
nowMs: Date.parse("2026-02-01T10:00:00.000Z"),
|
||||
});
|
||||
const staleSignalRank = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.parse("2026-04-05T10:00:00.000Z"),
|
||||
});
|
||||
await recordDreamingPhaseSignals({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
keys: [key!],
|
||||
nowMs: Date.parse("2026-04-05T10:00:00.000Z"),
|
||||
});
|
||||
const freshSignalRank = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.parse("2026-04-05T10:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(staleSignalRank).toHaveLength(1);
|
||||
expect(freshSignalRank).toHaveLength(1);
|
||||
expect(freshSignalRank[0]!.score).toBeGreaterThan(staleSignalRank[0]!.score);
|
||||
});
|
||||
});
|
||||
|
||||
it("reconciles existing promotion markers instead of appending duplicates", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
await writeDailyMemoryNote(workspaceDir, "2026-04-01", [
|
||||
|
||||
@@ -21,10 +21,14 @@ const PROMOTION_MARKER_PREFIX = "openclaw-memory-promotion:";
|
||||
const MAX_QUERY_HASHES = 32;
|
||||
const MAX_RECALL_DAYS = 16;
|
||||
const SHORT_TERM_STORE_RELATIVE_PATH = path.join("memory", ".dreams", "short-term-recall.json");
|
||||
const SHORT_TERM_PHASE_SIGNAL_RELATIVE_PATH = path.join("memory", ".dreams", "phase-signals.json");
|
||||
const SHORT_TERM_LOCK_RELATIVE_PATH = path.join("memory", ".dreams", "short-term-promotion.lock");
|
||||
const SHORT_TERM_LOCK_WAIT_TIMEOUT_MS = 10_000;
|
||||
const SHORT_TERM_LOCK_STALE_MS = 60_000;
|
||||
const SHORT_TERM_LOCK_RETRY_DELAY_MS = 40;
|
||||
const PHASE_SIGNAL_LIGHT_BOOST_MAX = 0.05;
|
||||
const PHASE_SIGNAL_REM_BOOST_MAX = 0.08;
|
||||
const PHASE_SIGNAL_HALF_LIFE_DAYS = 14;
|
||||
|
||||
export type PromotionWeights = {
|
||||
frequency: number;
|
||||
@@ -52,6 +56,7 @@ export type ShortTermRecallEntry = {
|
||||
source: "memory";
|
||||
snippet: string;
|
||||
recallCount: number;
|
||||
dailyCount: number;
|
||||
totalScore: number;
|
||||
maxScore: number;
|
||||
firstRecalledAt: string;
|
||||
@@ -68,6 +73,20 @@ type ShortTermRecallStore = {
|
||||
entries: Record<string, ShortTermRecallEntry>;
|
||||
};
|
||||
|
||||
type ShortTermPhaseSignalEntry = {
|
||||
key: string;
|
||||
lightHits: number;
|
||||
remHits: number;
|
||||
lastLightAt?: string;
|
||||
lastRemAt?: string;
|
||||
};
|
||||
|
||||
type ShortTermPhaseSignalStore = {
|
||||
version: 1;
|
||||
updatedAt: string;
|
||||
entries: Record<string, ShortTermPhaseSignalEntry>;
|
||||
};
|
||||
|
||||
export type PromotionComponents = {
|
||||
frequency: number;
|
||||
relevance: number;
|
||||
@@ -85,6 +104,8 @@ export type PromotionCandidate = {
|
||||
source: "memory";
|
||||
snippet: string;
|
||||
recallCount: number;
|
||||
dailyCount?: number;
|
||||
signalCount?: number;
|
||||
avgScore: number;
|
||||
maxScore: number;
|
||||
uniqueQueries: number;
|
||||
@@ -339,6 +360,7 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
|
||||
}
|
||||
|
||||
const recallCount = Math.max(0, Math.floor(Number(entry.recallCount) || 0));
|
||||
const dailyCount = Math.max(0, Math.floor(Number(entry.dailyCount) || 0));
|
||||
const totalScore = Math.max(0, Number(entry.totalScore) || 0);
|
||||
const maxScore = clampScore(Number(entry.maxScore) || 0);
|
||||
const firstRecalledAt =
|
||||
@@ -371,6 +393,7 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
|
||||
source,
|
||||
snippet,
|
||||
recallCount,
|
||||
dailyCount,
|
||||
totalScore,
|
||||
maxScore,
|
||||
firstRecalledAt,
|
||||
@@ -446,10 +469,50 @@ function calculateRecencyComponent(ageDays: number, halfLifeDays: number): numbe
|
||||
return Math.exp(-lambda * ageDays);
|
||||
}
|
||||
|
||||
function calculatePhaseSignalAgeDays(lastSeenAt: string | undefined, nowMs: number): number | null {
|
||||
if (!lastSeenAt) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Date.parse(lastSeenAt);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
return null;
|
||||
}
|
||||
return Math.max(0, (nowMs - parsed) / DAY_MS);
|
||||
}
|
||||
|
||||
function calculatePhaseSignalBoost(
|
||||
entry: ShortTermPhaseSignalEntry | undefined,
|
||||
nowMs: number,
|
||||
): number {
|
||||
if (!entry) {
|
||||
return 0;
|
||||
}
|
||||
const lightStrength = clampScore(Math.log1p(Math.max(0, entry.lightHits)) / Math.log1p(6));
|
||||
const remStrength = clampScore(Math.log1p(Math.max(0, entry.remHits)) / Math.log1p(6));
|
||||
const lightAgeDays = calculatePhaseSignalAgeDays(entry.lastLightAt, nowMs);
|
||||
const remAgeDays = calculatePhaseSignalAgeDays(entry.lastRemAt, nowMs);
|
||||
const lightRecency =
|
||||
lightAgeDays === null
|
||||
? 0
|
||||
: clampScore(calculateRecencyComponent(lightAgeDays, PHASE_SIGNAL_HALF_LIFE_DAYS));
|
||||
const remRecency =
|
||||
remAgeDays === null
|
||||
? 0
|
||||
: clampScore(calculateRecencyComponent(remAgeDays, PHASE_SIGNAL_HALF_LIFE_DAYS));
|
||||
return clampScore(
|
||||
PHASE_SIGNAL_LIGHT_BOOST_MAX * lightStrength * lightRecency +
|
||||
PHASE_SIGNAL_REM_BOOST_MAX * remStrength * remRecency,
|
||||
);
|
||||
}
|
||||
|
||||
function resolveStorePath(workspaceDir: string): string {
|
||||
return path.join(workspaceDir, SHORT_TERM_STORE_RELATIVE_PATH);
|
||||
}
|
||||
|
||||
function resolvePhaseSignalPath(workspaceDir: string): string {
|
||||
return path.join(workspaceDir, SHORT_TERM_PHASE_SIGNAL_RELATIVE_PATH);
|
||||
}
|
||||
|
||||
function resolveLockPath(workspaceDir: string): string {
|
||||
return path.join(workspaceDir, SHORT_TERM_LOCK_RELATIVE_PATH);
|
||||
}
|
||||
@@ -552,6 +615,89 @@ async function readStore(workspaceDir: string, nowIso: string): Promise<ShortTer
|
||||
}
|
||||
}
|
||||
|
||||
function emptyPhaseSignalStore(nowIso: string): ShortTermPhaseSignalStore {
|
||||
return {
|
||||
version: 1,
|
||||
updatedAt: nowIso,
|
||||
entries: {},
|
||||
};
|
||||
}
|
||||
|
||||
function normalizePhaseSignalStore(raw: unknown, nowIso: string): ShortTermPhaseSignalStore {
|
||||
const record = asRecord(raw);
|
||||
if (!record) {
|
||||
return emptyPhaseSignalStore(nowIso);
|
||||
}
|
||||
const entriesRaw = asRecord(record?.entries);
|
||||
if (!entriesRaw) {
|
||||
return emptyPhaseSignalStore(nowIso);
|
||||
}
|
||||
const entries: Record<string, ShortTermPhaseSignalEntry> = {};
|
||||
for (const [mapKey, value] of Object.entries(entriesRaw)) {
|
||||
const entry = asRecord(value);
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
const key = typeof entry.key === "string" && entry.key.trim().length > 0 ? entry.key : mapKey;
|
||||
const lightHits = toFiniteNonNegativeInt(entry.lightHits, 0);
|
||||
const remHits = toFiniteNonNegativeInt(entry.remHits, 0);
|
||||
if (lightHits === 0 && remHits === 0) {
|
||||
continue;
|
||||
}
|
||||
const lastLightAt =
|
||||
typeof entry.lastLightAt === "string" && entry.lastLightAt.trim().length > 0
|
||||
? entry.lastLightAt
|
||||
: undefined;
|
||||
const lastRemAt =
|
||||
typeof entry.lastRemAt === "string" && entry.lastRemAt.trim().length > 0
|
||||
? entry.lastRemAt
|
||||
: undefined;
|
||||
entries[key] = {
|
||||
key,
|
||||
lightHits,
|
||||
remHits,
|
||||
...(lastLightAt ? { lastLightAt } : {}),
|
||||
...(lastRemAt ? { lastRemAt } : {}),
|
||||
};
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
updatedAt:
|
||||
typeof record.updatedAt === "string" && record.updatedAt.trim().length > 0
|
||||
? record.updatedAt
|
||||
: nowIso,
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
async function readPhaseSignalStore(
|
||||
workspaceDir: string,
|
||||
nowIso: string,
|
||||
): Promise<ShortTermPhaseSignalStore> {
|
||||
const phaseSignalPath = resolvePhaseSignalPath(workspaceDir);
|
||||
try {
|
||||
const raw = await fs.readFile(phaseSignalPath, "utf-8");
|
||||
return normalizePhaseSignalStore(JSON.parse(raw) as unknown, nowIso);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code === "ENOENT" || err instanceof SyntaxError) {
|
||||
return emptyPhaseSignalStore(nowIso);
|
||||
}
|
||||
return emptyPhaseSignalStore(nowIso);
|
||||
}
|
||||
}
|
||||
|
||||
async function writePhaseSignalStore(
|
||||
workspaceDir: string,
|
||||
store: ShortTermPhaseSignalStore,
|
||||
): Promise<void> {
|
||||
const phaseSignalPath = resolvePhaseSignalPath(workspaceDir);
|
||||
await fs.mkdir(path.dirname(phaseSignalPath), { recursive: true });
|
||||
const tmpPath = `${phaseSignalPath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
||||
await fs.writeFile(tmpPath, `${JSON.stringify(store, null, 2)}\n`, "utf-8");
|
||||
await fs.rename(tmpPath, phaseSignalPath);
|
||||
}
|
||||
|
||||
async function writeStore(workspaceDir: string, store: ShortTermRecallStore): Promise<void> {
|
||||
const storePath = resolveStorePath(workspaceDir);
|
||||
await fs.mkdir(path.dirname(storePath), { recursive: true });
|
||||
@@ -572,6 +718,9 @@ export async function recordShortTermRecalls(params: {
|
||||
workspaceDir?: string;
|
||||
query: string;
|
||||
results: MemorySearchResult[];
|
||||
signalType?: "recall" | "daily";
|
||||
dedupeByQueryPerDay?: boolean;
|
||||
dayBucket?: string;
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
}): Promise<void> {
|
||||
@@ -592,7 +741,10 @@ export async function recordShortTermRecalls(params: {
|
||||
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const nowIso = new Date(nowMs).toISOString();
|
||||
const signalType = params.signalType ?? "recall";
|
||||
const queryHash = hashQuery(query);
|
||||
const todayBucket =
|
||||
normalizeIsoDay(params.dayBucket ?? "") ?? formatMemoryDreamingDay(nowMs, params.timezone);
|
||||
await withShortTermLock(workspaceDir, async () => {
|
||||
const store = await readStore(workspaceDir, nowIso);
|
||||
|
||||
@@ -602,15 +754,24 @@ export async function recordShortTermRecalls(params: {
|
||||
const existing = store.entries[key];
|
||||
const snippet = normalizeSnippet(result.snippet);
|
||||
const score = clampScore(result.score);
|
||||
const recallCount = Math.max(1, Math.floor(existing?.recallCount ?? 0) + 1);
|
||||
const totalScore = Math.max(0, (existing?.totalScore ?? 0) + score);
|
||||
const maxScore = Math.max(existing?.maxScore ?? 0, score);
|
||||
const recallDaysBase = existing?.recallDays ?? [];
|
||||
const queryHashesBase = existing?.queryHashes ?? [];
|
||||
const dedupeSignal =
|
||||
Boolean(params.dedupeByQueryPerDay) &&
|
||||
queryHashesBase.includes(queryHash) &&
|
||||
recallDaysBase.includes(todayBucket);
|
||||
const recallCount =
|
||||
signalType === "recall"
|
||||
? Math.max(0, Math.floor(existing?.recallCount ?? 0) + (dedupeSignal ? 0 : 1))
|
||||
: Math.max(0, Math.floor(existing?.recallCount ?? 0));
|
||||
const dailyCount =
|
||||
signalType === "daily"
|
||||
? Math.max(0, Math.floor(existing?.dailyCount ?? 0) + (dedupeSignal ? 0 : 1))
|
||||
: Math.max(0, Math.floor(existing?.dailyCount ?? 0));
|
||||
const totalScore = Math.max(0, (existing?.totalScore ?? 0) + (dedupeSignal ? 0 : score));
|
||||
const maxScore = Math.max(existing?.maxScore ?? 0, dedupeSignal ? 0 : score);
|
||||
const queryHashes = mergeQueryHashes(existing?.queryHashes ?? [], queryHash);
|
||||
const recallDays = mergeRecentDistinct(
|
||||
existing?.recallDays ?? [],
|
||||
formatMemoryDreamingDay(nowMs, params.timezone),
|
||||
MAX_RECALL_DAYS,
|
||||
);
|
||||
const recallDays = mergeRecentDistinct(recallDaysBase, todayBucket, MAX_RECALL_DAYS);
|
||||
const conceptTags = deriveConceptTags({ path: normalizedPath, snippet });
|
||||
|
||||
store.entries[key] = {
|
||||
@@ -621,6 +782,7 @@ export async function recordShortTermRecalls(params: {
|
||||
source: "memory",
|
||||
snippet: snippet || existing?.snippet || "",
|
||||
recallCount,
|
||||
dailyCount,
|
||||
totalScore,
|
||||
maxScore,
|
||||
firstRecalledAt: existing?.firstRecalledAt ?? nowIso,
|
||||
@@ -637,6 +799,60 @@ export async function recordShortTermRecalls(params: {
|
||||
});
|
||||
}
|
||||
|
||||
export async function recordDreamingPhaseSignals(params: {
|
||||
workspaceDir?: string;
|
||||
phase: "light" | "rem";
|
||||
keys: string[];
|
||||
nowMs?: number;
|
||||
}): Promise<void> {
|
||||
const workspaceDir = params.workspaceDir?.trim();
|
||||
if (!workspaceDir) {
|
||||
return;
|
||||
}
|
||||
const keys = [...new Set(params.keys.map((key) => key.trim()).filter(Boolean))];
|
||||
if (keys.length === 0) {
|
||||
return;
|
||||
}
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const nowIso = new Date(nowMs).toISOString();
|
||||
|
||||
await withShortTermLock(workspaceDir, async () => {
|
||||
const [store, phaseSignals] = await Promise.all([
|
||||
readStore(workspaceDir, nowIso),
|
||||
readPhaseSignalStore(workspaceDir, nowIso),
|
||||
]);
|
||||
const knownKeys = new Set(Object.keys(store.entries));
|
||||
|
||||
for (const key of keys) {
|
||||
if (!knownKeys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
const entry = phaseSignals.entries[key] ?? {
|
||||
key,
|
||||
lightHits: 0,
|
||||
remHits: 0,
|
||||
};
|
||||
if (params.phase === "light") {
|
||||
entry.lightHits = Math.min(9999, entry.lightHits + 1);
|
||||
entry.lastLightAt = nowIso;
|
||||
} else {
|
||||
entry.remHits = Math.min(9999, entry.remHits + 1);
|
||||
entry.lastRemAt = nowIso;
|
||||
}
|
||||
phaseSignals.entries[key] = entry;
|
||||
}
|
||||
|
||||
for (const [key, entry] of Object.entries(phaseSignals.entries)) {
|
||||
if (!knownKeys.has(key) || (entry.lightHits <= 0 && entry.remHits <= 0)) {
|
||||
delete phaseSignals.entries[key];
|
||||
}
|
||||
}
|
||||
|
||||
phaseSignals.updatedAt = nowIso;
|
||||
await writePhaseSignalStore(workspaceDir, phaseSignals);
|
||||
});
|
||||
}
|
||||
|
||||
export async function rankShortTermPromotionCandidates(
|
||||
options: RankShortTermPromotionOptions,
|
||||
): Promise<PromotionCandidate[]> {
|
||||
@@ -664,7 +880,10 @@ export async function rankShortTermPromotionCandidates(
|
||||
);
|
||||
const weights = normalizeWeights(options.weights);
|
||||
|
||||
const store = await readStore(workspaceDir, nowIso);
|
||||
const [store, phaseSignals] = await Promise.all([
|
||||
readStore(workspaceDir, nowIso),
|
||||
readPhaseSignalStore(workspaceDir, nowIso),
|
||||
]);
|
||||
const candidates: PromotionCandidate[] = [];
|
||||
|
||||
for (const entry of Object.values(store.entries)) {
|
||||
@@ -674,20 +893,24 @@ export async function rankShortTermPromotionCandidates(
|
||||
if (!includePromoted && entry.promotedAt) {
|
||||
continue;
|
||||
}
|
||||
if (!Number.isFinite(entry.recallCount) || entry.recallCount <= 0) {
|
||||
const recallCount = Math.max(0, Math.floor(entry.recallCount ?? 0));
|
||||
const dailyCount = Math.max(0, Math.floor(entry.dailyCount ?? 0));
|
||||
const signalCount = recallCount + dailyCount;
|
||||
if (signalCount <= 0) {
|
||||
continue;
|
||||
}
|
||||
if (entry.recallCount < minRecallCount) {
|
||||
if (signalCount < minRecallCount) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const avgScore = clampScore(entry.totalScore / Math.max(1, entry.recallCount));
|
||||
const frequency = clampScore(Math.log1p(entry.recallCount) / Math.log1p(10));
|
||||
const avgScore = clampScore(entry.totalScore / Math.max(1, signalCount));
|
||||
const frequency = clampScore(Math.log1p(signalCount) / Math.log1p(10));
|
||||
const uniqueQueries = entry.queryHashes?.length ?? 0;
|
||||
if (uniqueQueries < minUniqueQueries) {
|
||||
const contextDiversity = Math.max(uniqueQueries, entry.recallDays?.length ?? 0);
|
||||
if (contextDiversity < minUniqueQueries) {
|
||||
continue;
|
||||
}
|
||||
const diversity = clampScore(uniqueQueries / 5);
|
||||
const diversity = clampScore(contextDiversity / 5);
|
||||
const lastRecalledAtMs = Date.parse(entry.lastRecalledAt);
|
||||
const ageDays = Number.isFinite(lastRecalledAtMs)
|
||||
? Math.max(0, (nowMs - lastRecalledAtMs) / DAY_MS)
|
||||
@@ -701,13 +924,15 @@ export async function rankShortTermPromotionCandidates(
|
||||
const consolidation = calculateConsolidationComponent(recallDays);
|
||||
const conceptual = calculateConceptualComponent(conceptTags);
|
||||
|
||||
const phaseBoost = calculatePhaseSignalBoost(phaseSignals.entries[entry.key], nowMs);
|
||||
const score =
|
||||
weights.frequency * frequency +
|
||||
weights.relevance * avgScore +
|
||||
weights.diversity * diversity +
|
||||
weights.recency * recency +
|
||||
weights.consolidation * consolidation +
|
||||
weights.conceptual * conceptual;
|
||||
weights.conceptual * conceptual +
|
||||
phaseBoost;
|
||||
|
||||
if (score < minScore) {
|
||||
continue;
|
||||
@@ -720,7 +945,9 @@ export async function rankShortTermPromotionCandidates(
|
||||
endLine: entry.endLine,
|
||||
source: entry.source,
|
||||
snippet: entry.snippet,
|
||||
recallCount: entry.recallCount,
|
||||
recallCount,
|
||||
dailyCount,
|
||||
signalCount,
|
||||
avgScore,
|
||||
maxScore: clampScore(entry.maxScore),
|
||||
uniqueQueries,
|
||||
@@ -998,10 +1225,13 @@ export async function applyShortTermPromotions(
|
||||
if (candidate.score < minScore) {
|
||||
return false;
|
||||
}
|
||||
if (candidate.recallCount < minRecallCount) {
|
||||
const candidateSignalCount =
|
||||
candidate.signalCount ??
|
||||
Math.max(0, candidate.recallCount) + Math.max(0, candidate.dailyCount ?? 0);
|
||||
if (candidateSignalCount < minRecallCount) {
|
||||
return false;
|
||||
}
|
||||
if (candidate.uniqueQueries < minUniqueQueries) {
|
||||
if (Math.max(candidate.uniqueQueries, candidate.recallDays.length) < minUniqueQueries) {
|
||||
return false;
|
||||
}
|
||||
if (maxAgeDays >= 0 && candidate.ageDays > maxAgeDays) {
|
||||
@@ -1082,6 +1312,10 @@ export function resolveShortTermRecallStorePath(workspaceDir: string): string {
|
||||
return resolveStorePath(workspaceDir);
|
||||
}
|
||||
|
||||
export function resolveShortTermPhaseSignalStorePath(workspaceDir: string): string {
|
||||
return resolvePhaseSignalPath(workspaceDir);
|
||||
}
|
||||
|
||||
export function resolveShortTermRecallLockPath(workspaceDir: string): string {
|
||||
return resolveLockPath(workspaceDir);
|
||||
}
|
||||
@@ -1286,6 +1520,10 @@ export async function repairShortTermPromotionArtifacts(params: {
|
||||
key,
|
||||
{
|
||||
...entry,
|
||||
dailyCount: Math.max(
|
||||
0,
|
||||
Math.floor((entry as { dailyCount?: number }).dailyCount ?? 0),
|
||||
),
|
||||
queryHashes: (entry.queryHashes ?? []).slice(-MAX_QUERY_HASHES),
|
||||
recallDays: mergeRecentDistinct(entry.recallDays ?? [], fallbackDay, MAX_RECALL_DAYS),
|
||||
conceptTags: conceptTags.length > 0 ? conceptTags : (entry.conceptTags ?? []),
|
||||
@@ -1327,4 +1565,5 @@ export const __testing = {
|
||||
isProcessLikelyAlive,
|
||||
deriveConceptTags,
|
||||
calculateConsolidationComponent,
|
||||
calculatePhaseSignalBoost,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user