vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-12 17:51:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Memory/dreaming: feed grounded backfill into short-term promotion

Browse Source
This commit is contained in:
Mariano Belinky
2026-04-08 23:11:58 +02:00
parent f6e1da3ab3
commit 5dfe246ef9
12 changed files with 648 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -10,6 +10,7 @@ Docs: https://docs.openclaw.ai
- Plugins/provider-auth: let provider manifests declare `providerAuthAliases` so provider variants can share env vars, auth profiles, config-backed auth, and API-key onboarding choices without core-specific wiring.
- Memory/dreaming: add a grounded REM backfill lane with historical `rem-harness --path`, diary commit, and reset flows so old daily notes can be replayed safely into `DREAMS.md`. Thanks @mbelinky.
- Memory/dreaming: harden grounded diary extraction so `What Happened`, `Reflections`, and durable candidates suppress operational noise and preserve more atomic lasting facts. Thanks @mbelinky.
- Memory/dreaming: feed grounded durable backfill candidates into the live short-term promotion store so historical diary replays can flow through the normal deep promotion path without a second memory stack. Thanks @mbelinky.
- Control UI/dreaming: add a structured diary view with timeline navigation, backfill/reset controls, and traceable dreaming summaries. Thanks @mbelinky.
### Fixes

185
extensions/memory-core/src/cli.runtime.ts
View File

@@ -42,8 +42,10 @@ import { previewGroundedRemMarkdown } from "./rem-evidence.js";
import {
applyShortTermPromotions,
auditShortTermPromotionArtifacts,
removeGroundedShortTermCandidates,
repairShortTermPromotionArtifacts,
readShortTermRecallEntries,
recordGroundedShortTermCandidates,
recordShortTermRecalls,
rankShortTermPromotionCandidates,
resolveShortTermRecallLockPath,
@@ -296,6 +298,100 @@ function groundedMarkdownToDiaryLines(markdown: string): string[] {
.filter((line, index, lines) => !(line.length === 0 && lines[index - 1]?.length === 0));
}
function parseGroundedRef(
fallbackPath: string,
ref: string,
): { path: string; startLine: number; endLine: number } | null {
const trimmed = ref.trim();
if (!trimmed) {
return null;
}
const match = trimmed.match(/^(.*?):(\d+)(?:-(\d+))?$/);
if (!match) {
return null;
}
return {
path: (match[1] ?? fallbackPath).replaceAll("\\", "/").replace(/^\.\//, ""),
startLine: Math.max(1, Number(match[2])),
endLine: Math.max(1, Number(match[3] ?? match[2])),
};
}
function collectGroundedShortTermSeedItems(
previews: Awaited<ReturnType<typeof previewGroundedRemMarkdown>>["files"],
): Array<{
path: string;
startLine: number;
endLine: number;
snippet: string;
score: number;
query: string;
signalCount: number;
dayBucket?: string;
}> {
const items: Array<{
path: string;
startLine: number;
endLine: number;
snippet: string;
score: number;
query: string;
signalCount: number;
dayBucket?: string;
}> = [];
const seen = new Set<string>();
for (const file of previews) {
const dayBucket = extractIsoDayFromPath(file.path) ?? undefined;
const signals = [
...file.memoryImplications.map((item) => ({
text: item.text,
refs: item.refs,
score: 0.92,
query: "__dreaming_grounded_backfill__:lasting-update",
signalCount: 2,
})),
...file.candidates
.filter((candidate) => candidate.lean === "likely_durable")
.map((candidate) => ({
text: candidate.text,
refs: candidate.refs,
score: 0.82,
query: "__dreaming_grounded_backfill__:candidate",
signalCount: 1,
})),
];
for (const signal of signals) {
if (!signal.text.trim()) {
continue;
}
const firstRef = signal.refs.find((ref) => ref.trim().length > 0);
const parsedRef = firstRef ? parseGroundedRef(file.path, firstRef) : null;
if (!parsedRef) {
continue;
}
const key = `${parsedRef.path}:${parsedRef.startLine}:${parsedRef.endLine}:${signal.query}:${signal.text.toLowerCase()}`;
if (seen.has(key)) {
continue;
}
seen.add(key);
items.push({
path: parsedRef.path,
startLine: parsedRef.startLine,
endLine: parsedRef.endLine,
snippet: signal.text,
score: signal.score,
query: signal.query,
signalCount: signal.signalCount,
...(dayBucket ? { dayBucket } : {}),
});
}
}
return items;
}
function matchesPromotionSelector(
candidate: {
key: string;
@@ -551,9 +647,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
purpose: managerPurpose,
run: async (manager) => {
const deep = Boolean(opts.deep || opts.index);
let embeddingProbe:
| Awaited<ReturnType<typeof manager.probeEmbeddingAvailability>>
| undefined;
let embeddingProbe: { ok?: boolean; error?: string } | undefined;
let indexError: string | undefined;
const syncFn = manager.sync ? manager.sync.bind(manager) : undefined;
if (deep) {
@@ -1548,14 +1642,30 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
return;
}
if (opts.rollback) {
const removed = await removeBackfillDiaryEntries({ workspaceDir });
if (opts.rollback || opts.rollbackShortTerm) {
const diaryRollback = opts.rollback
? await removeBackfillDiaryEntries({ workspaceDir })
: null;
const shortTermRollback = opts.rollbackShortTerm
? await removeGroundedShortTermCandidates({ workspaceDir })
: null;
if (opts.json) {
defaultRuntime.writeJson({
workspaceDir,
rollback: true,
dreamsPath: removed.dreamsPath,
removedEntries: removed.removed,
rollback: Boolean(opts.rollback),
rollbackShortTerm: Boolean(opts.rollbackShortTerm),
...(diaryRollback
? {
dreamsPath: diaryRollback.dreamsPath,
removedEntries: diaryRollback.removed,
}
: {}),
...(shortTermRollback
? {
shortTermStorePath: shortTermRollback.storePath,
removedShortTermEntries: shortTermRollback.removed,
}
: {}),
});
return;
}
@@ -1563,8 +1673,30 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
[
`${colorize(isRich(), theme.heading, "REM Backfill")} ${colorize(isRich(), theme.muted, "(rollback)")}`,
colorize(isRich(), theme.muted, `workspace=${shortenHomePath(workspaceDir)}`),
colorize(isRich(), theme.muted, `dreamsPath=${shortenHomePath(removed.dreamsPath)}`),
colorize(isRich(), theme.muted, `removedEntries=${removed.removed}`),
...(diaryRollback
? [
colorize(
isRich(),
theme.muted,
`dreamsPath=${shortenHomePath(diaryRollback.dreamsPath)}`,
),
colorize(isRich(), theme.muted, `removedEntries=${diaryRollback.removed}`),
]
: []),
...(shortTermRollback
? [
colorize(
isRich(),
theme.muted,
`shortTermStorePath=${shortenHomePath(shortTermRollback.storePath)}`,
),
colorize(
isRich(),
theme.muted,
`removedShortTermEntries=${shortTermRollback.removed}`,
),
]
: []),
].join("\n"),
);
return;
@@ -1619,6 +1751,24 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
entries,
timezone: remConfig.timezone,
});
let stagedShortTermEntries = 0;
let replacedShortTermEntries = 0;
if (opts.stageShortTerm) {
const cleared = await removeGroundedShortTermCandidates({ workspaceDir });
replacedShortTermEntries = cleared.removed;
const shortTermSeedItems = collectGroundedShortTermSeedItems(grounded.files);
if (shortTermSeedItems.length > 0) {
await recordGroundedShortTermCandidates({
workspaceDir,
query: "__dreaming_grounded_backfill__",
items: shortTermSeedItems,
dedupeByQueryPerDay: true,
nowMs: Date.now(),
timezone: remConfig.timezone,
});
}
stagedShortTermEntries = shortTermSeedItems.length;
}
if (opts.json) {
defaultRuntime.writeJson({
@@ -1629,6 +1779,12 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
writtenEntries: written.written,
replacedEntries: written.replaced,
dreamsPath: written.dreamsPath,
...(opts.stageShortTerm
? {
stagedShortTermEntries,
replacedShortTermEntries,
}
: {}),
});
return;
}
@@ -1644,6 +1800,15 @@ export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
theme.muted,
`historicalFiles=${sourceFiles.length} writtenEntries=${written.written} replacedEntries=${written.replaced}`,
),
...(opts.stageShortTerm
? [
colorize(
rich,
theme.muted,
`stagedShortTermEntries=${stagedShortTermEntries} replacedShortTermEntries=${replacedShortTermEntries}`,
),
]
: []),
colorize(rich, theme.muted, `dreamsPath=${shortenHomePath(written.dreamsPath)}`),
].join("\n"),
);

67
extensions/memory-core/src/cli.test.ts
View File

@@ -9,7 +9,7 @@ import {
spyRuntimeJson,
spyRuntimeLogs,
} from "../../../src/cli/test-runtime-capture.js";
import { recordShortTermRecalls } from "./short-term-promotion.js";
import { readShortTermRecallEntries, recordShortTermRecalls } from "./short-term-promotion.js";
const getMemorySearchManager = vi.hoisted(() => vi.fn());
const loadConfig = vi.hoisted(() => vi.fn(() => ({})));
@@ -1074,6 +1074,71 @@ describe("memory cli", () => {
});
});
it("stages grounded durable candidates into the live short-term store", async () => {
await withTempWorkspace(async (workspaceDir) => {
const historyDir = path.join(workspaceDir, "history");
await fs.mkdir(historyDir, { recursive: true });
const historyPath = path.join(historyDir, "2025-01-01.md");
await fs.writeFile(
historyPath,
[
"## Preferences Learned",
'- Always use "Happy Together" calendar for flights and reservations.',
].join("\n") + "\n",
"utf-8",
);
const close = vi.fn(async () => {});
mockManager({
status: () => makeMemoryStatus({ workspaceDir }),
close,
});
await runMemoryCli(["rem-backfill", "--path", historyPath, "--stage-short-term"]);
const entries = await readShortTermRecallEntries({ workspaceDir });
expect(entries).toHaveLength(1);
expect(entries[0]?.snippet).toContain("Happy Together");
expect(entries[0]?.groundedCount).toBe(3);
expect(entries[0]?.queryHashes).toHaveLength(2);
expect(entries[0]?.recallCount).toBe(0);
expect(close).toHaveBeenCalled();
});
});
it("rolls back grounded staged short-term entries without touching diary rollback", async () => {
await withTempWorkspace(async (workspaceDir) => {
const historyDir = path.join(workspaceDir, "history");
await fs.mkdir(historyDir, { recursive: true });
const historyPath = path.join(historyDir, "2025-01-01.md");
await fs.writeFile(
historyPath,
[
"## Preferences Learned",
'- Always use "Happy Together" calendar for flights and reservations.',
].join("\n") + "\n",
"utf-8",
);
const close = vi.fn(async () => {});
mockManager({
status: () => makeMemoryStatus({ workspaceDir }),
close,
});
await runMemoryCli(["rem-backfill", "--path", historyPath, "--stage-short-term"]);
mockManager({
status: () => makeMemoryStatus({ workspaceDir }),
close,
});
await runMemoryCli(["rem-backfill", "--rollback-short-term"]);
const entries = await readShortTermRecallEntries({ workspaceDir });
expect(entries).toHaveLength(0);
expect(close).toHaveBeenCalled();
});
});
it("prefers persistence-relevant evidence over narrated operational logs in grounded what happened", async () => {
await withTempWorkspace(async (workspaceDir) => {
const historyDir = path.join(workspaceDir, "history");

14
extensions/memory-core/src/cli.ts
View File

@@ -105,6 +105,10 @@ export function registerMemoryCli(program: Command) {
"openclaw memory rem-backfill --path ./memory",
"Write grounded historical REM entries into DREAMS.md for UI review.",
],
[
"openclaw memory rem-backfill --path ./memory --stage-short-term",
"Also seed durable grounded candidates into the live short-term promotion store.",
],
["openclaw memory status --json", "Output machine-readable JSON (good for scripts)."],
])}\n\n${theme.muted("Docs:")} ${formatDocsLink("/cli/memory", "docs.openclaw.ai/cli/memory")}\n`,
);
@@ -201,6 +205,16 @@ export function registerMemoryCli(program: Command) {
.option("--agent <id>", "Agent id (default: default agent)")
.option("--path <file-or-dir>", "Historical daily memory file(s) or directory")
.option("--rollback", "Remove previously written grounded REM backfill entries", false)
.option(
"--stage-short-term",
"Also seed grounded durable candidates into the short-term promotion store",
false,
)
.option(
"--rollback-short-term",
"Remove previously seeded grounded short-term candidates",
false,
)
.option("--json", "Print JSON")
.action(async (opts: MemoryRemBackfillOptions) => {
await runMemoryRemBackfill(opts);

2
extensions/memory-core/src/cli.types.ts
View File

@@ -36,4 +36,6 @@ export type MemoryRemHarnessOptions = MemoryCommandOptions & {
export type MemoryRemBackfillOptions = MemoryCommandOptions & {
path?: string;
rollback?: boolean;
stageShortTerm?: boolean;
rollbackShortTerm?: boolean;
};

125
extensions/memory-core/src/short-term-promotion.test.ts
View File

@@ -11,9 +11,11 @@ import {
applyShortTermPromotions,
auditShortTermPromotionArtifacts,
isShortTermMemoryPath,
recordGroundedShortTermCandidates,
rankShortTermPromotionCandidates,
recordDreamingPhaseSignals,
recordShortTermRecalls,
removeGroundedShortTermCandidates,
repairShortTermPromotionArtifacts,
resolveShortTermRecallLockPath,
resolveShortTermPhaseSignalStorePath,
@@ -177,6 +179,128 @@ describe("short-term promotion", () => {
});
});
it("lets grounded durable evidence satisfy default deep thresholds", async () => {
await withTempWorkspace(async (workspaceDir) => {
await writeDailyMemoryNote(workspaceDir, "2026-04-03", [
'Always use "Happy Together" calendar for flights and reservations.',
]);
await recordGroundedShortTermCandidates({
workspaceDir,
query: "__dreaming_grounded_backfill__",
items: [
{
path: "memory/2026-04-03.md",
startLine: 1,
endLine: 1,
snippet: 'Always use "Happy Together" calendar for flights and reservations.',
score: 0.92,
query: "__dreaming_grounded_backfill__:lasting-update",
signalCount: 2,
dayBucket: "2026-04-03",
},
{
path: "memory/2026-04-03.md",
startLine: 1,
endLine: 1,
snippet: 'Always use "Happy Together" calendar for flights and reservations.',
score: 0.82,
query: "__dreaming_grounded_backfill__:candidate",
signalCount: 1,
dayBucket: "2026-04-03",
},
],
dedupeByQueryPerDay: true,
nowMs: Date.parse("2026-04-03T10:00:00.000Z"),
});
const ranked = await rankShortTermPromotionCandidates({
workspaceDir,
nowMs: Date.parse("2026-04-03T10:00:00.000Z"),
});
expect(ranked).toHaveLength(1);
expect(ranked[0]?.groundedCount).toBe(3);
expect(ranked[0]?.uniqueQueries).toBe(2);
expect(ranked[0]?.avgScore).toBeGreaterThan(0.85);
const applied = await applyShortTermPromotions({
workspaceDir,
candidates: ranked,
nowMs: Date.parse("2026-04-03T10:00:00.000Z"),
});
expect(applied.applied).toBe(1);
const memory = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8");
expect(memory).toContain('Always use "Happy Together" calendar');
});
});
it("removes grounded-only staged entries without deleting mixed live entries", async () => {
await withTempWorkspace(async (workspaceDir) => {
await writeDailyMemoryNote(workspaceDir, "2026-04-03", [
"Grounded only rule.",
"Live recall-backed rule.",
]);
await recordGroundedShortTermCandidates({
workspaceDir,
query: "__dreaming_grounded_backfill__",
items: [
{
path: "memory/2026-04-03.md",
startLine: 1,
endLine: 1,
snippet: "Grounded only rule.",
score: 0.92,
query: "__dreaming_grounded_backfill__:lasting-update",
signalCount: 2,
dayBucket: "2026-04-03",
},
{
path: "memory/2026-04-03.md",
startLine: 2,
endLine: 2,
snippet: "Live recall-backed rule.",
score: 0.92,
query: "__dreaming_grounded_backfill__:lasting-update",
signalCount: 2,
dayBucket: "2026-04-03",
},
],
dedupeByQueryPerDay: true,
});
await recordShortTermRecalls({
workspaceDir,
query: "live recall",
results: [
{
path: "memory/2026-04-03.md",
startLine: 2,
endLine: 2,
score: 0.87,
snippet: "Live recall-backed rule.",
source: "memory",
},
],
});
const result = await removeGroundedShortTermCandidates({ workspaceDir });
expect(result.removed).toBe(1);
const ranked = await rankShortTermPromotionCandidates({
workspaceDir,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
});
expect(ranked).toHaveLength(1);
expect(ranked[0]?.snippet).toContain("Live recall-backed rule");
expect(ranked[0]?.groundedCount).toBe(2);
expect(ranked[0]?.recallCount).toBe(1);
});
});
it("rewards spaced recalls as consolidation instead of only raw count", async () => {
await withTempWorkspace(async (workspaceDir) => {
await recordShortTermRecalls({
@@ -1100,6 +1224,7 @@ describe("short-term promotion", () => {
snippet,
recallCount: 2,
dailyCount: 0,
groundedCount: 0,
totalScore: 1.8,
maxScore: 0.95,
firstRecalledAt: "2026-04-01T00:00:00.000Z",

249
extensions/memory-core/src/short-term-promotion.ts
View File

@@ -64,6 +64,7 @@ export type ShortTermRecallEntry = {
snippet: string;
recallCount: number;
dailyCount: number;
groundedCount: number;
totalScore: number;
maxScore: number;
firstRecalledAt: string;
@@ -71,6 +72,7 @@ export type ShortTermRecallEntry = {
queryHashes: string[];
recallDays: string[];
conceptTags: string[];
claimHash?: string;
promotedAt?: string;
};
@@ -112,10 +114,12 @@ export type PromotionCandidate = {
snippet: string;
recallCount: number;
dailyCount?: number;
groundedCount?: number;
signalCount?: number;
avgScore: number;
maxScore: number;
uniqueQueries: number;
claimHash?: string;
promotedAt?: string;
firstRecalledAt: string;
lastRecalledAt: string;
@@ -232,13 +236,19 @@ function normalizeMemoryPath(rawPath: string): string {
return rawPath.replaceAll("\\", "/").replace(/^\.\//, "");
}
function buildClaimHash(snippet: string): string {
return createHash("sha1").update(normalizeSnippet(snippet)).digest("hex").slice(0, 12);
}
function buildEntryKey(result: {
path: string;
startLine: number;
endLine: number;
source: string;
claimHash?: string;
}): string {
return `${result.source}:${normalizeMemoryPath(result.path)}:${result.startLine}:${result.endLine}`;
const base = `${result.source}:${normalizeMemoryPath(result.path)}:${result.startLine}:${result.endLine}`;
return result.claimHash ? `${base}:${result.claimHash}` : base;
}
function hashQuery(query: string): string {
@@ -315,6 +325,18 @@ function normalizeDistinctStrings(values: unknown[], limit: number): string[] {
return normalized;
}
function totalSignalCountForEntry(entry: {
recallCount?: number;
dailyCount?: number;
groundedCount?: number;
}): number {
return (
Math.max(0, Math.floor(entry.recallCount ?? 0)) +
Math.max(0, Math.floor(entry.dailyCount ?? 0)) +
Math.max(0, Math.floor(entry.groundedCount ?? 0))
);
}
function calculateConsolidationComponent(recallDays: string[]): number {
if (recallDays.length === 0) {
return 0;
@@ -371,6 +393,7 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
const recallCount = Math.max(0, Math.floor(Number(entry.recallCount) || 0));
const dailyCount = Math.max(0, Math.floor(Number(entry.dailyCount) || 0));
const groundedCount = Math.max(0, Math.floor(Number(entry.groundedCount) || 0));
const totalScore = Math.max(0, Number(entry.totalScore) || 0);
const maxScore = clampScore(Number(entry.maxScore) || 0);
const firstRecalledAt =
@@ -378,6 +401,10 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
const lastRecalledAt =
typeof entry.lastRecalledAt === "string" ? entry.lastRecalledAt : nowIso;
const promotedAt = typeof entry.promotedAt === "string" ? entry.promotedAt : undefined;
const claimHash =
typeof entry.claimHash === "string" && entry.claimHash.trim().length > 0
? entry.claimHash.trim()
: undefined;
const snippet = typeof entry.snippet === "string" ? normalizeSnippet(entry.snippet) : "";
const queryHashes = Array.isArray(entry.queryHashes)
? normalizeDistinctStrings(entry.queryHashes, MAX_QUERY_HASHES)
@@ -396,7 +423,8 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
)
: deriveConceptTags({ path: entryPath, snippet });
const normalizedKey = key || buildEntryKey({ path: entryPath, startLine, endLine, source });
const normalizedKey =
key || buildEntryKey({ path: entryPath, startLine, endLine, source, claimHash });
entries[normalizedKey] = {
key: normalizedKey,
path: entryPath,
@@ -406,6 +434,7 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
snippet,
recallCount,
dailyCount,
groundedCount,
totalScore,
maxScore,
firstRecalledAt,
@@ -413,6 +442,7 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
queryHashes,
recallDays: recallDays.slice(-MAX_RECALL_DAYS),
conceptTags,
...(claimHash ? { claimHash } : {}),
...(promotedAt ? { promotedAt } : {}),
};
}
@@ -568,7 +598,7 @@ function isProcessLikelyAlive(pid: number): boolean {
process.kill(pid, 0);
return true;
} catch (err) {
const code = (err as NodeJS.ErrnoException | undefined)?.code;
const code = (err as NodeJS.ErrnoException).code;
if (code === "ESRCH") {
return false;
}
@@ -621,9 +651,8 @@ async function withShortTermLock<T>(workspaceDir: string, task: () => Promise<T>
const startedAt = Date.now();
while (true) {
let lockHandle: Awaited<ReturnType<typeof fs.open>> | undefined;
try {
lockHandle = await fs.open(lockPath, "wx");
const lockHandle = await fs.open(lockPath, "wx");
await lockHandle
.writeFile(`${process.pid}:${Date.now()}\n`, "utf-8")
.catch(() => undefined);
@@ -812,10 +841,21 @@ export async function recordShortTermRecalls(params: {
const store = await readStore(workspaceDir, nowIso);
for (const result of relevant) {
const key = buildEntryKey(result);
const normalizedPath = normalizeMemoryPath(result.path);
const existing = store.entries[key];
const snippet = normalizeSnippet(result.snippet);
const claimHash = snippet ? buildClaimHash(snippet) : undefined;
const groundedKey = claimHash
? buildEntryKey({
path: normalizedPath,
startLine: Math.max(1, Math.floor(result.startLine)),
endLine: Math.max(1, Math.floor(result.endLine)),
source: "memory",
claimHash,
})
: null;
const baseKey = buildEntryKey(result);
const key = groundedKey && store.entries[groundedKey] ? groundedKey : baseKey;
const existing = store.entries[key];
const score = clampScore(result.score);
const recallDaysBase = existing?.recallDays ?? [];
const queryHashesBase = existing?.queryHashes ?? [];
@@ -846,6 +886,7 @@ export async function recordShortTermRecalls(params: {
snippet: snippet || existing?.snippet || "",
recallCount,
dailyCount,
groundedCount: Math.max(0, Math.floor(existing?.groundedCount ?? 0)),
totalScore,
maxScore,
firstRecalledAt: existing?.firstRecalledAt ?? nowIso,
@@ -853,6 +894,7 @@ export async function recordShortTermRecalls(params: {
queryHashes,
recallDays,
conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []),
...(existing?.claimHash ? { claimHash: existing.claimHash } : {}),
...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}),
};
}
@@ -874,6 +916,129 @@ export async function recordShortTermRecalls(params: {
});
}
export async function recordGroundedShortTermCandidates(params: {
workspaceDir?: string;
query: string;
items: Array<{
path: string;
startLine: number;
endLine: number;
snippet: string;
score: number;
query?: string;
signalCount?: number;
dayBucket?: string;
}>;
dedupeByQueryPerDay?: boolean;
dayBucket?: string;
nowMs?: number;
timezone?: string;
}): Promise<void> {
const workspaceDir = params.workspaceDir?.trim();
if (!workspaceDir) {
return;
}
const query = params.query.trim();
if (!query) {
return;
}
const relevant = params.items
.map((item) => {
const snippet = normalizeSnippet(item.snippet);
const normalizedPath = normalizeMemoryPath(item.path);
if (
!snippet ||
!normalizedPath ||
!isShortTermMemoryPath(normalizedPath) ||
!Number.isFinite(item.startLine) ||
!Number.isFinite(item.endLine)
) {
return null;
}
return {
path: normalizedPath,
startLine: Math.max(1, Math.floor(item.startLine)),
endLine: Math.max(1, Math.floor(item.endLine)),
snippet,
score: clampScore(item.score),
query: normalizeSnippet(item.query ?? query),
signalCount: Math.max(1, Math.floor(item.signalCount ?? 1)),
dayBucket: normalizeIsoDay(item.dayBucket ?? params.dayBucket ?? ""),
};
})
.filter((item): item is NonNullable<typeof item> => item !== null);
if (relevant.length === 0) {
return;
}
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
const nowIso = new Date(nowMs).toISOString();
const fallbackDayBucket = formatMemoryDreamingDay(nowMs, params.timezone);
await withShortTermLock(workspaceDir, async () => {
const store = await readStore(workspaceDir, nowIso);
for (const item of relevant) {
const dayBucket = item.dayBucket ?? fallbackDayBucket;
const effectiveQuery = item.query || query;
if (!effectiveQuery) {
continue;
}
const queryHash = hashQuery(effectiveQuery);
const claimHash = buildClaimHash(item.snippet);
const key = buildEntryKey({
path: item.path,
startLine: item.startLine,
endLine: item.endLine,
source: "memory",
claimHash,
});
const existing = store.entries[key];
const recallDaysBase = existing?.recallDays ?? [];
const queryHashesBase = existing?.queryHashes ?? [];
const dedupeSignal =
Boolean(params.dedupeByQueryPerDay) &&
queryHashesBase.includes(queryHash) &&
recallDaysBase.includes(dayBucket);
const groundedCount = Math.max(
0,
Math.floor(existing?.groundedCount ?? 0) + (dedupeSignal ? 0 : item.signalCount),
);
const totalScore = Math.max(
0,
(existing?.totalScore ?? 0) + (dedupeSignal ? 0 : item.score * item.signalCount),
);
const maxScore = Math.max(existing?.maxScore ?? 0, dedupeSignal ? 0 : item.score);
const queryHashes = mergeQueryHashes(existing?.queryHashes ?? [], queryHash);
const recallDays = mergeRecentDistinct(recallDaysBase, dayBucket, MAX_RECALL_DAYS);
const conceptTags = deriveConceptTags({ path: item.path, snippet: item.snippet });
store.entries[key] = {
key,
path: item.path,
startLine: item.startLine,
endLine: item.endLine,
source: "memory",
snippet: item.snippet,
recallCount: Math.max(0, Math.floor(existing?.recallCount ?? 0)),
dailyCount: Math.max(0, Math.floor(existing?.dailyCount ?? 0)),
groundedCount,
totalScore,
maxScore,
firstRecalledAt: existing?.firstRecalledAt ?? nowIso,
lastRecalledAt: nowIso,
queryHashes,
recallDays,
conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []),
claimHash,
...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}),
};
}
store.updatedAt = nowIso;
await writeStore(workspaceDir, store);
});
}
export async function recordDreamingPhaseSignals(params: {
workspaceDir?: string;
phase: "light" | "rem";
@@ -970,7 +1135,8 @@ export async function rankShortTermPromotionCandidates(
}
const recallCount = Math.max(0, Math.floor(entry.recallCount ?? 0));
const dailyCount = Math.max(0, Math.floor(entry.dailyCount ?? 0));
const signalCount = recallCount + dailyCount;
const groundedCount = Math.max(0, Math.floor(entry.groundedCount ?? 0));
const signalCount = totalSignalCountForEntry(entry);
if (signalCount <= 0) {
continue;
}
@@ -996,7 +1162,10 @@ export async function rankShortTermPromotionCandidates(
const recency = clampScore(calculateRecencyComponent(ageDays, halfLifeDays));
const recallDays = entry.recallDays ?? [];
const conceptTags = entry.conceptTags ?? [];
const consolidation = calculateConsolidationComponent(recallDays);
const consolidation = Math.max(
calculateConsolidationComponent(recallDays),
clampScore(groundedCount / 3),
);
const conceptual = calculateConceptualComponent(conceptTags);
const phaseBoost = calculatePhaseSignalBoost(phaseSignals.entries[entry.key], nowMs);
@@ -1022,10 +1191,12 @@ export async function rankShortTermPromotionCandidates(
snippet: entry.snippet,
recallCount,
dailyCount,
groundedCount,
signalCount,
avgScore,
maxScore: clampScore(entry.maxScore),
uniqueQueries,
...(entry.claimHash ? { claimHash: entry.claimHash } : {}),
promotedAt: entry.promotedAt,
firstRecalledAt: entry.firstRecalledAt,
lastRecalledAt: entry.lastRecalledAt,
@@ -1300,9 +1471,15 @@ export async function applyShortTermPromotions(
if (candidate.score < minScore) {
return false;
}
const candidateSignalCount =
const candidateSignalCount = Math.max(
0,
candidate.signalCount ??
Math.max(0, candidate.recallCount) + Math.max(0, candidate.dailyCount ?? 0);
totalSignalCountForEntry({
recallCount: candidate.recallCount,
dailyCount: candidate.dailyCount,
groundedCount: candidate.groundedCount,
}),
);
if (candidateSignalCount < minRecallCount) {
return false;
}
@@ -1606,6 +1783,10 @@ export async function repairShortTermPromotionArtifacts(params: {
0,
Math.floor((entry as { dailyCount?: number }).dailyCount ?? 0),
),
groundedCount: Math.max(
0,
Math.floor((entry as { groundedCount?: number }).groundedCount ?? 0),
),
queryHashes: (entry.queryHashes ?? []).slice(-MAX_QUERY_HASHES),
recallDays: mergeRecentDistinct(entry.recallDays ?? [], fallbackDay, MAX_RECALL_DAYS),
conceptTags: conceptTags.length > 0 ? conceptTags : (entry.conceptTags ?? []),
@@ -1641,6 +1822,50 @@ export async function repairShortTermPromotionArtifacts(params: {
};
}
export async function removeGroundedShortTermCandidates(params: {
workspaceDir: string;
}): Promise<{ removed: number; storePath: string }> {
const workspaceDir = params.workspaceDir.trim();
const storePath = resolveStorePath(workspaceDir);
const nowIso = new Date().toISOString();
let removed = 0;
await withShortTermLock(workspaceDir, async () => {
const [store, phaseSignals] = await Promise.all([
readStore(workspaceDir, nowIso),
readPhaseSignalStore(workspaceDir, nowIso),
]);
for (const [key, entry] of Object.entries(store.entries)) {
if (
Math.max(0, Math.floor(entry.groundedCount ?? 0)) > 0 &&
Math.max(0, Math.floor(entry.recallCount ?? 0)) === 0 &&
Math.max(0, Math.floor(entry.dailyCount ?? 0)) === 0
) {
delete store.entries[key];
removed += 1;
}
}
for (const key of Object.keys(phaseSignals.entries)) {
if (!Object.hasOwn(store.entries, key)) {
delete phaseSignals.entries[key];
}
}
if (removed > 0) {
store.updatedAt = nowIso;
phaseSignals.updatedAt = nowIso;
await Promise.all([
writeStore(workspaceDir, store),
writePhaseSignalStore(workspaceDir, phaseSignals),
]);
}
});
return { removed, storePath };
}
export const __testing = {
parseLockOwnerPid,
canStealStaleLock,
@@ -1648,4 +1873,6 @@ export const __testing = {
deriveConceptTags,
calculateConsolidationComponent,
calculatePhaseSignalBoost,
buildClaimHash,
totalSignalCountForEntry,
};

11
src/gateway/server-methods/doctor.ts
View File

@@ -64,6 +64,7 @@ type DoctorMemoryDreamingEntryPayload = {
snippet: string;
recallCount: number;
dailyCount: number;
groundedCount: number;
totalSignalCount: number;
lightHits: number;
remHits: number;
@@ -81,6 +82,7 @@ type DoctorMemoryDreamingPayload = {
shortTermCount: number;
recallSignalCount: number;
dailySignalCount: number;
groundedSignalCount: number;
totalSignalCount: number;
phaseSignalCount: number;
lightPhaseHitCount: number;
@@ -166,6 +168,7 @@ function resolveDreamingConfig(
| "shortTermCount"
| "recallSignalCount"
| "dailySignalCount"
| "groundedSignalCount"
| "totalSignalCount"
| "phaseSignalCount"
| "lightPhaseHitCount"
@@ -258,6 +261,7 @@ type DreamingStoreStats = Pick<
| "shortTermCount"
| "recallSignalCount"
| "dailySignalCount"
| "groundedSignalCount"
| "totalSignalCount"
| "phaseSignalCount"
| "lightPhaseHitCount"
@@ -370,6 +374,7 @@ async function loadDreamingStoreStats(
let shortTermCount = 0;
let recallSignalCount = 0;
let dailySignalCount = 0;
let groundedSignalCount = 0;
let totalSignalCount = 0;
let phaseSignalCount = 0;
let lightPhaseHitCount = 0;
@@ -396,7 +401,8 @@ async function loadDreamingStoreStats(
const range = parseEntryRangeFromKey(entryKey, entry.startLine, entry.endLine);
const recallCount = toNonNegativeInt(entry.recallCount);
const dailyCount = toNonNegativeInt(entry.dailyCount);
const totalEntrySignalCount = recallCount + dailyCount;
const groundedCount = toNonNegativeInt(entry.groundedCount);
const totalEntrySignalCount = recallCount + dailyCount + groundedCount;
const snippet =
normalizeTrimmedString(entry.snippet) ??
normalizeTrimmedString(entry.summary) ??
@@ -410,6 +416,7 @@ async function loadDreamingStoreStats(
snippet,
recallCount,
dailyCount,
groundedCount,
totalSignalCount: totalEntrySignalCount,
lightHits: 0,
remHits: 0,
@@ -422,6 +429,7 @@ async function loadDreamingStoreStats(
activeKeys.add(entryKey);
recallSignalCount += recallCount;
dailySignalCount += dailyCount;
groundedSignalCount += groundedCount;
totalSignalCount += totalEntrySignalCount;
shortTermEntries.push(detail);
activeEntries.set(entryKey, detail);
@@ -476,6 +484,7 @@ async function loadDreamingStoreStats(
shortTermCount,
recallSignalCount,
dailySignalCount,
groundedSignalCount,
totalSignalCount,
phaseSignalCount,
lightPhaseHitCount,

6
ui/src/ui/controllers/dreaming.test.ts
View File

@@ -52,6 +52,7 @@ describe("dreaming controller", () => {
shortTermCount: 8,
recallSignalCount: 14,
dailySignalCount: 6,
groundedSignalCount: 5,
totalSignalCount: 20,
phaseSignalCount: 11,
lightPhaseHitCount: 7,
@@ -67,6 +68,7 @@ describe("dreaming controller", () => {
snippet: "Emma prefers shorter, lower-pressure check-ins.",
recallCount: 2,
dailyCount: 1,
groundedCount: 1,
totalSignalCount: 3,
lightHits: 1,
remHits: 2,
@@ -83,6 +85,7 @@ describe("dreaming controller", () => {
snippet: "Emma prefers shorter, lower-pressure check-ins.",
recallCount: 2,
dailyCount: 1,
groundedCount: 1,
totalSignalCount: 3,
lightHits: 1,
remHits: 2,
@@ -98,6 +101,7 @@ describe("dreaming controller", () => {
snippet: "Use the Happy Together calendar for flights.",
recallCount: 3,
dailyCount: 2,
groundedCount: 0,
totalSignalCount: 5,
lightHits: 0,
remHits: 0,
@@ -146,6 +150,7 @@ describe("dreaming controller", () => {
expect.objectContaining({
enabled: true,
shortTermCount: 8,
groundedSignalCount: 5,
totalSignalCount: 20,
phaseSignalCount: 11,
promotedToday: 2,
@@ -153,6 +158,7 @@ describe("dreaming controller", () => {
expect.objectContaining({
snippet: "Emma prefers shorter, lower-pressure check-ins.",
totalSignalCount: 3,
groundedCount: 1,
phaseHitCount: 3,
}),
],

4
ui/src/ui/controllers/dreaming.ts
View File

@@ -40,6 +40,7 @@ export type DreamingEntry = {
snippet: string;
recallCount: number;
dailyCount: number;
groundedCount: number;
totalSignalCount: number;
lightHits: number;
remHits: number;
@@ -57,6 +58,7 @@ export type DreamingStatus = {
shortTermCount: number;
recallSignalCount: number;
dailySignalCount: number;
groundedSignalCount: number;
totalSignalCount: number;
phaseSignalCount: number;
lightPhaseHitCount: number;
@@ -211,6 +213,7 @@ function normalizeDreamingEntry(raw: unknown): DreamingEntry | null {
snippet,
recallCount: normalizeFiniteInt(record?.recallCount, 0),
dailyCount: normalizeFiniteInt(record?.dailyCount, 0),
groundedCount: normalizeFiniteInt(record?.groundedCount, 0),
totalSignalCount: normalizeFiniteInt(record?.totalSignalCount, 0),
lightHits: normalizeFiniteInt(record?.lightHits, 0),
remHits: normalizeFiniteInt(record?.remHits, 0),
@@ -252,6 +255,7 @@ function normalizeDreamingStatus(raw: unknown): DreamingStatus | null {
shortTermCount: normalizeFiniteInt(record.shortTermCount, 0),
recallSignalCount: normalizeFiniteInt(record.recallSignalCount, 0),
dailySignalCount: normalizeFiniteInt(record.dailySignalCount, 0),
groundedSignalCount: normalizeFiniteInt(record.groundedSignalCount, 0),
totalSignalCount: normalizeFiniteInt(record.totalSignalCount, 0),
phaseSignalCount: normalizeFiniteInt(record.phaseSignalCount, 0),
lightPhaseHitCount: normalizeFiniteInt(record.lightPhaseHitCount, 0),

3
ui/src/ui/views/dreaming.test.ts
View File

@@ -20,6 +20,7 @@ function buildProps(overrides?: Partial<DreamingProps>): DreamingProps {
snippet: "Emma prefers shorter, lower-pressure check-ins.",
recallCount: 2,
dailyCount: 1,
groundedCount: 1,
totalSignalCount: 3,
lightHits: 1,
remHits: 1,
@@ -35,6 +36,7 @@ function buildProps(overrides?: Partial<DreamingProps>): DreamingProps {
snippet: "Emma prefers shorter, lower-pressure check-ins.",
recallCount: 2,
dailyCount: 1,
groundedCount: 1,
totalSignalCount: 3,
lightHits: 1,
remHits: 1,
@@ -50,6 +52,7 @@ function buildProps(overrides?: Partial<DreamingProps>): DreamingProps {
snippet: "Use the Happy Together calendar for flights.",
recallCount: 3,
dailyCount: 2,
groundedCount: 0,
totalSignalCount: 5,
lightHits: 0,
remHits: 0,

4
ui/src/ui/views/dreaming.ts
View File

@@ -269,6 +269,7 @@ export type DreamingProps = {
snippet: string;
recallCount: number;
dailyCount: number;
groundedCount: number;
totalSignalCount: number;
lightHits: number;
remHits: number;
@@ -284,6 +285,7 @@ export type DreamingProps = {
snippet: string;
recallCount: number;
dailyCount: number;
groundedCount: number;
totalSignalCount: number;
lightHits: number;
remHits: number;
@@ -299,6 +301,7 @@ export type DreamingProps = {
snippet: string;
recallCount: number;
dailyCount: number;
groundedCount: number;
totalSignalCount: number;
lightHits: number;
remHits: number;
@@ -580,6 +583,7 @@ function renderScene(props: DreamingProps, idle: boolean, dreamText: string) {
? `${entry.recallCount} recall${entry.recallCount === 1 ? "" : "s"}`
: null,
entry.dailyCount > 0 ? `${entry.dailyCount} daily` : null,
entry.groundedCount > 0 ? `${entry.groundedCount} grounded` : null,
entry.phaseHitCount > 0
? `${entry.phaseHitCount} phase hit${entry.phaseHitCount === 1 ? "" : "s"}`
: null,