mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-12 09:41:11 +00:00
feat(memory): add grounded REM backfill lane (#63273)
Merged via squash.
Prepared head SHA: 4450f25485
Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com>
Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com>
Reviewed-by: @mbelinky
This commit is contained in:
Mariano
@@ -8,6 +8,7 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
- iOS: pin release versioning to an explicit CalVer in `apps/ios/version.json`, keep TestFlight iteration on the same short version until maintainers intentionally promote the next gateway version, and add the documented `pnpm ios:version:pin -- --from-gateway` workflow for release trains. (#63001) Thanks @ngutman.
|
||||
- Plugins/provider-auth: let provider manifests declare `providerAuthAliases` so provider variants can share env vars, auth profiles, config-backed auth, and API-key onboarding choices without core-specific wiring.
|
||||
- Memory/dreaming: add a grounded REM backfill lane with historical `rem-harness --path`, diary commit, and reset flows so old daily notes can be replayed safely into `DREAMS.md`. Thanks @mbelinky.
|
||||
|
||||
### Fixes
|
||||
|
||||
|
||||
@@ -4,3 +4,8 @@ export type {
|
||||
MemoryProviderStatus,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
export {
|
||||
removeBackfillDiaryEntries,
|
||||
writeBackfillDiaryEntries,
|
||||
} from "./src/dreaming-narrative.js";
|
||||
export { previewGroundedRemMarkdown } from "./src/rem-evidence.js";
|
||||
|
||||
@@ -4,7 +4,6 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { resolveMemoryRemDreamingConfig } from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import {
|
||||
colorize,
|
||||
defaultRuntime,
|
||||
@@ -31,10 +30,13 @@ import type {
|
||||
MemoryCommandOptions,
|
||||
MemoryPromoteCommandOptions,
|
||||
MemoryPromoteExplainOptions,
|
||||
MemoryRemBackfillOptions,
|
||||
MemoryRemHarnessOptions,
|
||||
MemorySearchCommandOptions,
|
||||
} from "./cli.types.js";
|
||||
import { previewRemDreaming } from "./dreaming-phases.js";
|
||||
import { previewRemDreaming, seedHistoricalDailyMemorySignals } from "./dreaming-phases.js";
|
||||
import { removeBackfillDiaryEntries, writeBackfillDiaryEntries } from "./dreaming-narrative.js";
|
||||
import { previewGroundedRemMarkdown } from "./rem-evidence.js";
|
||||
import { asRecord } from "./dreaming-shared.js";
|
||||
import { resolveShortTermPromotionDreamingConfig } from "./dreaming.js";
|
||||
import {
|
||||
@@ -114,6 +116,78 @@ function resolveMemoryPluginConfig(cfg: OpenClawConfig): Record<string, unknown>
|
||||
return asRecord(entry?.config) ?? {};
|
||||
}
|
||||
|
||||
const DAILY_MEMORY_FILE_NAME_RE = /^(\d{4}-\d{2}-\d{2})\.md$/;
|
||||
|
||||
async function listHistoricalDailyFiles(inputPath: string): Promise<string[]> {
|
||||
const resolvedPath = path.resolve(inputPath);
|
||||
const stat = await fs.stat(resolvedPath);
|
||||
if (stat.isFile()) {
|
||||
return DAILY_MEMORY_FILE_NAME_RE.test(path.basename(resolvedPath)) ? [resolvedPath] : [];
|
||||
}
|
||||
if (!stat.isDirectory()) {
|
||||
return [];
|
||||
}
|
||||
const entries = await fs.readdir(resolvedPath, { withFileTypes: true });
|
||||
return entries
|
||||
.filter((entry) => entry.isFile() && DAILY_MEMORY_FILE_NAME_RE.test(entry.name))
|
||||
.map((entry) => path.join(resolvedPath, entry.name))
|
||||
.toSorted((a, b) => path.basename(a).localeCompare(path.basename(b)));
|
||||
}
|
||||
|
||||
async function createHistoricalRemHarnessWorkspace(params: {
|
||||
inputPath: string;
|
||||
remLimit: number;
|
||||
nowMs: number;
|
||||
timezone?: string;
|
||||
}): Promise<{
|
||||
workspaceDir: string;
|
||||
sourceFiles: string[];
|
||||
workspaceSourceFiles: string[];
|
||||
importedFileCount: number;
|
||||
importedSignalCount: number;
|
||||
skippedPaths: string[];
|
||||
}> {
|
||||
const sourceFiles = await listHistoricalDailyFiles(params.inputPath);
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rem-harness-"));
|
||||
const memoryDir = path.join(workspaceDir, "memory");
|
||||
await fs.mkdir(memoryDir, { recursive: true });
|
||||
for (const filePath of sourceFiles) {
|
||||
await fs.copyFile(filePath, path.join(memoryDir, path.basename(filePath)));
|
||||
}
|
||||
const workspaceSourceFiles = sourceFiles.map((entry) => path.join(memoryDir, path.basename(entry)));
|
||||
const seeded = await seedHistoricalDailyMemorySignals({
|
||||
workspaceDir,
|
||||
filePaths: workspaceSourceFiles,
|
||||
limit: params.remLimit,
|
||||
nowMs: params.nowMs,
|
||||
timezone: params.timezone,
|
||||
});
|
||||
return {
|
||||
workspaceDir,
|
||||
sourceFiles,
|
||||
workspaceSourceFiles,
|
||||
importedFileCount: seeded.importedFileCount,
|
||||
importedSignalCount: seeded.importedSignalCount,
|
||||
skippedPaths: seeded.skippedPaths,
|
||||
};
|
||||
}
|
||||
|
||||
async function listWorkspaceDailyFiles(workspaceDir: string, limit: number): Promise<string[]> {
|
||||
const memoryDir = path.join(workspaceDir, "memory");
|
||||
try {
|
||||
const files = await listHistoricalDailyFiles(memoryDir);
|
||||
if (!Number.isFinite(limit) || limit <= 0 || files.length <= limit) {
|
||||
return files;
|
||||
}
|
||||
return files.slice(-limit);
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return [];
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function formatDreamingSummary(cfg: OpenClawConfig): string {
|
||||
const pluginConfig = resolveMemoryPluginConfig(cfg);
|
||||
const dreaming = resolveShortTermPromotionDreamingConfig({ pluginConfig, cfg });
|
||||
@@ -208,6 +282,18 @@ function formatExtraPaths(workspaceDir: string, extraPaths: string[]): string[]
|
||||
return normalizeExtraMemoryPaths(workspaceDir, extraPaths).map((entry) => shortenHomePath(entry));
|
||||
}
|
||||
|
||||
function extractIsoDayFromPath(filePath: string): string | null {
|
||||
const match = path.basename(filePath).match(DAILY_MEMORY_FILE_NAME_RE);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function groundedMarkdownToDiaryLines(markdown: string): string[] {
|
||||
return markdown
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/^##\s+/, "").trimEnd())
|
||||
.filter((line, index, lines) => !(line.length === 0 && lines[index - 1]?.length === 0));
|
||||
}
|
||||
|
||||
function matchesPromotionSelector(
|
||||
candidate: {
|
||||
key: string;
|
||||
@@ -216,15 +302,15 @@ function matchesPromotionSelector(
|
||||
},
|
||||
selector: string,
|
||||
): boolean {
|
||||
const trimmed = normalizeLowercaseStringOrEmpty(selector);
|
||||
const trimmed = selector.trim().toLowerCase();
|
||||
if (!trimmed) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
normalizeLowercaseStringOrEmpty(candidate.key) === trimmed ||
|
||||
normalizeLowercaseStringOrEmpty(candidate.key).includes(trimmed) ||
|
||||
normalizeLowercaseStringOrEmpty(candidate.path).includes(trimmed) ||
|
||||
normalizeLowercaseStringOrEmpty(candidate.snippet).includes(trimmed)
|
||||
candidate.key.toLowerCase() === trimmed ||
|
||||
candidate.key.toLowerCase().includes(trimmed) ||
|
||||
candidate.path.toLowerCase().includes(trimmed) ||
|
||||
candidate.snippet.toLowerCase().includes(trimmed)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1250,13 +1336,13 @@ export async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) {
|
||||
purpose: "status",
|
||||
run: async (manager) => {
|
||||
const status = manager.status();
|
||||
const workspaceDir = status.workspaceDir?.trim();
|
||||
const managerWorkspaceDir = status.workspaceDir?.trim();
|
||||
const pluginConfig = resolveMemoryPluginConfig(cfg);
|
||||
const deep = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig,
|
||||
cfg,
|
||||
});
|
||||
if (!workspaceDir) {
|
||||
if (!managerWorkspaceDir && !opts.path) {
|
||||
defaultRuntime.error("Memory rem-harness requires a resolvable workspace directory.");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
@@ -1266,69 +1352,297 @@ export async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) {
|
||||
cfg,
|
||||
});
|
||||
const nowMs = Date.now();
|
||||
const cutoffMs = nowMs - Math.max(0, remConfig.lookbackDays) * 24 * 60 * 60 * 1000;
|
||||
const recallEntries = (await readShortTermRecallEntries({ workspaceDir, nowMs })).filter(
|
||||
(entry) => Date.parse(entry.lastRecalledAt) >= cutoffMs,
|
||||
);
|
||||
const remPreview = previewRemDreaming({
|
||||
entries: recallEntries,
|
||||
limit: remConfig.limit,
|
||||
minPatternStrength: remConfig.minPatternStrength,
|
||||
});
|
||||
const deepCandidates = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
includePromoted: Boolean(opts.includePromoted),
|
||||
recencyHalfLifeDays: deep.recencyHalfLifeDays,
|
||||
maxAgeDays: deep.maxAgeDays,
|
||||
});
|
||||
|
||||
if (opts.json) {
|
||||
defaultRuntime.writeJson({
|
||||
workspaceDir,
|
||||
remConfig,
|
||||
deepConfig: {
|
||||
minScore: deep.minScore,
|
||||
minRecallCount: deep.minRecallCount,
|
||||
minUniqueQueries: deep.minUniqueQueries,
|
||||
recencyHalfLifeDays: deep.recencyHalfLifeDays,
|
||||
maxAgeDays: deep.maxAgeDays ?? null,
|
||||
},
|
||||
rem: remPreview,
|
||||
deep: {
|
||||
candidateCount: deepCandidates.length,
|
||||
candidates: deepCandidates,
|
||||
},
|
||||
let workspaceDir = managerWorkspaceDir ?? "";
|
||||
let sourceFiles: string[] = [];
|
||||
let groundedInputPaths: string[] = [];
|
||||
let importedFileCount = 0;
|
||||
let importedSignalCount = 0;
|
||||
let skippedPaths: string[] = [];
|
||||
let cleanupWorkspaceDir: string | null = null;
|
||||
if (opts.path) {
|
||||
const historical = await createHistoricalRemHarnessWorkspace({
|
||||
inputPath: opts.path,
|
||||
remLimit: remConfig.limit,
|
||||
nowMs,
|
||||
timezone: remConfig.timezone,
|
||||
});
|
||||
workspaceDir = historical.workspaceDir;
|
||||
cleanupWorkspaceDir = historical.workspaceDir;
|
||||
sourceFiles = historical.sourceFiles;
|
||||
groundedInputPaths = historical.workspaceSourceFiles;
|
||||
importedFileCount = historical.importedFileCount;
|
||||
importedSignalCount = historical.importedSignalCount;
|
||||
skippedPaths = historical.skippedPaths;
|
||||
if (sourceFiles.length === 0) {
|
||||
await fs.rm(historical.workspaceDir, { recursive: true, force: true });
|
||||
defaultRuntime.error(
|
||||
`Memory rem-harness found no YYYY-MM-DD.md files at ${shortenHomePath(path.resolve(opts.path))}.`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!workspaceDir) {
|
||||
defaultRuntime.error("Memory rem-harness requires a resolvable workspace directory.");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (groundedInputPaths.length === 0 && opts.grounded) {
|
||||
groundedInputPaths = await listWorkspaceDailyFiles(workspaceDir, remConfig.limit);
|
||||
}
|
||||
const cutoffMs = nowMs - Math.max(0, remConfig.lookbackDays) * 24 * 60 * 60 * 1000;
|
||||
const recallEntries = (await readShortTermRecallEntries({ workspaceDir, nowMs })).filter(
|
||||
(entry) => Date.parse(entry.lastRecalledAt) >= cutoffMs,
|
||||
);
|
||||
const remPreview = previewRemDreaming({
|
||||
entries: recallEntries,
|
||||
limit: remConfig.limit,
|
||||
minPatternStrength: remConfig.minPatternStrength,
|
||||
});
|
||||
const groundedPreview =
|
||||
opts.grounded && groundedInputPaths.length > 0
|
||||
? await previewGroundedRemMarkdown({
|
||||
workspaceDir,
|
||||
inputPaths: groundedInputPaths,
|
||||
})
|
||||
: null;
|
||||
const deepCandidates = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
includePromoted: Boolean(opts.includePromoted),
|
||||
recencyHalfLifeDays: deep.recencyHalfLifeDays,
|
||||
maxAgeDays: deep.maxAgeDays,
|
||||
});
|
||||
|
||||
const rich = isRich();
|
||||
const lines = [
|
||||
`${colorize(rich, theme.heading, "REM Harness")} ${colorize(rich, theme.muted, `(${agentId})`)}`,
|
||||
colorize(rich, theme.muted, `workspace=${shortenHomePath(workspaceDir)}`),
|
||||
colorize(
|
||||
rich,
|
||||
theme.muted,
|
||||
`recentRecallEntries=${recallEntries.length} deepCandidates=${deepCandidates.length}`,
|
||||
),
|
||||
"",
|
||||
colorize(rich, theme.heading, "REM Preview"),
|
||||
...remPreview.bodyLines,
|
||||
"",
|
||||
colorize(rich, theme.heading, "Deep Candidates"),
|
||||
...(deepCandidates.length > 0
|
||||
? deepCandidates
|
||||
.slice(0, 10)
|
||||
.map(
|
||||
(candidate) =>
|
||||
`${candidate.score.toFixed(3)} ${candidate.snippet} [${shortenHomePath(candidate.path)}:${candidate.startLine}-${candidate.endLine}]`,
|
||||
)
|
||||
: ["- No deep candidates."]),
|
||||
];
|
||||
defaultRuntime.log(lines.join("\n"));
|
||||
if (opts.json) {
|
||||
defaultRuntime.writeJson({
|
||||
workspaceDir,
|
||||
sourcePath: opts.path ? path.resolve(opts.path) : null,
|
||||
sourceFiles,
|
||||
historicalImport:
|
||||
opts.path
|
||||
? {
|
||||
importedFileCount,
|
||||
importedSignalCount,
|
||||
skippedPaths,
|
||||
}
|
||||
: null,
|
||||
remConfig,
|
||||
deepConfig: {
|
||||
minScore: deep.minScore,
|
||||
minRecallCount: deep.minRecallCount,
|
||||
minUniqueQueries: deep.minUniqueQueries,
|
||||
recencyHalfLifeDays: deep.recencyHalfLifeDays,
|
||||
maxAgeDays: deep.maxAgeDays ?? null,
|
||||
},
|
||||
rem: remPreview,
|
||||
grounded: groundedPreview,
|
||||
deep: {
|
||||
candidateCount: deepCandidates.length,
|
||||
candidates: deepCandidates,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const rich = isRich();
|
||||
const lines = [
|
||||
`${colorize(rich, theme.heading, "REM Harness")} ${colorize(rich, theme.muted, `(${agentId})`)}`,
|
||||
colorize(rich, theme.muted, `workspace=${shortenHomePath(workspaceDir)}`),
|
||||
...(opts.path
|
||||
? [
|
||||
colorize(
|
||||
rich,
|
||||
theme.muted,
|
||||
`sourcePath=${shortenHomePath(path.resolve(opts.path))}`,
|
||||
),
|
||||
colorize(
|
||||
rich,
|
||||
theme.muted,
|
||||
`historicalFiles=${sourceFiles.length} importedFiles=${importedFileCount} importedSignals=${importedSignalCount}`,
|
||||
),
|
||||
...(skippedPaths.length > 0
|
||||
? [
|
||||
colorize(
|
||||
rich,
|
||||
theme.warn,
|
||||
`skipped=${skippedPaths.map((entry) => shortenHomePath(entry)).join(", ")}`,
|
||||
),
|
||||
]
|
||||
: []),
|
||||
]
|
||||
: []),
|
||||
...(opts.grounded
|
||||
? [
|
||||
colorize(
|
||||
rich,
|
||||
theme.muted,
|
||||
`groundedInputs=${groundedInputPaths.length > 0 ? groundedInputPaths.map((entry) => shortenHomePath(entry)).join(", ") : "none"}`,
|
||||
),
|
||||
]
|
||||
: []),
|
||||
colorize(
|
||||
rich,
|
||||
theme.muted,
|
||||
`recentRecallEntries=${recallEntries.length} deepCandidates=${deepCandidates.length}`,
|
||||
),
|
||||
"",
|
||||
colorize(rich, theme.heading, "REM Preview"),
|
||||
...remPreview.bodyLines,
|
||||
...(groundedPreview
|
||||
? [
|
||||
"",
|
||||
colorize(rich, theme.heading, "Grounded REM"),
|
||||
...groundedPreview.files.flatMap((file) => [
|
||||
colorize(rich, theme.label, file.path),
|
||||
file.renderedMarkdown,
|
||||
"",
|
||||
]),
|
||||
]
|
||||
: []),
|
||||
"",
|
||||
colorize(rich, theme.heading, "Deep Candidates"),
|
||||
...(deepCandidates.length > 0
|
||||
? deepCandidates
|
||||
.slice(0, 10)
|
||||
.map(
|
||||
(candidate) =>
|
||||
`${candidate.score.toFixed(3)} ${candidate.snippet} [${shortenHomePath(candidate.path)}:${candidate.startLine}-${candidate.endLine}]`,
|
||||
)
|
||||
: ["- No deep candidates."]),
|
||||
];
|
||||
defaultRuntime.log(lines.join("\n"));
|
||||
} finally {
|
||||
if (cleanupWorkspaceDir) {
|
||||
await fs.rm(cleanupWorkspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
|
||||
const { config: cfg, diagnostics } = await loadMemoryCommandConfig("memory rem-backfill");
|
||||
emitMemorySecretResolveDiagnostics(diagnostics, { json: Boolean(opts.json) });
|
||||
const agentId = resolveAgent(cfg, opts.agent);
|
||||
|
||||
await withMemoryManagerForAgent({
|
||||
cfg,
|
||||
agentId,
|
||||
purpose: "status",
|
||||
run: async (manager) => {
|
||||
const status = manager.status();
|
||||
const workspaceDir = status.workspaceDir?.trim();
|
||||
const pluginConfig = resolveMemoryPluginConfig(cfg);
|
||||
const remConfig = resolveMemoryRemDreamingConfig({
|
||||
pluginConfig,
|
||||
cfg,
|
||||
});
|
||||
if (!workspaceDir) {
|
||||
defaultRuntime.error("Memory rem-backfill requires a resolvable workspace directory.");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (opts.rollback) {
|
||||
const removed = await removeBackfillDiaryEntries({ workspaceDir });
|
||||
if (opts.json) {
|
||||
defaultRuntime.writeJson({
|
||||
workspaceDir,
|
||||
rollback: true,
|
||||
dreamsPath: removed.dreamsPath,
|
||||
removedEntries: removed.removed,
|
||||
});
|
||||
return;
|
||||
}
|
||||
defaultRuntime.log(
|
||||
[
|
||||
`${colorize(isRich(), theme.heading, "REM Backfill")} ${colorize(isRich(), theme.muted, "(rollback)")}`,
|
||||
colorize(isRich(), theme.muted, `workspace=${shortenHomePath(workspaceDir)}`),
|
||||
colorize(isRich(), theme.muted, `dreamsPath=${shortenHomePath(removed.dreamsPath)}`),
|
||||
colorize(isRich(), theme.muted, `removedEntries=${removed.removed}`),
|
||||
].join("\n"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!opts.path) {
|
||||
defaultRuntime.error("Memory rem-backfill requires --path <file-or-dir> unless using --rollback.");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const scratchDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rem-backfill-"));
|
||||
try {
|
||||
const sourceFiles = await listHistoricalDailyFiles(opts.path);
|
||||
if (sourceFiles.length === 0) {
|
||||
defaultRuntime.error(
|
||||
`Memory rem-backfill found no YYYY-MM-DD.md files at ${shortenHomePath(path.resolve(opts.path))}.`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const scratchMemoryDir = path.join(scratchDir, "memory");
|
||||
await fs.mkdir(scratchMemoryDir, { recursive: true });
|
||||
const workspaceSourceFiles: string[] = [];
|
||||
for (const filePath of sourceFiles) {
|
||||
const dst = path.join(scratchMemoryDir, path.basename(filePath));
|
||||
await fs.copyFile(filePath, dst);
|
||||
workspaceSourceFiles.push(dst);
|
||||
}
|
||||
const grounded = await previewGroundedRemMarkdown({
|
||||
workspaceDir: scratchDir,
|
||||
inputPaths: workspaceSourceFiles,
|
||||
});
|
||||
const entries = grounded.files
|
||||
.map((file) => {
|
||||
const isoDay = extractIsoDayFromPath(file.path);
|
||||
if (!isoDay) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
isoDay,
|
||||
sourcePath: file.path,
|
||||
bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown),
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
|
||||
|
||||
const written = await writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
entries,
|
||||
timezone: remConfig.timezone,
|
||||
});
|
||||
|
||||
if (opts.json) {
|
||||
defaultRuntime.writeJson({
|
||||
workspaceDir,
|
||||
sourcePath: path.resolve(opts.path),
|
||||
sourceFiles,
|
||||
groundedFiles: grounded.scannedFiles,
|
||||
writtenEntries: written.written,
|
||||
replacedEntries: written.replaced,
|
||||
dreamsPath: written.dreamsPath,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const rich = isRich();
|
||||
defaultRuntime.log(
|
||||
[
|
||||
`${colorize(rich, theme.heading, "REM Backfill")} ${colorize(rich, theme.muted, `(${agentId})`)}`,
|
||||
colorize(rich, theme.muted, `workspace=${shortenHomePath(workspaceDir)}`),
|
||||
colorize(rich, theme.muted, `sourcePath=${shortenHomePath(path.resolve(opts.path))}`),
|
||||
colorize(rich, theme.muted, `historicalFiles=${sourceFiles.length} writtenEntries=${written.written} replacedEntries=${written.replaced}`),
|
||||
colorize(rich, theme.muted, `dreamsPath=${shortenHomePath(written.dreamsPath)}`),
|
||||
].join("\n"),
|
||||
);
|
||||
} finally {
|
||||
await fs.rm(scratchDir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -948,6 +948,177 @@ describe("memory cli", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("previews rem harness output from a historical daily file path", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const historyDir = path.join(workspaceDir, "history");
|
||||
await fs.mkdir(historyDir, { recursive: true });
|
||||
const historyPath = path.join(historyDir, "2025-01-01.md");
|
||||
await fs.writeFile(
|
||||
historyPath,
|
||||
[
|
||||
"# Preferences Learned",
|
||||
'- Always use "Happy Together" calendar for flights and reservations.',
|
||||
"- Calendar ID: udolnrooml2f2ha8jaio24v1r8@group.calendar.google.com",
|
||||
].join("\n") + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const close = vi.fn(async () => {});
|
||||
mockManager({
|
||||
status: () => makeMemoryStatus({ workspaceDir }),
|
||||
close,
|
||||
});
|
||||
|
||||
const writeJson = spyRuntimeJson(defaultRuntime);
|
||||
await runMemoryCli(["rem-harness", "--json", "--path", historyPath]);
|
||||
|
||||
const payload = firstWrittenJsonArg<{
|
||||
sourcePath?: string | null;
|
||||
sourceFiles?: string[];
|
||||
historicalImport?: { importedFileCount?: number; importedSignalCount?: number } | null;
|
||||
rem?: { candidateTruths?: Array<{ snippet?: string }> };
|
||||
deep?: { candidates?: Array<{ snippet?: string; path?: string }> };
|
||||
}>(writeJson);
|
||||
expect(payload?.sourcePath).toBe(historyPath);
|
||||
expect(payload?.sourceFiles).toEqual([historyPath]);
|
||||
expect(payload?.historicalImport?.importedFileCount).toBe(1);
|
||||
expect(payload?.historicalImport?.importedSignalCount).toBeGreaterThan(0);
|
||||
expect(Array.isArray(payload?.rem?.candidateTruths)).toBe(true);
|
||||
expect(payload?.deep?.candidates?.[0]?.snippet).toContain("Happy Together");
|
||||
expect(payload?.deep?.candidates?.[0]?.path).toBe("memory/2025-01-01.md");
|
||||
expect(close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("previews grounded rem output from a historical daily file path", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const historyDir = path.join(workspaceDir, "history");
|
||||
await fs.mkdir(historyDir, { recursive: true });
|
||||
const historyPath = path.join(historyDir, "2025-01-01.md");
|
||||
await fs.writeFile(
|
||||
historyPath,
|
||||
[
|
||||
"## Preferences Learned",
|
||||
'- Always use "Happy Together" calendar for flights and reservations.',
|
||||
"- Calendar ID: udolnrooml2f2ha8jaio24v1r8@group.calendar.google.com",
|
||||
"",
|
||||
"## Setup",
|
||||
"- Set up Gmail access via gog.",
|
||||
].join("\n") + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const close = vi.fn(async () => {});
|
||||
mockManager({
|
||||
status: () => makeMemoryStatus({ workspaceDir }),
|
||||
close,
|
||||
});
|
||||
|
||||
const writeJson = spyRuntimeJson(defaultRuntime);
|
||||
await runMemoryCli(["rem-harness", "--json", "--grounded", "--path", historyPath]);
|
||||
|
||||
const payload = firstWrittenJsonArg<{
|
||||
grounded?: {
|
||||
scannedFiles?: number;
|
||||
files?: Array<{
|
||||
path?: string;
|
||||
renderedMarkdown?: string;
|
||||
memoryImplications?: Array<{ text?: string }>;
|
||||
}>;
|
||||
} | null;
|
||||
}>(writeJson);
|
||||
expect(payload?.grounded?.scannedFiles).toBe(1);
|
||||
expect(payload?.grounded?.files?.[0]?.path).toBe("memory/2025-01-01.md");
|
||||
expect(payload?.grounded?.files?.[0]?.renderedMarkdown).toContain("## What Happened");
|
||||
expect(payload?.grounded?.files?.[0]?.renderedMarkdown).toContain("## Reflections");
|
||||
expect(payload?.grounded?.files?.[0]?.renderedMarkdown).toContain(
|
||||
"## Possible Lasting Updates",
|
||||
);
|
||||
expect(payload?.grounded?.files?.[0]?.memoryImplications?.[0]?.text).toContain(
|
||||
'Always use "Happy Together" calendar for flights and reservations',
|
||||
);
|
||||
expect(close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("writes grounded rem backfill entries into DREAMS.md", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const historyDir = path.join(workspaceDir, "history");
|
||||
await fs.mkdir(historyDir, { recursive: true });
|
||||
const historyPath = path.join(historyDir, "2025-01-01.md");
|
||||
await fs.writeFile(
|
||||
historyPath,
|
||||
[
|
||||
"## Preferences Learned",
|
||||
'- Always use "Happy Together" calendar for flights and reservations.',
|
||||
"- Calendar ID: udolnrooml2f2ha8jaio24v1r8@group.calendar.google.com",
|
||||
].join("\n") + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const close = vi.fn(async () => {});
|
||||
mockManager({
|
||||
status: () => makeMemoryStatus({ workspaceDir }),
|
||||
close,
|
||||
});
|
||||
|
||||
await runMemoryCli(["rem-backfill", "--path", historyPath]);
|
||||
|
||||
const dreams = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(dreams).toContain("openclaw:dreaming:backfill-entry");
|
||||
expect(dreams).toContain("January 1, 2025");
|
||||
expect(dreams).toContain("What Happened");
|
||||
expect(dreams).toContain("Possible Lasting Updates");
|
||||
expect(dreams).toContain("Happy Together");
|
||||
expect(close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("rolls back grounded rem backfill entries from DREAMS.md", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(
|
||||
dreamsPath,
|
||||
[
|
||||
"# Dream Diary",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:diary:start -->",
|
||||
"---",
|
||||
"",
|
||||
"*April 5, 2026, 3:00 AM*",
|
||||
"",
|
||||
"Keep this normal dream.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"*January 1, 2025*",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:backfill-entry day=2025-01-01 source=memory/2025-01-01.md -->",
|
||||
"",
|
||||
"What Happened",
|
||||
"1. Remove this entry.",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:diary:end -->",
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const close = vi.fn(async () => {});
|
||||
mockManager({
|
||||
status: () => makeMemoryStatus({ workspaceDir }),
|
||||
close,
|
||||
});
|
||||
|
||||
await runMemoryCli(["rem-backfill", "--rollback"]);
|
||||
|
||||
const dreams = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(dreams).toContain("Keep this normal dream.");
|
||||
expect(dreams).not.toContain("Remove this entry.");
|
||||
expect(close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("applies top promote candidates into MEMORY.md", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
await writeDailyMemoryNote(workspaceDir, "2026-04-01", [
|
||||
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
MemoryCommandOptions,
|
||||
MemoryPromoteCommandOptions,
|
||||
MemoryPromoteExplainOptions,
|
||||
MemoryRemBackfillOptions,
|
||||
MemoryRemHarnessOptions,
|
||||
MemorySearchCommandOptions,
|
||||
} from "./cli.types.js";
|
||||
@@ -59,6 +60,11 @@ async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) {
|
||||
await runtime.runMemoryRemHarness(opts);
|
||||
}
|
||||
|
||||
async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryRemBackfill(opts);
|
||||
}
|
||||
|
||||
export function registerMemoryCli(program: Command) {
|
||||
const memory = program
|
||||
.command("memory")
|
||||
@@ -95,6 +101,10 @@ export function registerMemoryCli(program: Command) {
|
||||
"openclaw memory rem-harness --json",
|
||||
"Preview REM reflections, candidate truths, and deep promotion output.",
|
||||
],
|
||||
[
|
||||
"openclaw memory rem-backfill --path ./memory",
|
||||
"Write grounded historical REM entries into DREAMS.md for UI review.",
|
||||
],
|
||||
["openclaw memory status --json", "Output machine-readable JSON (good for scripts)."],
|
||||
])}\n\n${theme.muted("Docs:")} ${formatDocsLink("/cli/memory", "docs.openclaw.ai/cli/memory")}\n`,
|
||||
);
|
||||
@@ -177,9 +187,22 @@ export function registerMemoryCli(program: Command) {
|
||||
.command("rem-harness")
|
||||
.description("Preview REM reflections, candidate truths, and deep promotions without writing")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--path <file-or-dir>", "Seed the harness from historical daily memory file(s)")
|
||||
.option("--grounded", "Also render a grounded day-level REM preview")
|
||||
.option("--include-promoted", "Include already promoted deep candidates", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: MemoryRemHarnessOptions) => {
|
||||
await runMemoryRemHarness(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("rem-backfill")
|
||||
.description("Write grounded historical REM summaries into DREAMS.md for UI review")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--path <file-or-dir>", "Historical daily memory file(s) or directory")
|
||||
.option("--rollback", "Remove previously written grounded REM backfill entries", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: MemoryRemBackfillOptions) => {
|
||||
await runMemoryRemBackfill(opts);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -29,4 +29,11 @@ export type MemoryPromoteExplainOptions = MemoryCommandOptions & {
|
||||
|
||||
export type MemoryRemHarnessOptions = MemoryCommandOptions & {
|
||||
includePromoted?: boolean;
|
||||
path?: string;
|
||||
grounded?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryRemBackfillOptions = MemoryCommandOptions & {
|
||||
path?: string;
|
||||
rollback?: boolean;
|
||||
};
|
||||
|
||||
@@ -3,12 +3,16 @@ import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
appendNarrativeEntry,
|
||||
buildBackfillDiaryEntry,
|
||||
buildDiaryEntry,
|
||||
buildNarrativePrompt,
|
||||
extractNarrativeText,
|
||||
formatNarrativeDate,
|
||||
formatBackfillDiaryDate,
|
||||
generateAndAppendDreamNarrative,
|
||||
removeBackfillDiaryEntries,
|
||||
type NarrativePhaseData,
|
||||
writeBackfillDiaryEntries,
|
||||
} from "./dreaming-narrative.js";
|
||||
import { createMemoryCoreTestHarness } from "./test-helpers.js";
|
||||
|
||||
@@ -117,6 +121,88 @@ describe("buildDiaryEntry", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("backfill diary entries", () => {
|
||||
it("formats a backfill date without time", () => {
|
||||
expect(formatBackfillDiaryDate("2026-01-01", "UTC")).toBe("January 1, 2026");
|
||||
});
|
||||
|
||||
it("builds a marked backfill diary entry", () => {
|
||||
const entry = buildBackfillDiaryEntry({
|
||||
isoDay: "2026-01-01",
|
||||
sourcePath: "memory/2026-01-01.md",
|
||||
bodyLines: ["What Happened", "1. A durable preference appeared."],
|
||||
timezone: "UTC",
|
||||
});
|
||||
expect(entry).toContain("*January 1, 2026*");
|
||||
expect(entry).toContain("openclaw:dreaming:backfill-entry");
|
||||
expect(entry).toContain("What Happened");
|
||||
});
|
||||
|
||||
it("writes and replaces backfill diary entries", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-backfill-");
|
||||
const first = await writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
timezone: "UTC",
|
||||
entries: [
|
||||
{
|
||||
isoDay: "2026-01-01",
|
||||
sourcePath: "memory/2026-01-01.md",
|
||||
bodyLines: ["What Happened", "1. First pass."],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(first.written).toBe(1);
|
||||
expect(first.replaced).toBe(0);
|
||||
|
||||
const second = await writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
timezone: "UTC",
|
||||
entries: [
|
||||
{
|
||||
isoDay: "2026-01-02",
|
||||
sourcePath: "memory/2026-01-02.md",
|
||||
bodyLines: ["Reflections", "1. Second pass."],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(second.written).toBe(1);
|
||||
expect(second.replaced).toBe(1);
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).not.toContain("First pass.");
|
||||
expect(content).toContain("Second pass.");
|
||||
expect(content.match(/openclaw:dreaming:backfill-entry/g)?.length).toBe(1);
|
||||
});
|
||||
|
||||
it("removes only backfill diary entries", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-backfill-");
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Keep this real dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
await writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
timezone: "UTC",
|
||||
entries: [
|
||||
{
|
||||
isoDay: "2026-01-01",
|
||||
sourcePath: "memory/2026-01-01.md",
|
||||
bodyLines: ["What Happened", "1. Remove this backfill."],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const removed = await removeBackfillDiaryEntries({ workspaceDir });
|
||||
expect(removed.removed).toBe(1);
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("Keep this real dream.");
|
||||
expect(content).not.toContain("Remove this backfill.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendNarrativeEntry", () => {
|
||||
it("creates DREAMS.md with diary header on fresh workspace", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
|
||||
@@ -70,6 +70,7 @@ const NARRATIVE_TIMEOUT_MS = 60_000;
|
||||
const DREAMS_FILENAMES = ["DREAMS.md", "dreams.md"] as const;
|
||||
const DIARY_START_MARKER = "<!-- openclaw:dreaming:diary:start -->";
|
||||
const DIARY_END_MARKER = "<!-- openclaw:dreaming:diary:end -->";
|
||||
const BACKFILL_ENTRY_MARKER = "openclaw:dreaming:backfill-entry";
|
||||
|
||||
// ── Prompt building ────────────────────────────────────────────────────
|
||||
|
||||
@@ -167,6 +168,157 @@ async function resolveDreamsPath(workspaceDir: string): Promise<string> {
|
||||
return path.join(workspaceDir, DREAMS_FILENAMES[0]);
|
||||
}
|
||||
|
||||
async function readDreamsFile(dreamsPath: string): Promise<string> {
|
||||
try {
|
||||
return await fs.readFile(dreamsPath, "utf-8");
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function ensureDiarySection(existing: string): string {
|
||||
if (existing.includes(DIARY_START_MARKER) && existing.includes(DIARY_END_MARKER)) {
|
||||
return existing;
|
||||
}
|
||||
const diarySection = `# Dream Diary\n\n${DIARY_START_MARKER}\n${DIARY_END_MARKER}\n`;
|
||||
if (existing.trim().length === 0) {
|
||||
return diarySection;
|
||||
}
|
||||
return diarySection + "\n" + existing;
|
||||
}
|
||||
|
||||
function replaceDiaryContent(existing: string, diaryContent: string): string {
|
||||
const ensured = ensureDiarySection(existing);
|
||||
const startIdx = ensured.indexOf(DIARY_START_MARKER);
|
||||
const endIdx = ensured.indexOf(DIARY_END_MARKER);
|
||||
if (startIdx < 0 || endIdx < 0 || endIdx < startIdx) {
|
||||
return ensured;
|
||||
}
|
||||
const before = ensured.slice(0, startIdx + DIARY_START_MARKER.length);
|
||||
const after = ensured.slice(endIdx);
|
||||
const normalized = diaryContent.trim().length > 0 ? `\n${diaryContent.trim()}\n` : "\n";
|
||||
return before + normalized + after;
|
||||
}
|
||||
|
||||
function splitDiaryBlocks(diaryContent: string): string[] {
|
||||
return diaryContent
|
||||
.split(/\n---\n/)
|
||||
.map((block) => block.trim())
|
||||
.filter((block) => block.length > 0);
|
||||
}
|
||||
|
||||
function joinDiaryBlocks(blocks: string[]): string {
|
||||
if (blocks.length === 0) {
|
||||
return "";
|
||||
}
|
||||
return blocks.map((block) => `---\n\n${block.trim()}\n`).join("\n");
|
||||
}
|
||||
|
||||
function stripBackfillDiaryBlocks(existing: string): { updated: string; removed: number } {
|
||||
const ensured = ensureDiarySection(existing);
|
||||
const startIdx = ensured.indexOf(DIARY_START_MARKER);
|
||||
const endIdx = ensured.indexOf(DIARY_END_MARKER);
|
||||
if (startIdx < 0 || endIdx < 0 || endIdx < startIdx) {
|
||||
return { updated: ensured, removed: 0 };
|
||||
}
|
||||
const inner = ensured.slice(startIdx + DIARY_START_MARKER.length, endIdx);
|
||||
const kept: string[] = [];
|
||||
let removed = 0;
|
||||
for (const block of splitDiaryBlocks(inner)) {
|
||||
if (block.includes(BACKFILL_ENTRY_MARKER)) {
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
kept.push(block);
|
||||
}
|
||||
return {
|
||||
updated: replaceDiaryContent(ensured, joinDiaryBlocks(kept)),
|
||||
removed,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatBackfillDiaryDate(isoDay: string, timezone?: string): string {
|
||||
const opts: Intl.DateTimeFormatOptions = {
|
||||
timeZone: timezone ?? "UTC",
|
||||
year: "numeric",
|
||||
month: "long",
|
||||
day: "numeric",
|
||||
};
|
||||
const epochMs = Date.parse(`${isoDay}T12:00:00Z`);
|
||||
return new Intl.DateTimeFormat("en-US", opts).format(new Date(epochMs));
|
||||
}
|
||||
|
||||
export function buildBackfillDiaryEntry(params: {
|
||||
isoDay: string;
|
||||
bodyLines: string[];
|
||||
sourcePath?: string;
|
||||
timezone?: string;
|
||||
}): string {
|
||||
const dateStr = formatBackfillDiaryDate(params.isoDay, params.timezone);
|
||||
const marker = `<!-- ${BACKFILL_ENTRY_MARKER} day=${params.isoDay}${params.sourcePath ? ` source=${params.sourcePath}` : ""} -->`;
|
||||
const body = params.bodyLines.map((line) => line.trimEnd()).join("\n").trim();
|
||||
return [`*${dateStr}*`, marker, body].filter((part) => part.length > 0).join("\n\n");
|
||||
}
|
||||
|
||||
export async function writeBackfillDiaryEntries(params: {
|
||||
workspaceDir: string;
|
||||
entries: Array<{
|
||||
isoDay: string;
|
||||
bodyLines: string[];
|
||||
sourcePath?: string;
|
||||
}>;
|
||||
timezone?: string;
|
||||
}): Promise<{ dreamsPath: string; written: number; replaced: number }> {
|
||||
const dreamsPath = await resolveDreamsPath(params.workspaceDir);
|
||||
await fs.mkdir(path.dirname(dreamsPath), { recursive: true });
|
||||
const existing = await readDreamsFile(dreamsPath);
|
||||
const stripped = stripBackfillDiaryBlocks(existing);
|
||||
const startIdx = stripped.updated.indexOf(DIARY_START_MARKER);
|
||||
const endIdx = stripped.updated.indexOf(DIARY_END_MARKER);
|
||||
const inner =
|
||||
startIdx >= 0 && endIdx > startIdx
|
||||
? stripped.updated.slice(startIdx + DIARY_START_MARKER.length, endIdx)
|
||||
: "";
|
||||
const preservedBlocks = splitDiaryBlocks(inner);
|
||||
const nextBlocks = [
|
||||
...preservedBlocks,
|
||||
...params.entries.map((entry) =>
|
||||
buildBackfillDiaryEntry({
|
||||
isoDay: entry.isoDay,
|
||||
bodyLines: entry.bodyLines,
|
||||
sourcePath: entry.sourcePath,
|
||||
timezone: params.timezone,
|
||||
}),
|
||||
),
|
||||
];
|
||||
const updated = replaceDiaryContent(stripped.updated, joinDiaryBlocks(nextBlocks));
|
||||
await fs.writeFile(dreamsPath, updated, "utf-8");
|
||||
return {
|
||||
dreamsPath,
|
||||
written: params.entries.length,
|
||||
replaced: stripped.removed,
|
||||
};
|
||||
}
|
||||
|
||||
export async function removeBackfillDiaryEntries(params: {
|
||||
workspaceDir: string;
|
||||
}): Promise<{ dreamsPath: string; removed: number }> {
|
||||
const dreamsPath = await resolveDreamsPath(params.workspaceDir);
|
||||
const existing = await readDreamsFile(dreamsPath);
|
||||
const stripped = stripBackfillDiaryBlocks(existing);
|
||||
if (stripped.removed > 0 || existing.length > 0) {
|
||||
await fs.mkdir(path.dirname(dreamsPath), { recursive: true });
|
||||
await fs.writeFile(dreamsPath, stripped.updated, "utf-8");
|
||||
}
|
||||
return {
|
||||
dreamsPath,
|
||||
removed: stripped.removed,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildDiaryEntry(narrative: string, dateStr: string): string {
|
||||
return `\n---\n\n*${dateStr}*\n\n${narrative}\n`;
|
||||
}
|
||||
|
||||
@@ -18,18 +18,9 @@ import {
|
||||
type MemoryLightDreamingConfig,
|
||||
type MemoryRemDreamingConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import {
|
||||
lowercasePreservingWhitespace,
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
} from "openclaw/plugin-sdk/text-runtime";
|
||||
import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js";
|
||||
import { generateAndAppendDreamNarrative, type NarrativePhaseData } from "./dreaming-narrative.js";
|
||||
import {
|
||||
asRecord,
|
||||
formatErrorMessage,
|
||||
includesSystemEventToken,
|
||||
normalizeTrimmedString,
|
||||
} from "./dreaming-shared.js";
|
||||
import { asRecord, formatErrorMessage, normalizeTrimmedString } from "./dreaming-shared.js";
|
||||
import {
|
||||
readShortTermRecallEntries,
|
||||
recordDreamingPhaseSignals,
|
||||
@@ -131,7 +122,7 @@ function isGenericDailyHeading(heading: string): boolean {
|
||||
if (!normalized) {
|
||||
return true;
|
||||
}
|
||||
const lower = normalizeLowercaseStringOrEmpty(normalized);
|
||||
const lower = normalized.toLowerCase();
|
||||
if (lower === "today" || lower === "yesterday" || lower === "tomorrow") {
|
||||
return true;
|
||||
}
|
||||
@@ -428,7 +419,7 @@ type SessionIngestionCollectionResult = {
|
||||
|
||||
function normalizeWorkspaceKey(workspaceDir: string): string {
|
||||
const resolved = path.resolve(workspaceDir).replace(/\\/g, "/");
|
||||
return process.platform === "win32" ? lowercasePreservingWhitespace(resolved) : resolved;
|
||||
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
||||
}
|
||||
|
||||
function resolveSessionIngestionStatePath(workspaceDir: string): string {
|
||||
@@ -1100,13 +1091,117 @@ async function ingestDailyMemorySignals(params: {
|
||||
}
|
||||
}
|
||||
|
||||
export async function seedHistoricalDailyMemorySignals(params: {
|
||||
workspaceDir: string;
|
||||
filePaths: string[];
|
||||
limit: number;
|
||||
nowMs: number;
|
||||
timezone?: string;
|
||||
}): Promise<{
|
||||
importedFileCount: number;
|
||||
importedSignalCount: number;
|
||||
skippedPaths: string[];
|
||||
}> {
|
||||
const normalizedPaths = [...new Set(params.filePaths.map((entry) => entry.trim()).filter(Boolean))];
|
||||
if (normalizedPaths.length === 0) {
|
||||
return {
|
||||
importedFileCount: 0,
|
||||
importedSignalCount: 0,
|
||||
skippedPaths: [],
|
||||
};
|
||||
}
|
||||
|
||||
const resolved = normalizedPaths
|
||||
.map((filePath) => {
|
||||
const fileName = path.basename(filePath);
|
||||
const match = fileName.match(DAILY_MEMORY_FILENAME_RE);
|
||||
if (!match) {
|
||||
return { filePath, day: null as string | null };
|
||||
}
|
||||
return { filePath, day: match[1] ?? null };
|
||||
})
|
||||
.toSorted((a, b) => {
|
||||
if (a.day && b.day) {
|
||||
return b.day.localeCompare(a.day);
|
||||
}
|
||||
if (a.day) {
|
||||
return -1;
|
||||
}
|
||||
if (b.day) {
|
||||
return 1;
|
||||
}
|
||||
return a.filePath.localeCompare(b.filePath);
|
||||
});
|
||||
|
||||
const valid = resolved.filter((entry): entry is { filePath: string; day: string } => Boolean(entry.day));
|
||||
const skippedPaths = resolved.filter((entry) => !entry.day).map((entry) => entry.filePath);
|
||||
const totalCap = Math.max(20, params.limit * 4);
|
||||
const perFileCap = Math.max(6, Math.ceil(totalCap / Math.max(1, valid.length)));
|
||||
let importedSignalCount = 0;
|
||||
let importedFileCount = 0;
|
||||
|
||||
for (const entry of valid) {
|
||||
if (importedSignalCount >= totalCap) {
|
||||
break;
|
||||
}
|
||||
const raw = await fs.readFile(entry.filePath, "utf-8").catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
skippedPaths.push(entry.filePath);
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (!raw) {
|
||||
continue;
|
||||
}
|
||||
const lines = stripManagedDailyDreamingLines(raw.split(/\r?\n/));
|
||||
const chunks = buildDailySnippetChunks(lines, perFileCap);
|
||||
const results: MemorySearchResult[] = [];
|
||||
for (const chunk of chunks) {
|
||||
results.push({
|
||||
path: `memory/${entry.day}.md`,
|
||||
startLine: chunk.startLine,
|
||||
endLine: chunk.endLine,
|
||||
score: DAILY_INGESTION_SCORE,
|
||||
snippet: chunk.snippet,
|
||||
source: "memory",
|
||||
});
|
||||
if (results.length >= perFileCap || importedSignalCount + results.length >= totalCap) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (results.length === 0) {
|
||||
continue;
|
||||
}
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir: params.workspaceDir,
|
||||
query: `__dreaming_daily__:${entry.day}`,
|
||||
results,
|
||||
signalType: "daily",
|
||||
dedupeByQueryPerDay: true,
|
||||
dayBucket: entry.day,
|
||||
nowMs: params.nowMs,
|
||||
timezone: params.timezone,
|
||||
});
|
||||
importedSignalCount += results.length;
|
||||
importedFileCount += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
importedFileCount,
|
||||
importedSignalCount,
|
||||
skippedPaths,
|
||||
};
|
||||
}
|
||||
|
||||
function entryAverageScore(entry: ShortTermRecallEntry): number {
|
||||
return entry.recallCount > 0 ? Math.max(0, Math.min(1, entry.totalScore / entry.recallCount)) : 0;
|
||||
}
|
||||
|
||||
function tokenizeSnippet(snippet: string): Set<string> {
|
||||
return new Set(
|
||||
normalizeLowercaseStringOrEmpty(snippet)
|
||||
snippet
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/i)
|
||||
.map((token) => token.trim())
|
||||
.filter(Boolean),
|
||||
@@ -1117,7 +1212,7 @@ function jaccardSimilarity(left: string, right: string): number {
|
||||
const leftTokens = tokenizeSnippet(left);
|
||||
const rightTokens = tokenizeSnippet(right);
|
||||
if (leftTokens.size === 0 || rightTokens.size === 0) {
|
||||
return normalizeLowercaseStringOrEmpty(left) === normalizeLowercaseStringOrEmpty(right) ? 1 : 0;
|
||||
return left.trim().toLowerCase() === right.trim().toLowerCase() ? 1 : 0;
|
||||
}
|
||||
let intersection = 0;
|
||||
for (const token of leftTokens) {
|
||||
@@ -1529,10 +1624,7 @@ async function runPhaseIfTriggered(params: {
|
||||
storage: { mode: "inline" | "separate" | "both"; separateReports: boolean };
|
||||
});
|
||||
}): Promise<{ handled: true; reason: string } | undefined> {
|
||||
if (
|
||||
params.trigger !== "heartbeat" ||
|
||||
!includesSystemEventToken(params.cleanedBody, params.eventText)
|
||||
) {
|
||||
if (params.trigger !== "heartbeat" || params.cleanedBody.trim() !== params.eventText) {
|
||||
return undefined;
|
||||
}
|
||||
if (!params.config.enabled) {
|
||||
@@ -1558,10 +1650,7 @@ async function runPhaseIfTriggered(params: {
|
||||
await runLightDreaming({
|
||||
workspaceDir,
|
||||
cfg: params.cfg,
|
||||
config: params.config as MemoryLightDreamingConfig & {
|
||||
timezone?: string;
|
||||
storage: { mode: "inline" | "separate" | "both"; separateReports: boolean };
|
||||
},
|
||||
config: params.config,
|
||||
logger: params.logger,
|
||||
subagent: params.subagent,
|
||||
});
|
||||
@@ -1569,10 +1658,7 @@ async function runPhaseIfTriggered(params: {
|
||||
await runRemDreaming({
|
||||
workspaceDir,
|
||||
cfg: params.cfg,
|
||||
config: params.config as MemoryRemDreamingConfig & {
|
||||
timezone?: string;
|
||||
storage: { mode: "inline" | "separate" | "both"; separateReports: boolean };
|
||||
},
|
||||
config: params.config,
|
||||
logger: params.logger,
|
||||
subagent: params.subagent,
|
||||
});
|
||||
|
||||
903
extensions/memory-core/src/rem-evidence.ts
Normal file
903
extensions/memory-core/src/rem-evidence.ts
Normal file
@@ -0,0 +1,903 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const REM_BLOCKED_SECTION_RE =
|
||||
/\b(morning reminders|tasks? for today|to-?do|pickups?|action items?|next steps?|open questions?|stats|setup tasks?|priority contacts|visitors?|top priority candidates|timeline coverage|action items for morning review|test .* skill|heartbeat checks?|date semantics guardrail|still broken|last message (?:&|and) status|plugin \/ service warning|email triage cron)\b/i;
|
||||
const REM_GENERIC_SECTION_RE =
|
||||
/^(setup|session notes?|notes|summary|major accomplishments?|infrastructure|process improvements?)$/i;
|
||||
const REM_MEMORY_SIGNAL_RE =
|
||||
/\b(always use|prefers?|preference|preferences|standing rule|rule:|use .* calendar|durable|remember)\b/i;
|
||||
const REM_BUILD_SIGNAL_RE =
|
||||
/\b(set up|setup|created|built|rewrite|rewrote|implemented|installed|configured|added|updated|exported|documented)\b/i;
|
||||
const REM_INCIDENT_SIGNAL_RE =
|
||||
/\b(fail(?:ed|ing)?|error|issue|problem|auth|expired|broken|unable|missing|required|root cause|consecutive failures?)\b/i;
|
||||
const REM_LOGISTICS_SIGNAL_RE =
|
||||
/\b(visitor|arriv(?:e|al|ing)|flight|calendar|reservation|schedule|coordinate|travel|pickup)\b/i;
|
||||
const REM_TASK_SIGNAL_RE =
|
||||
/\b(reminder|task|to-?do|action item|next step|need to|follow up|respond to|call\b|check\b)\b/i;
|
||||
const REM_ROUTING_SIGNAL_RE =
|
||||
/\b(categor(?:ize|ized|ization)|route|routing|workflow|processor|read later|auto-implement|codex|razor)\b/i;
|
||||
const REM_OPERATOR_RULE_SIGNAL_RE = /\b(learned:|rule:|always [a-z])\b/i;
|
||||
const REM_EXTERNALIZATION_SIGNAL_RE =
|
||||
/\b(obsidian|memory|tracker|notes captured|committed to memory|updated .*md|documented|file comparison table)\b/i;
|
||||
const REM_RETRY_SIGNAL_RE =
|
||||
/\b(repeat(?:ed|edly)?|again|retry|root cause|third attempt|fourth|fifth|consecutive failures?)\b/i;
|
||||
const REM_PERSON_PATTERN_SIGNAL_RE =
|
||||
/\b(relationship|who:|patterns?:|failure modes?:|best stance:|space|boundaries|timing|family quick reference)\b/i;
|
||||
const REM_SITUATIONAL_SIGNAL_RE =
|
||||
/\b(hotel|address|phone|reservation|check-?in|check-?out|flight|arrival|departure|terminal|price shown|invoice|pending items|screenshot|butler)\b/i;
|
||||
const REM_PERSISTENCE_SIGNAL_RE =
|
||||
/\b(always|preference|prefers?|standing rule|best stance|failure modes?|key patterns?|relationship|who:|important .* keep track|people in .* life|partner|wife|husband|boyfriend|girlfriend)\b/i;
|
||||
const REM_TRANSIENT_SIGNAL_RE =
|
||||
/\b(today|this session|in progress|installed|booked|confirmed|pending|status:|action pending|open items?|next steps?|issue:|diagnostics|screenshot|source file|insight files|thread\b|ticket|price shown|calendar fix|cron fixes|security audit|updates? this session|bought:|order\b)\b/i;
|
||||
const REM_SECTION_PERSISTENCE_TITLE_RE =
|
||||
/\b(preferences? learned|preference|people update|relationship|standing|patterns?|identity|memory)\b/i;
|
||||
const REM_SECTION_TRANSIENT_TITLE_RE =
|
||||
/\b(setup|fix|fixes|audit|booked|call|today|session|updates?|file paths|open items?|next steps?|research pipeline|info gathered|calendar|tickets?)\b/i;
|
||||
const REM_METADATA_HEAVY_SIGNAL_RE =
|
||||
/\b(address|phone|email|website|google maps|source file|insight files|conversation id|thread has|order\b|reservation\b|price\b|cost\b|ticket|uuid|url:|model:|workspace:|bindings:|accountid|config change|path:)\b/i;
|
||||
const REM_PROJECT_META_SIGNAL_RE =
|
||||
/\b(strategy|audit|discussion|research|topic|candidate|north star|pipeline|data dump|export|draft|insights? draft|weekly|analysis|findings)\b/i;
|
||||
const REM_PROCESS_FRAME_SIGNAL_RE =
|
||||
/\b(dossier|registry|cadence|framework|facts,\s*timeline|open loops|next actions|auto preference rollups?|insights? draft created)\b/i;
|
||||
const REM_TOOLING_META_SIGNAL_RE =
|
||||
/\b(cli|tool|tools\.md|agents\.md|sessionssend|subagents?|spawn|tmux|xurl|bird|codex exec|interactive codex)\b/i;
|
||||
const REM_TRAVEL_DECISION_SIGNAL_RE =
|
||||
/\b(routing|cabin|business class|trip brief|departure|arrival|hotel|reservation|tickets?|show tonight|cheaper alternatives?|venue timing)\b/i;
|
||||
const REM_STABLE_PERSON_SIGNAL_RE =
|
||||
/\b(partner|wife|husband|boyfriend|girlfriend|relationship interest|lives in)\b/i;
|
||||
const REM_EXPLICIT_PREFERENCE_SIGNAL_RE =
|
||||
/\b(explicitly|wants?|does not want|don't want|default .* should|should default to|likes?|dislikes?|treat .* as|prefers?)\b/i;
|
||||
const REM_SPECIFICITY_BURDEN_RE =
|
||||
/\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\b|€|\$\d|→|\b\d{1,2}:\d{2}\b|\+\d{6,}/i;
|
||||
const REM_TIME_PREFIX_RE = /^\d{1,2}:\d{2}\s*-\s*/;
|
||||
const REM_CODE_FENCE_RE = /^\s*```/;
|
||||
const REM_TABLE_RE = /^\s*\|.*\|\s*$/;
|
||||
const REM_TABLE_DIVIDER_RE = /^\s*\|?[\s:-]+\|[\s|:-]*$/;
|
||||
const REM_SUMMARY_FACT_LIMIT = 4;
|
||||
const REM_SUMMARY_REFLECTION_LIMIT = 4;
|
||||
const REM_SUMMARY_MEMORY_LIMIT = 3;
|
||||
|
||||
export type GroundedRemPreviewItem = {
|
||||
text: string;
|
||||
refs: string[];
|
||||
};
|
||||
|
||||
export type GroundedRemCandidate = GroundedRemPreviewItem & {
|
||||
lean: "likely_durable" | "unclear" | "likely_situational";
|
||||
};
|
||||
|
||||
export type GroundedRemFilePreview = {
|
||||
path: string;
|
||||
facts: GroundedRemPreviewItem[];
|
||||
reflections: GroundedRemPreviewItem[];
|
||||
memoryImplications: GroundedRemPreviewItem[];
|
||||
candidates: GroundedRemCandidate[];
|
||||
renderedMarkdown: string;
|
||||
};
|
||||
|
||||
export type GroundedRemPreviewResult = {
|
||||
workspaceDir: string;
|
||||
scannedFiles: number;
|
||||
files: GroundedRemFilePreview[];
|
||||
};
|
||||
|
||||
type CandidateSnippetSummary = GroundedRemCandidate & {
|
||||
score: number;
|
||||
};
|
||||
|
||||
type ParsedSectionLine = {
|
||||
line: number;
|
||||
text: string;
|
||||
};
|
||||
|
||||
type ParsedMarkdownSection = {
|
||||
title: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
lines: ParsedSectionLine[];
|
||||
};
|
||||
|
||||
type SectionSnippet = {
|
||||
text: string;
|
||||
line: number;
|
||||
};
|
||||
|
||||
type SectionSummary = {
|
||||
title: string;
|
||||
text: string;
|
||||
refs: string[];
|
||||
scores: {
|
||||
preference: number;
|
||||
build: number;
|
||||
incident: number;
|
||||
logistics: number;
|
||||
tasks: number;
|
||||
routing: number;
|
||||
externalization: number;
|
||||
retries: number;
|
||||
overall: number;
|
||||
};
|
||||
};
|
||||
|
||||
function normalizeWhitespace(value: string): string {
|
||||
return value.trim().replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
function normalizePath(rawPath: string): string {
|
||||
return rawPath.replaceAll("\\", "/").replace(/^\.\//, "");
|
||||
}
|
||||
|
||||
function stripMarkdown(text: string): string {
|
||||
return normalizeWhitespace(
|
||||
text
|
||||
.replace(/!\[[^\]]*]\([^)]*\)/g, "")
|
||||
.replace(/\[([^\]]+)]\([^)]*\)/g, "$1")
|
||||
.replace(/[`*_~>#]/g, "")
|
||||
.replace(/\s+/g, " "),
|
||||
);
|
||||
}
|
||||
|
||||
function sanitizeSectionTitle(title: string): string {
|
||||
return normalizeWhitespace(stripMarkdown(title).replace(REM_TIME_PREFIX_RE, ""));
|
||||
}
|
||||
|
||||
function makeRef(pathValue: string, startLine: number, endLine = startLine): string {
|
||||
return startLine === endLine
|
||||
? `${pathValue}:${startLine}`
|
||||
: `${pathValue}:${startLine}-${endLine}`;
|
||||
}
|
||||
|
||||
function parseMarkdownSections(content: string): ParsedMarkdownSection[] {
|
||||
const sections: ParsedMarkdownSection[] = [];
|
||||
const lines = content.split(/\r?\n/);
|
||||
let current: ParsedMarkdownSection | null = null;
|
||||
let inCodeFence = false;
|
||||
|
||||
const flush = () => {
|
||||
if (!current) {
|
||||
return;
|
||||
}
|
||||
const meaningfulLines = current.lines.filter(
|
||||
(entry) => normalizeWhitespace(entry.text).length > 0,
|
||||
);
|
||||
if (meaningfulLines.length > 0) {
|
||||
const endLine = meaningfulLines[meaningfulLines.length - 1]?.line ?? current.endLine;
|
||||
sections.push({ ...current, endLine, lines: meaningfulLines });
|
||||
}
|
||||
current = null;
|
||||
};
|
||||
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const rawLine = lines[index] ?? "";
|
||||
const lineNumber = index + 1;
|
||||
if (REM_CODE_FENCE_RE.test(rawLine)) {
|
||||
inCodeFence = !inCodeFence;
|
||||
continue;
|
||||
}
|
||||
if (inCodeFence) {
|
||||
continue;
|
||||
}
|
||||
const headingMatch = rawLine.match(/^\s{0,3}(#{2,6})\s+(.+)$/);
|
||||
if (headingMatch?.[2]) {
|
||||
flush();
|
||||
current = {
|
||||
title: sanitizeSectionTitle(headingMatch[2]),
|
||||
startLine: lineNumber,
|
||||
endLine: lineNumber,
|
||||
lines: [],
|
||||
};
|
||||
continue;
|
||||
}
|
||||
if (!current) {
|
||||
continue;
|
||||
}
|
||||
current.endLine = lineNumber;
|
||||
const trimmed = rawLine.trim();
|
||||
if (
|
||||
!trimmed ||
|
||||
/^---+$/.test(trimmed) ||
|
||||
REM_TABLE_RE.test(trimmed) ||
|
||||
REM_TABLE_DIVIDER_RE.test(trimmed)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
current.lines.push({ line: lineNumber, text: rawLine });
|
||||
}
|
||||
|
||||
flush();
|
||||
return sections;
|
||||
}
|
||||
|
||||
function sectionToSnippets(section: ParsedMarkdownSection): SectionSnippet[] {
|
||||
const snippets: SectionSnippet[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const entry of section.lines) {
|
||||
const trimmed = entry.text.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
const bulletMatch = trimmed.match(/^(?:[-*+]|\d+\.)\s+(?:\[[ xX]\]\s*)?(.*)$/);
|
||||
const candidateText = bulletMatch?.[1] ?? trimmed;
|
||||
const text = normalizeWhitespace(stripMarkdown(candidateText));
|
||||
if (text.length < 10) {
|
||||
continue;
|
||||
}
|
||||
const dedupeKey = text.toLowerCase();
|
||||
if (seen.has(dedupeKey)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(dedupeKey);
|
||||
snippets.push({ text, line: entry.line });
|
||||
}
|
||||
return snippets;
|
||||
}
|
||||
|
||||
function countMatchingSnippets(snippets: SectionSnippet[], pattern: RegExp): number {
|
||||
let count = 0;
|
||||
for (const snippet of snippets) {
|
||||
if (pattern.test(snippet.text)) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function scoreSection(section: ParsedMarkdownSection, snippets: SectionSnippet[]) {
|
||||
const title = section.title;
|
||||
const titleBonus = (pattern: RegExp) => (pattern.test(title) ? 1 : 0);
|
||||
const preference =
|
||||
countMatchingSnippets(snippets, REM_MEMORY_SIGNAL_RE) + titleBonus(REM_MEMORY_SIGNAL_RE);
|
||||
const build =
|
||||
countMatchingSnippets(snippets, REM_BUILD_SIGNAL_RE) + titleBonus(REM_BUILD_SIGNAL_RE);
|
||||
const incident =
|
||||
countMatchingSnippets(snippets, REM_INCIDENT_SIGNAL_RE) + titleBonus(REM_INCIDENT_SIGNAL_RE);
|
||||
const logistics =
|
||||
countMatchingSnippets(snippets, REM_LOGISTICS_SIGNAL_RE) + titleBonus(REM_LOGISTICS_SIGNAL_RE);
|
||||
const tasks =
|
||||
countMatchingSnippets(snippets, REM_TASK_SIGNAL_RE) + titleBonus(REM_TASK_SIGNAL_RE);
|
||||
const routing =
|
||||
countMatchingSnippets(snippets, REM_ROUTING_SIGNAL_RE) + titleBonus(REM_ROUTING_SIGNAL_RE);
|
||||
const externalization =
|
||||
countMatchingSnippets(snippets, REM_EXTERNALIZATION_SIGNAL_RE) +
|
||||
titleBonus(REM_EXTERNALIZATION_SIGNAL_RE);
|
||||
const retries =
|
||||
countMatchingSnippets(snippets, REM_RETRY_SIGNAL_RE) + titleBonus(REM_RETRY_SIGNAL_RE);
|
||||
const overall =
|
||||
preference * 2 +
|
||||
build * 1.6 +
|
||||
incident * 1.6 +
|
||||
logistics * 1.2 +
|
||||
routing * 1.8 +
|
||||
externalization * 1.4 +
|
||||
Math.min(snippets.length, 3) * 0.3 -
|
||||
(REM_GENERIC_SECTION_RE.test(title) ? 0.8 : 0);
|
||||
return {
|
||||
preference,
|
||||
build,
|
||||
incident,
|
||||
logistics,
|
||||
tasks,
|
||||
routing,
|
||||
externalization,
|
||||
retries,
|
||||
overall,
|
||||
};
|
||||
}
|
||||
|
||||
function scoreSnippet(text: string, title: string): number {
|
||||
let score = 1;
|
||||
if (REM_MEMORY_SIGNAL_RE.test(text)) {
|
||||
score += 2.2;
|
||||
}
|
||||
if (REM_BUILD_SIGNAL_RE.test(text)) {
|
||||
score += 1.2;
|
||||
}
|
||||
if (REM_INCIDENT_SIGNAL_RE.test(text)) {
|
||||
score += 1.2;
|
||||
}
|
||||
if (REM_LOGISTICS_SIGNAL_RE.test(text)) {
|
||||
score += 0.9;
|
||||
}
|
||||
if (REM_ROUTING_SIGNAL_RE.test(text)) {
|
||||
score += 1.4;
|
||||
}
|
||||
if (REM_EXTERNALIZATION_SIGNAL_RE.test(text)) {
|
||||
score += 1.1;
|
||||
}
|
||||
if (REM_RETRY_SIGNAL_RE.test(text)) {
|
||||
score += 0.9;
|
||||
}
|
||||
if (REM_TASK_SIGNAL_RE.test(text) && !REM_BUILD_SIGNAL_RE.test(text)) {
|
||||
score -= 0.8;
|
||||
}
|
||||
if (title && !REM_GENERIC_SECTION_RE.test(title)) {
|
||||
score += 0.25;
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
function chooseSummarySnippets(
|
||||
section: ParsedMarkdownSection,
|
||||
snippets: SectionSnippet[],
|
||||
): SectionSnippet[] {
|
||||
const selectionLimit = REM_GENERIC_SECTION_RE.test(section.title) ? 2 : 3;
|
||||
return [...snippets]
|
||||
.toSorted((left, right) => {
|
||||
const scoreDelta =
|
||||
scoreSnippet(right.text, section.title) - scoreSnippet(left.text, section.title);
|
||||
if (scoreDelta !== 0) {
|
||||
return scoreDelta;
|
||||
}
|
||||
return left.line - right.line;
|
||||
})
|
||||
.slice(0, selectionLimit)
|
||||
.toSorted((left, right) => left.line - right.line);
|
||||
}
|
||||
|
||||
function joinSummaryParts(parts: string[]): string {
|
||||
if (parts.length <= 1) {
|
||||
return parts[0] ?? "";
|
||||
}
|
||||
if (parts.length === 2) {
|
||||
return `${parts[0]} and ${parts[1]}`;
|
||||
}
|
||||
return `${parts.slice(0, -1).join("; ")}; and ${parts[parts.length - 1]}`;
|
||||
}
|
||||
|
||||
function summarizeSection(
|
||||
pathValue: string,
|
||||
section: ParsedMarkdownSection,
|
||||
): SectionSummary | null {
|
||||
if (REM_BLOCKED_SECTION_RE.test(section.title)) {
|
||||
return null;
|
||||
}
|
||||
const snippets = sectionToSnippets(section);
|
||||
if (snippets.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const selected = chooseSummarySnippets(section, snippets);
|
||||
if (selected.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const title = sanitizeSectionTitle(section.title);
|
||||
const body = joinSummaryParts(selected.map((snippet) => snippet.text));
|
||||
const text = !title || REM_GENERIC_SECTION_RE.test(title) ? body : `${title}: ${body}`;
|
||||
return {
|
||||
title,
|
||||
text,
|
||||
refs: selected.map((snippet) => makeRef(pathValue, snippet.line)),
|
||||
scores: scoreSection(section, snippets),
|
||||
};
|
||||
}
|
||||
|
||||
function compactCandidateTitle(title: string): string {
|
||||
let compact = sanitizeSectionTitle(title)
|
||||
.replace(/\s*\((?:via:|from qmd \+ memory|this session)[^)]+\)\s*/gi, " ")
|
||||
.replace(
|
||||
/\s*[—-]\s*(?:research results.*|in progress.*|working.*|installed.*|booked.*|proposed.*|clarified.*|candidate.*|fixes.*|updates?.*)$/i,
|
||||
"",
|
||||
)
|
||||
.trim();
|
||||
if (/^(?:preferences? learned|candidate facts?)$/i.test(compact)) {
|
||||
return "";
|
||||
}
|
||||
compact = compact.replace(/^preference:\s*/i, "");
|
||||
return compact;
|
||||
}
|
||||
|
||||
function compactCandidateSnippetText(text: string, title: string): string {
|
||||
const normalized = normalizeWhitespace(text);
|
||||
if (REM_STABLE_PERSON_SIGNAL_RE.test(`${title} ${normalized}`)) {
|
||||
return (normalized.split(/(?<=[.?!])\s+/)[0] ?? normalized).trim();
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function scoreCandidateSnippet(text: string, title: string): number {
|
||||
let score = 0;
|
||||
if (REM_PERSISTENCE_SIGNAL_RE.test(text)) {
|
||||
score += 3.2;
|
||||
}
|
||||
if (REM_MEMORY_SIGNAL_RE.test(text)) {
|
||||
score += 2.4;
|
||||
}
|
||||
if (REM_EXPLICIT_PREFERENCE_SIGNAL_RE.test(text)) {
|
||||
score += 1.8;
|
||||
}
|
||||
if (REM_PERSON_PATTERN_SIGNAL_RE.test(text)) {
|
||||
score += 2.3;
|
||||
}
|
||||
if (REM_OPERATOR_RULE_SIGNAL_RE.test(text)) {
|
||||
score += 1.6;
|
||||
}
|
||||
if (REM_SECTION_PERSISTENCE_TITLE_RE.test(title)) {
|
||||
score += 1.2;
|
||||
}
|
||||
if (REM_STABLE_PERSON_SIGNAL_RE.test(text)) {
|
||||
score += 1.5;
|
||||
}
|
||||
if (REM_METADATA_HEAVY_SIGNAL_RE.test(text)) {
|
||||
score -= 2.4;
|
||||
}
|
||||
if (REM_PROJECT_META_SIGNAL_RE.test(`${title} ${text}`)) {
|
||||
score -= 2.2;
|
||||
}
|
||||
if (REM_PROCESS_FRAME_SIGNAL_RE.test(text)) {
|
||||
score -= 2.4;
|
||||
}
|
||||
if (REM_TOOLING_META_SIGNAL_RE.test(text) && !REM_STABLE_PERSON_SIGNAL_RE.test(text)) {
|
||||
score -= 2.1;
|
||||
}
|
||||
if (REM_TRAVEL_DECISION_SIGNAL_RE.test(text)) {
|
||||
score -= 2.6;
|
||||
}
|
||||
if (REM_SPECIFICITY_BURDEN_RE.test(text) && !REM_STABLE_PERSON_SIGNAL_RE.test(text)) {
|
||||
score -= 1.2;
|
||||
}
|
||||
if (REM_SITUATIONAL_SIGNAL_RE.test(text)) {
|
||||
score -= 2.8;
|
||||
}
|
||||
if (REM_TRANSIENT_SIGNAL_RE.test(text)) {
|
||||
score -= 2;
|
||||
}
|
||||
if (REM_INCIDENT_SIGNAL_RE.test(text)) {
|
||||
score -= 1.6;
|
||||
}
|
||||
if (REM_TASK_SIGNAL_RE.test(text)) {
|
||||
score -= 1.2;
|
||||
}
|
||||
if (REM_LOGISTICS_SIGNAL_RE.test(text) && !REM_MEMORY_SIGNAL_RE.test(text)) {
|
||||
score -= 1.4;
|
||||
}
|
||||
if (REM_BUILD_SIGNAL_RE.test(text) && !REM_MEMORY_SIGNAL_RE.test(text)) {
|
||||
score -= 0.8;
|
||||
}
|
||||
if (REM_SECTION_TRANSIENT_TITLE_RE.test(title) && !REM_SECTION_PERSISTENCE_TITLE_RE.test(title)) {
|
||||
score -= 1.2;
|
||||
}
|
||||
if (/[`/]/.test(text) || /https?:\/\//i.test(text)) {
|
||||
score -= 0.8;
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
function chooseFactSnippets(
|
||||
section: ParsedMarkdownSection,
|
||||
snippets: SectionSnippet[],
|
||||
): SectionSnippet[] {
|
||||
return [...snippets]
|
||||
.map((snippet) => {
|
||||
const text = compactCandidateSnippetText(snippet.text, section.title);
|
||||
const score =
|
||||
scoreCandidateSnippet(text, section.title) + (REM_MEMORY_SIGNAL_RE.test(text) ? 0.6 : 0);
|
||||
return { snippet: { ...snippet, text }, score };
|
||||
})
|
||||
.filter((entry) => entry.snippet.text.length >= 18 && entry.score >= 1.4)
|
||||
.toSorted((left, right) => {
|
||||
if (right.score !== left.score) {
|
||||
return right.score - left.score;
|
||||
}
|
||||
return left.snippet.line - right.snippet.line;
|
||||
})
|
||||
.slice(0, 2)
|
||||
.map((entry) => entry.snippet)
|
||||
.toSorted((left, right) => left.line - right.line);
|
||||
}
|
||||
|
||||
type FactSnippetSummary = GroundedRemPreviewItem & {
|
||||
score: number;
|
||||
};
|
||||
|
||||
function buildFactText(title: string, text: string): string {
|
||||
const compactTitle = compactCandidateTitle(title);
|
||||
if (!compactTitle) {
|
||||
return text;
|
||||
}
|
||||
if (
|
||||
REM_SECTION_PERSISTENCE_TITLE_RE.test(compactTitle) ||
|
||||
REM_STABLE_PERSON_SIGNAL_RE.test(compactTitle) ||
|
||||
/\b(relationship|people mentioned|people update|identity)\b/i.test(compactTitle)
|
||||
) {
|
||||
return `${compactTitle}: ${text}`;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function chooseCandidateSnippets(
|
||||
section: ParsedMarkdownSection,
|
||||
snippets: SectionSnippet[],
|
||||
): SectionSnippet[] {
|
||||
return [...snippets]
|
||||
.map((snippet) => {
|
||||
const text = compactCandidateSnippetText(snippet.text, section.title);
|
||||
const score = scoreCandidateSnippet(text, section.title);
|
||||
return { snippet: { ...snippet, text }, score };
|
||||
})
|
||||
.filter((entry) => entry.snippet.text.length >= 18 && entry.score >= 1.8)
|
||||
.toSorted((left, right) => {
|
||||
if (right.score !== left.score) {
|
||||
return right.score - left.score;
|
||||
}
|
||||
return left.snippet.line - right.snippet.line;
|
||||
})
|
||||
.slice(0, 2)
|
||||
.map((entry) => entry.snippet)
|
||||
.toSorted((left, right) => left.line - right.line);
|
||||
}
|
||||
|
||||
function buildCandidateSnippetText(title: string, text: string): string {
|
||||
return buildFactText(title, text);
|
||||
}
|
||||
|
||||
function classifyCandidateLeanFromText(text: string, title: string): GroundedRemCandidate["lean"] {
|
||||
const score = scoreCandidateSnippet(text, title);
|
||||
if (score >= 4) {
|
||||
return "likely_durable";
|
||||
}
|
||||
if (score <= 0.25 || REM_SITUATIONAL_SIGNAL_RE.test(text) || REM_TRANSIENT_SIGNAL_RE.test(text)) {
|
||||
return "likely_situational";
|
||||
}
|
||||
return "unclear";
|
||||
}
|
||||
|
||||
function addReflection(
|
||||
reflections: GroundedRemPreviewItem[],
|
||||
seen: Set<string>,
|
||||
text: string,
|
||||
refs: string[],
|
||||
) {
|
||||
const normalized = normalizeWhitespace(text);
|
||||
const key = normalized.toLowerCase();
|
||||
if (!normalized || seen.has(key)) {
|
||||
return;
|
||||
}
|
||||
seen.add(key);
|
||||
reflections.push({ text: normalized, refs });
|
||||
}
|
||||
|
||||
function isOperatorRuleSummary(summary: SectionSummary): boolean {
|
||||
return (
|
||||
/process improvements?/i.test(summary.title) || REM_OPERATOR_RULE_SIGNAL_RE.test(summary.text)
|
||||
);
|
||||
}
|
||||
|
||||
function isRoutingSummary(summary: SectionSummary): boolean {
|
||||
return summary.scores.routing > 0 || REM_ROUTING_SIGNAL_RE.test(summary.text);
|
||||
}
|
||||
|
||||
function previewGroundedRemForFile(params: {
|
||||
relPath: string;
|
||||
content: string;
|
||||
}): GroundedRemFilePreview {
|
||||
const sections = parseMarkdownSections(params.content);
|
||||
const sectionScores = sections.map((section) => ({
|
||||
section,
|
||||
snippets: sectionToSnippets(section),
|
||||
}));
|
||||
const summaries = sectionScores
|
||||
.map(({ section }) => summarizeSection(params.relPath, section))
|
||||
.filter((summary): summary is SectionSummary => summary !== null);
|
||||
const factSummaries: FactSnippetSummary[] = sections.flatMap((section) => {
|
||||
if (REM_BLOCKED_SECTION_RE.test(section.title)) {
|
||||
return [];
|
||||
}
|
||||
const snippets = sectionToSnippets(section);
|
||||
if (snippets.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return chooseFactSnippets(section, snippets).map((snippet) => ({
|
||||
text: buildFactText(section.title, snippet.text),
|
||||
refs: [makeRef(params.relPath, snippet.line)],
|
||||
score: scoreCandidateSnippet(snippet.text, section.title),
|
||||
}));
|
||||
});
|
||||
|
||||
const memoryImplications = summaries
|
||||
.filter((summary) => summary.scores.preference > 0 || isOperatorRuleSummary(summary))
|
||||
.map((summary) => ({
|
||||
text: summary.text.replace(/^[^:]+:\s*/, ""),
|
||||
refs: summary.refs,
|
||||
}))
|
||||
.filter((item, index, items) => items.findIndex((entry) => entry.text === item.text) === index)
|
||||
.slice(0, REM_SUMMARY_MEMORY_LIMIT);
|
||||
|
||||
const candidateSnippets: CandidateSnippetSummary[] = sections.flatMap((section) => {
|
||||
if (REM_BLOCKED_SECTION_RE.test(section.title)) {
|
||||
return [];
|
||||
}
|
||||
const snippets = sectionToSnippets(section);
|
||||
if (snippets.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return chooseCandidateSnippets(section, snippets)
|
||||
.map((snippet) => {
|
||||
const score = scoreCandidateSnippet(snippet.text, section.title);
|
||||
const text = buildCandidateSnippetText(section.title, snippet.text);
|
||||
return {
|
||||
text,
|
||||
refs: [makeRef(params.relPath, snippet.line)],
|
||||
lean: classifyCandidateLeanFromText(snippet.text, section.title),
|
||||
score,
|
||||
};
|
||||
})
|
||||
.filter((candidate) => candidate.text.length >= 12 && candidate.score >= 1.8);
|
||||
});
|
||||
|
||||
const candidates = candidateSnippets
|
||||
.toSorted((left, right) => {
|
||||
const leanRank = { likely_durable: 0, unclear: 1, likely_situational: 2 };
|
||||
const leanDelta = leanRank[left.lean] - leanRank[right.lean];
|
||||
if (leanDelta !== 0) {
|
||||
return leanDelta;
|
||||
}
|
||||
return right.score - left.score;
|
||||
})
|
||||
.filter(
|
||||
(candidate, index, items) =>
|
||||
items.findIndex((entry) => entry.text === candidate.text) === index,
|
||||
)
|
||||
.slice(0, 4);
|
||||
|
||||
const durableImplications = candidateSnippets
|
||||
.filter((candidate) => candidate.lean === "likely_durable" || candidate.score >= 4)
|
||||
.filter(
|
||||
(candidate, index, items) =>
|
||||
items.findIndex((entry) => entry.text === candidate.text) === index,
|
||||
)
|
||||
.toSorted((left, right) => right.score - left.score)
|
||||
.slice(0, REM_SUMMARY_MEMORY_LIMIT)
|
||||
.map((candidate) => ({ text: candidate.text, refs: candidate.refs }));
|
||||
|
||||
const candidateDrivenImplications = candidateSnippets
|
||||
.filter((candidate) => candidate.lean !== "likely_situational" && candidate.score >= 2.2)
|
||||
.filter(
|
||||
(candidate, index, items) =>
|
||||
items.findIndex((entry) => entry.text === candidate.text) === index,
|
||||
)
|
||||
.toSorted((left, right) => right.score - left.score)
|
||||
.slice(0, REM_SUMMARY_MEMORY_LIMIT)
|
||||
.map((candidate) => ({ text: candidate.text, refs: candidate.refs }));
|
||||
|
||||
const effectiveMemoryImplications =
|
||||
durableImplications.length > 0
|
||||
? durableImplications
|
||||
: candidateDrivenImplications.length > 0
|
||||
? candidateDrivenImplications
|
||||
: memoryImplications;
|
||||
|
||||
const facts: GroundedRemPreviewItem[] = [];
|
||||
const usedFactTexts = new Set<string>();
|
||||
for (const summary of factSummaries.toSorted((left, right) => right.score - left.score)) {
|
||||
const key = summary.text.toLowerCase();
|
||||
if (usedFactTexts.has(key)) {
|
||||
continue;
|
||||
}
|
||||
usedFactTexts.add(key);
|
||||
facts.push({ text: summary.text, refs: summary.refs });
|
||||
if (facts.length >= REM_SUMMARY_FACT_LIMIT) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (facts.length === 0) {
|
||||
const bestFor = (metric: keyof SectionSummary["scores"]) =>
|
||||
summaries
|
||||
.filter((summary) => summary.scores[metric] > 0)
|
||||
.toSorted((left, right) => {
|
||||
if (right.scores[metric] !== left.scores[metric]) {
|
||||
return right.scores[metric] - left.scores[metric];
|
||||
}
|
||||
return right.scores.overall - left.scores.overall;
|
||||
})[0];
|
||||
for (const summary of [
|
||||
bestFor("preference"),
|
||||
bestFor("routing"),
|
||||
bestFor("externalization"),
|
||||
...summaries.toSorted((left, right) => right.scores.overall - left.scores.overall),
|
||||
]) {
|
||||
if (!summary) {
|
||||
continue;
|
||||
}
|
||||
const key = summary.text.toLowerCase();
|
||||
if (usedFactTexts.has(key)) {
|
||||
continue;
|
||||
}
|
||||
usedFactTexts.add(key);
|
||||
facts.push({ text: summary.text, refs: summary.refs });
|
||||
if (facts.length >= REM_SUMMARY_FACT_LIMIT) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const reflections: GroundedRemPreviewItem[] = [];
|
||||
const seenReflections = new Set<string>();
|
||||
const buildSignal = summaries.reduce((sum, item) => sum + item.scores.build, 0);
|
||||
const incidentSignal = summaries.reduce((sum, item) => sum + item.scores.incident, 0);
|
||||
const logisticsSignal = summaries.reduce((sum, item) => sum + item.scores.logistics, 0);
|
||||
const routingSignal = summaries.reduce((sum, item) => sum + item.scores.routing, 0);
|
||||
const externalizationSignal = summaries.reduce(
|
||||
(sum, item) => sum + item.scores.externalization,
|
||||
0,
|
||||
);
|
||||
const retrySignal = summaries.reduce((sum, item) => sum + item.scores.retries, 0);
|
||||
const taskSignal = sectionScores.reduce(
|
||||
(sum, { section, snippets }) => sum + scoreSection(section, snippets).tasks,
|
||||
0,
|
||||
);
|
||||
const strongestRoutingSummary = summaries
|
||||
.filter((summary) => isRoutingSummary(summary))
|
||||
.toSorted((left, right) => right.scores.overall - left.scores.overall)[0];
|
||||
const strongestIncidentSummary = summaries
|
||||
.filter((summary) => summary.scores.incident > 0)
|
||||
.toSorted((left, right) => right.scores.overall - left.scores.overall)[0];
|
||||
const strongestExternalizationSummary = summaries
|
||||
.filter((summary) => summary.scores.externalization > 0)
|
||||
.toSorted((left, right) => right.scores.overall - left.scores.overall)[0];
|
||||
|
||||
if (effectiveMemoryImplications.length > 0) {
|
||||
addReflection(
|
||||
reflections,
|
||||
seenReflections,
|
||||
"A stable rule or preference was stated explicitly, which suggests operating choices are being made legible instead of left implicit.",
|
||||
effectiveMemoryImplications.flatMap((item) => item.refs).slice(0, 3),
|
||||
);
|
||||
}
|
||||
if (facts.length > 0 && routingSignal >= 2 && strongestRoutingSummary && buildSignal >= incidentSignal) {
|
||||
addReflection(
|
||||
reflections,
|
||||
seenReflections,
|
||||
"The strongest pattern here is a preference for converting messy inbound information into routed workflows with different downstream actions, instead of handling each case manually.",
|
||||
strongestRoutingSummary.refs,
|
||||
);
|
||||
}
|
||||
if (facts.length > 0 && externalizationSignal >= 2 && strongestExternalizationSummary) {
|
||||
addReflection(
|
||||
reflections,
|
||||
seenReflections,
|
||||
"Important context tends to get externalized quickly into notes, trackers, or memory surfaces, which suggests a preference for explicit systems over holding context informally.",
|
||||
strongestExternalizationSummary.refs,
|
||||
);
|
||||
}
|
||||
if (facts.length > 0 && buildSignal >= 2) {
|
||||
const buildRefs = facts
|
||||
.filter((item) => REM_BUILD_SIGNAL_RE.test(item.text))
|
||||
.flatMap((item) => item.refs)
|
||||
.slice(0, 3);
|
||||
if (buildRefs.length > 0) {
|
||||
addReflection(
|
||||
reflections,
|
||||
seenReflections,
|
||||
"The day leaned toward building operator infrastructure, which suggests the interaction is often used to reshape the system around recurring needs rather than just complete isolated tasks.",
|
||||
buildRefs,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (facts.length > 0 && incidentSignal >= 2 && strongestIncidentSummary) {
|
||||
addReflection(
|
||||
reflections,
|
||||
seenReflections,
|
||||
retrySignal >= 2
|
||||
? "When something breaks repeatedly, the response is systematic: retries, root-cause narrowing, and preserving enough state to resume once the blocker is fixed."
|
||||
: "A meaningful share of the day went into friction, and the interaction pattern looks pragmatic rather than emotional: diagnose the blocker, preserve state, and move on.",
|
||||
strongestIncidentSummary.refs,
|
||||
);
|
||||
}
|
||||
if (facts.length > 0 && logisticsSignal >= 2) {
|
||||
const logisticsRefs = facts
|
||||
.filter((item) => REM_LOGISTICS_SIGNAL_RE.test(item.text))
|
||||
.flatMap((item) => item.refs)
|
||||
.slice(0, 3);
|
||||
if (logisticsRefs.length > 0) {
|
||||
addReflection(
|
||||
reflections,
|
||||
seenReflections,
|
||||
"Personal logistics and operating-system work are being managed in the same surface, which suggests a preference for one integrated control plane rather than separate personal and technical loops.",
|
||||
logisticsRefs,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (taskSignal >= 3 && reflections.length === 0) {
|
||||
addReflection(
|
||||
reflections,
|
||||
seenReflections,
|
||||
"The raw note is mostly task and current-state material, so it should not be over-read as memory.",
|
||||
[
|
||||
makeRef(
|
||||
params.relPath,
|
||||
sections[0]?.startLine ?? 1,
|
||||
sections[sections.length - 1]?.endLine ?? 1,
|
||||
),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
const visibleReflections = reflections.slice(0, REM_SUMMARY_REFLECTION_LIMIT);
|
||||
|
||||
const renderedLines: string[] = [];
|
||||
renderedLines.push("## What Happened");
|
||||
if (facts.length === 0) {
|
||||
renderedLines.push("1. No grounded facts were extracted.");
|
||||
} else {
|
||||
for (const [index, fact] of facts.entries()) {
|
||||
renderedLines.push(`${index + 1}. ${fact.text} [${fact.refs.join(", ")}]`);
|
||||
}
|
||||
}
|
||||
renderedLines.push("");
|
||||
renderedLines.push("## Reflections");
|
||||
if (visibleReflections.length === 0) {
|
||||
renderedLines.push("1. No grounded reflections emerged from this note yet.");
|
||||
} else {
|
||||
for (const [index, reflection] of visibleReflections.entries()) {
|
||||
renderedLines.push(`${index + 1}. ${reflection.text} [${reflection.refs.join(", ")}]`);
|
||||
}
|
||||
}
|
||||
if (candidates.length > 0) {
|
||||
renderedLines.push("");
|
||||
renderedLines.push("## Candidates");
|
||||
for (const candidate of candidates) {
|
||||
renderedLines.push(`- [${candidate.lean}] ${candidate.text} [${candidate.refs.join(", ")}]`);
|
||||
}
|
||||
}
|
||||
if (effectiveMemoryImplications.length > 0) {
|
||||
renderedLines.push("");
|
||||
renderedLines.push("## Possible Lasting Updates");
|
||||
for (const implication of effectiveMemoryImplications) {
|
||||
renderedLines.push(`- ${implication.text} [${implication.refs.join(", ")}]`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
path: params.relPath,
|
||||
facts,
|
||||
reflections: visibleReflections,
|
||||
memoryImplications: effectiveMemoryImplications,
|
||||
candidates,
|
||||
renderedMarkdown: renderedLines.join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
async function collectMarkdownFiles(inputPaths: string[]): Promise<string[]> {
|
||||
const found = new Set<string>();
|
||||
async function walk(targetPath: string): Promise<void> {
|
||||
const resolved = path.resolve(targetPath);
|
||||
const stat = await fs.stat(resolved);
|
||||
if (stat.isDirectory()) {
|
||||
const entries = await fs.readdir(resolved, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
await walk(path.join(resolved, entry.name));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (stat.isFile() && resolved.toLowerCase().endsWith(".md")) {
|
||||
found.add(resolved);
|
||||
}
|
||||
}
|
||||
for (const inputPath of inputPaths) {
|
||||
const trimmed = inputPath.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
await walk(trimmed);
|
||||
}
|
||||
return Array.from(found).toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export async function previewGroundedRemMarkdown(params: {
|
||||
workspaceDir: string;
|
||||
inputPaths: string[];
|
||||
}): Promise<GroundedRemPreviewResult> {
|
||||
const workspaceDir = params.workspaceDir.trim();
|
||||
const files = await collectMarkdownFiles(params.inputPaths);
|
||||
const previews: GroundedRemFilePreview[] = [];
|
||||
for (const filePath of files) {
|
||||
const content = await fs.readFile(filePath, "utf-8");
|
||||
const relPath = normalizePath(path.relative(workspaceDir, filePath));
|
||||
previews.push(previewGroundedRemForFile({ relPath, content }));
|
||||
}
|
||||
return {
|
||||
workspaceDir,
|
||||
scannedFiles: files.length,
|
||||
files: previews,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user