mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-05 03:11:39 +00:00
99d8d90572
* refactor(memory-core): split CLI runtime * refactor(memory-core): narrow CLI helper exports * test(memory-core): follow secret resolver split
550 lines
19 KiB
TypeScript
550 lines
19 KiB
TypeScript
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { resolveMemoryRemDreamingConfig } from "openclaw/plugin-sdk/memory-core-host-status";
|
|
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
|
import {
|
|
emitMemorySecretResolveDiagnostics,
|
|
loadMemoryCommandConfig,
|
|
resolveAgent,
|
|
resolveMemoryPluginConfig,
|
|
withMemoryManagerForAgent,
|
|
} from "./cli-runtime-common.js";
|
|
import { defaultRuntime, shortenHomePath, theme } from "./cli.host.runtime.js";
|
|
import type { MemoryRemBackfillOptions, MemoryRemHarnessOptions } from "./cli.types.js";
|
|
import { removeBackfillDiaryEntries, writeBackfillDiaryEntries } from "./dreaming-narrative.js";
|
|
import { seedHistoricalDailyMemorySignals } from "./dreaming-phases.js";
|
|
import type { MemoryCoreRuntimeHost } from "./memory/runtime-host.js";
|
|
import { previewGroundedRemMarkdown } from "./rem-evidence.js";
|
|
import { previewRemHarness } from "./rem-harness.js";
|
|
import {
|
|
recordGroundedShortTermCandidates,
|
|
removeGroundedShortTermCandidates,
|
|
} from "./short-term-promotion.js";
|
|
const { heading, muted, warn } = theme;
|
|
export async function runMemoryRemHarness(
|
|
opts: MemoryRemHarnessOptions,
|
|
hostOptions?: MemoryCoreRuntimeHost,
|
|
) {
|
|
const { config: cfg, diagnostics } = await loadMemoryCommandConfig("memory rem-harness");
|
|
emitMemorySecretResolveDiagnostics(diagnostics, { json: Boolean(opts.json) });
|
|
const agentId = resolveAgent(cfg, opts.agent);
|
|
await withMemoryManagerForAgent({
|
|
cfg,
|
|
agentId,
|
|
purpose: "status",
|
|
acquireLocalService: hostOptions?.acquireLocalService,
|
|
withLease: hostOptions?.withLease,
|
|
run: async (manager) => {
|
|
const status = manager.status();
|
|
const managerWorkspaceDir = status.workspaceDir?.trim();
|
|
const pluginConfig = resolveMemoryPluginConfig(cfg);
|
|
if (!managerWorkspaceDir && !opts.path) {
|
|
defaultRuntime.error("Memory rem-harness requires a resolvable workspace directory.");
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
const remConfig = resolveMemoryRemDreamingConfig({
|
|
pluginConfig,
|
|
cfg,
|
|
});
|
|
const nowMs = Date.now();
|
|
let workspaceDir = managerWorkspaceDir ?? "";
|
|
let sourceFiles: string[] = [];
|
|
let groundedInputPaths: string[] = [];
|
|
let importedFileCount = 0;
|
|
let importedSignalCount = 0;
|
|
let skippedPaths: string[] = [];
|
|
let cleanupWorkspaceDir: string | null = null;
|
|
if (opts.path) {
|
|
const historical = await createHistoricalRemHarnessWorkspace({
|
|
inputPath: opts.path,
|
|
remLimit: remConfig.limit,
|
|
nowMs,
|
|
timezone: remConfig.timezone,
|
|
});
|
|
workspaceDir = historical.workspaceDir;
|
|
cleanupWorkspaceDir = historical.workspaceDir;
|
|
sourceFiles = historical.sourceFiles;
|
|
groundedInputPaths = historical.workspaceSourceFiles;
|
|
importedFileCount = historical.importedFileCount;
|
|
importedSignalCount = historical.importedSignalCount;
|
|
skippedPaths = historical.skippedPaths;
|
|
if (sourceFiles.length === 0) {
|
|
await fs.rm(historical.workspaceDir, { recursive: true, force: true });
|
|
defaultRuntime.error(
|
|
`Memory rem-harness found no YYYY-MM-DD.md files at ${shortenHomePath(path.resolve(opts.path))}.`,
|
|
);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
}
|
|
if (!workspaceDir) {
|
|
defaultRuntime.error("Memory rem-harness requires a resolvable workspace directory.");
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
try {
|
|
const preview = await previewRemHarness({
|
|
workspaceDir,
|
|
cfg,
|
|
pluginConfig,
|
|
grounded: Boolean(opts.grounded),
|
|
groundedInputPaths,
|
|
includePromoted: Boolean(opts.includePromoted),
|
|
nowMs,
|
|
});
|
|
groundedInputPaths = preview.groundedInputPaths;
|
|
const remPreview = preview.rem;
|
|
const groundedPreview = preview.grounded;
|
|
const deepCandidates = preview.deep.candidates;
|
|
if (opts.json) {
|
|
defaultRuntime.writeJson({
|
|
workspaceDir,
|
|
sourcePath: opts.path ? path.resolve(opts.path) : null,
|
|
sourceFiles,
|
|
historicalImport: opts.path
|
|
? {
|
|
importedFileCount,
|
|
importedSignalCount,
|
|
skippedPaths,
|
|
}
|
|
: null,
|
|
remConfig: preview.remConfig,
|
|
deepConfig: {
|
|
minScore: preview.deepConfig.minScore,
|
|
minRecallCount: preview.deepConfig.minRecallCount,
|
|
minUniqueQueries: preview.deepConfig.minUniqueQueries,
|
|
recencyHalfLifeDays: preview.deepConfig.recencyHalfLifeDays,
|
|
maxAgeDays: preview.deepConfig.maxAgeDays ?? null,
|
|
maxPromotedSnippetTokens: preview.deepConfig.maxPromotedSnippetTokens,
|
|
},
|
|
rem: { skipped: preview.remSkipped, ...remPreview },
|
|
grounded: groundedPreview,
|
|
deep: {
|
|
candidateCount: preview.deep.candidateCount,
|
|
candidates: deepCandidates,
|
|
},
|
|
});
|
|
return;
|
|
}
|
|
const lines = [
|
|
`${heading("REM Harness")} ${muted(`(${agentId})`)}`,
|
|
muted(`workspace=${shortenHomePath(workspaceDir)}`),
|
|
...(opts.path
|
|
? [
|
|
muted(`sourcePath=${shortenHomePath(path.resolve(opts.path))}`),
|
|
muted(
|
|
`historicalFiles=${sourceFiles.length} importedFiles=${importedFileCount} importedSignals=${importedSignalCount}`,
|
|
),
|
|
...(skippedPaths.length > 0
|
|
? [
|
|
warn(
|
|
`skipped=${skippedPaths.map((entry) => shortenHomePath(entry)).join(", ")}`,
|
|
),
|
|
]
|
|
: []),
|
|
]
|
|
: []),
|
|
...(opts.grounded
|
|
? [
|
|
muted(
|
|
`groundedInputs=${groundedInputPaths.length > 0 ? groundedInputPaths.map((entry) => shortenHomePath(entry)).join(", ") : "none"}`,
|
|
),
|
|
]
|
|
: []),
|
|
muted(
|
|
`recentRecallEntries=${preview.recallEntryCount} deepCandidates=${deepCandidates.length}`,
|
|
),
|
|
"",
|
|
heading("REM Preview"),
|
|
...remPreview.bodyLines,
|
|
...(groundedPreview
|
|
? [
|
|
"",
|
|
heading("Grounded REM"),
|
|
...groundedPreview.files.flatMap((file) => [
|
|
muted(file.path),
|
|
file.renderedMarkdown,
|
|
"",
|
|
]),
|
|
]
|
|
: []),
|
|
"",
|
|
heading("Deep Candidates"),
|
|
...(deepCandidates.length > 0
|
|
? deepCandidates
|
|
.slice(0, 10)
|
|
.map(
|
|
(candidate) =>
|
|
`${candidate.score.toFixed(3)} ${candidate.snippet} [${shortenHomePath(candidate.path)}:${candidate.startLine}-${candidate.endLine}]`,
|
|
)
|
|
: ["- No deep candidates."]),
|
|
];
|
|
defaultRuntime.log(lines.join("\n"));
|
|
} finally {
|
|
if (cleanupWorkspaceDir) {
|
|
await fs.rm(cleanupWorkspaceDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
},
|
|
});
|
|
}
|
|
export async function runMemoryRemBackfill(
|
|
opts: MemoryRemBackfillOptions,
|
|
hostOptions?: MemoryCoreRuntimeHost,
|
|
) {
|
|
const { config: cfg, diagnostics } = await loadMemoryCommandConfig("memory rem-backfill");
|
|
emitMemorySecretResolveDiagnostics(diagnostics, { json: Boolean(opts.json) });
|
|
const agentId = resolveAgent(cfg, opts.agent);
|
|
await withMemoryManagerForAgent({
|
|
cfg,
|
|
agentId,
|
|
purpose: "status",
|
|
acquireLocalService: hostOptions?.acquireLocalService,
|
|
withLease: hostOptions?.withLease,
|
|
run: async (manager) => {
|
|
const status = manager.status();
|
|
const workspaceDir = status.workspaceDir?.trim();
|
|
const pluginConfig = resolveMemoryPluginConfig(cfg);
|
|
const remConfig = resolveMemoryRemDreamingConfig({
|
|
pluginConfig,
|
|
cfg,
|
|
});
|
|
if (!workspaceDir) {
|
|
defaultRuntime.error("Memory rem-backfill requires a resolvable workspace directory.");
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
if (opts.rollback || opts.rollbackShortTerm) {
|
|
const diaryRollback = opts.rollback
|
|
? await removeBackfillDiaryEntries({ workspaceDir })
|
|
: null;
|
|
const shortTermRollback = opts.rollbackShortTerm
|
|
? await removeGroundedShortTermCandidates({ workspaceDir })
|
|
: null;
|
|
if (opts.json) {
|
|
defaultRuntime.writeJson({
|
|
workspaceDir,
|
|
rollback: Boolean(opts.rollback),
|
|
rollbackShortTerm: Boolean(opts.rollbackShortTerm),
|
|
...(diaryRollback
|
|
? {
|
|
dreamsPath: diaryRollback.dreamsPath,
|
|
removedEntries: diaryRollback.removed,
|
|
}
|
|
: {}),
|
|
...(shortTermRollback
|
|
? {
|
|
shortTermStorePath: shortTermRollback.storePath,
|
|
removedShortTermEntries: shortTermRollback.removed,
|
|
}
|
|
: {}),
|
|
});
|
|
return;
|
|
}
|
|
defaultRuntime.log(
|
|
[
|
|
`${heading("REM Backfill")} ${muted("(rollback)")}`,
|
|
muted(`workspace=${shortenHomePath(workspaceDir)}`),
|
|
...(diaryRollback
|
|
? [
|
|
muted(`dreamsPath=${shortenHomePath(diaryRollback.dreamsPath)}`),
|
|
muted(`removedEntries=${diaryRollback.removed}`),
|
|
]
|
|
: []),
|
|
...(shortTermRollback
|
|
? [
|
|
muted(`shortTermStorePath=${shortenHomePath(shortTermRollback.storePath)}`),
|
|
muted(`removedShortTermEntries=${shortTermRollback.removed}`),
|
|
]
|
|
: []),
|
|
].join("\n"),
|
|
);
|
|
return;
|
|
}
|
|
if (!opts.path) {
|
|
defaultRuntime.error(
|
|
"Memory rem-backfill requires --path <file-or-dir> unless using --rollback.",
|
|
);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
const scratchDir = await fs.mkdtemp(
|
|
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-rem-backfill-"),
|
|
);
|
|
try {
|
|
const sourceFiles = await listHistoricalDailyFiles(opts.path);
|
|
if (sourceFiles.length === 0) {
|
|
defaultRuntime.error(
|
|
`Memory rem-backfill found no YYYY-MM-DD.md files at ${shortenHomePath(path.resolve(opts.path))}.`,
|
|
);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
const scratchMemoryDir = path.join(scratchDir, "memory");
|
|
await fs.mkdir(scratchMemoryDir, { recursive: true });
|
|
const workspaceSourceFiles: string[] = [];
|
|
for (const filePath of sourceFiles) {
|
|
const dst = path.join(scratchMemoryDir, path.basename(filePath));
|
|
await fs.copyFile(filePath, dst);
|
|
workspaceSourceFiles.push(dst);
|
|
}
|
|
const grounded = await previewGroundedRemMarkdown({
|
|
workspaceDir: scratchDir,
|
|
inputPaths: workspaceSourceFiles,
|
|
});
|
|
const sourcePathByScratchRelativePath = new Map(
|
|
workspaceSourceFiles.map(
|
|
(scratchPath, index) =>
|
|
[
|
|
normalizeRelativePath(scratchDir, scratchPath),
|
|
sourceFiles[index] ?? scratchPath,
|
|
] as const,
|
|
),
|
|
);
|
|
const entries = grounded.files
|
|
.map((file) => {
|
|
const isoDay = extractIsoDayFromPath(file.path);
|
|
if (!isoDay) {
|
|
return null;
|
|
}
|
|
return {
|
|
isoDay,
|
|
sourcePath: sourcePathByScratchRelativePath.get(file.path) ?? file.path,
|
|
bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown),
|
|
};
|
|
})
|
|
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
|
|
const written = await writeBackfillDiaryEntries({
|
|
workspaceDir,
|
|
entries,
|
|
timezone: remConfig.timezone,
|
|
});
|
|
let stagedShortTermEntries = 0;
|
|
let replacedShortTermEntries = 0;
|
|
if (opts.stageShortTerm) {
|
|
const cleared = await removeGroundedShortTermCandidates({ workspaceDir });
|
|
replacedShortTermEntries = cleared.removed;
|
|
const shortTermSeedItems = collectGroundedShortTermSeedItems(grounded.files);
|
|
if (shortTermSeedItems.length > 0) {
|
|
await recordGroundedShortTermCandidates({
|
|
workspaceDir,
|
|
query: "__dreaming_grounded_backfill__",
|
|
items: shortTermSeedItems,
|
|
dedupeByQueryPerDay: true,
|
|
nowMs: Date.now(),
|
|
timezone: remConfig.timezone,
|
|
});
|
|
}
|
|
stagedShortTermEntries = shortTermSeedItems.length;
|
|
}
|
|
if (opts.json) {
|
|
defaultRuntime.writeJson({
|
|
workspaceDir,
|
|
sourcePath: path.resolve(opts.path),
|
|
sourceFiles,
|
|
groundedFiles: grounded.scannedFiles,
|
|
writtenEntries: written.written,
|
|
replacedEntries: written.replaced,
|
|
dreamsPath: written.dreamsPath,
|
|
...(opts.stageShortTerm
|
|
? {
|
|
stagedShortTermEntries,
|
|
replacedShortTermEntries,
|
|
}
|
|
: {}),
|
|
});
|
|
return;
|
|
}
|
|
defaultRuntime.log(
|
|
[
|
|
`${heading("REM Backfill")} ${muted(`(${agentId})`)}`,
|
|
muted(`workspace=${shortenHomePath(workspaceDir)}`),
|
|
muted(`sourcePath=${shortenHomePath(path.resolve(opts.path))}`),
|
|
muted(
|
|
`historicalFiles=${sourceFiles.length} writtenEntries=${written.written} replacedEntries=${written.replaced}`,
|
|
),
|
|
...(opts.stageShortTerm
|
|
? [
|
|
muted(
|
|
`stagedShortTermEntries=${stagedShortTermEntries} replacedShortTermEntries=${replacedShortTermEntries}`,
|
|
),
|
|
]
|
|
: []),
|
|
muted(`dreamsPath=${shortenHomePath(written.dreamsPath)}`),
|
|
].join("\n"),
|
|
);
|
|
} finally {
|
|
await fs.rm(scratchDir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
});
|
|
}
|
|
const DAILY_MEMORY_FILE_NAME_RE = /^(\d{4}-\d{2}-\d{2})(?:-[^/]+)?\.md$/i;
|
|
async function listHistoricalDailyFiles(inputPath: string): Promise<string[]> {
|
|
const resolvedPath = path.resolve(inputPath);
|
|
let stat;
|
|
try {
|
|
stat = await fs.stat(resolvedPath);
|
|
} catch (err) {
|
|
if ((err as NodeJS.ErrnoException | undefined)?.code === "ENOENT") {
|
|
return [];
|
|
}
|
|
throw err;
|
|
}
|
|
if (stat.isFile()) {
|
|
return DAILY_MEMORY_FILE_NAME_RE.test(path.basename(resolvedPath)) ? [resolvedPath] : [];
|
|
}
|
|
if (!stat.isDirectory()) {
|
|
return [];
|
|
}
|
|
const entries = await fs.readdir(resolvedPath, { withFileTypes: true });
|
|
return entries
|
|
.filter((entry) => entry.isFile() && DAILY_MEMORY_FILE_NAME_RE.test(entry.name))
|
|
.map((entry) => path.join(resolvedPath, entry.name))
|
|
.toSorted((a, b) => path.basename(a).localeCompare(path.basename(b)));
|
|
}
|
|
async function createHistoricalRemHarnessWorkspace(params: {
|
|
inputPath: string;
|
|
remLimit: number;
|
|
nowMs: number;
|
|
timezone?: string;
|
|
}): Promise<{
|
|
workspaceDir: string;
|
|
sourceFiles: string[];
|
|
workspaceSourceFiles: string[];
|
|
importedFileCount: number;
|
|
importedSignalCount: number;
|
|
skippedPaths: string[];
|
|
}> {
|
|
const sourceFiles = await listHistoricalDailyFiles(params.inputPath);
|
|
const workspaceDir = await fs.mkdtemp(
|
|
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-rem-harness-"),
|
|
);
|
|
const memoryDir = path.join(workspaceDir, "memory");
|
|
await fs.mkdir(memoryDir, { recursive: true });
|
|
for (const filePath of sourceFiles) {
|
|
await fs.copyFile(filePath, path.join(memoryDir, path.basename(filePath)));
|
|
}
|
|
const workspaceSourceFiles = sourceFiles.map((entry) =>
|
|
path.join(memoryDir, path.basename(entry)),
|
|
);
|
|
const seeded = await seedHistoricalDailyMemorySignals({
|
|
workspaceDir,
|
|
filePaths: workspaceSourceFiles,
|
|
limit: params.remLimit,
|
|
nowMs: params.nowMs,
|
|
timezone: params.timezone,
|
|
});
|
|
return {
|
|
workspaceDir,
|
|
sourceFiles,
|
|
workspaceSourceFiles,
|
|
importedFileCount: seeded.importedFileCount,
|
|
importedSignalCount: seeded.importedSignalCount,
|
|
skippedPaths: seeded.skippedPaths,
|
|
};
|
|
}
|
|
function extractIsoDayFromPath(filePath: string): string | null {
|
|
const match = path.basename(filePath).match(DAILY_MEMORY_FILE_NAME_RE);
|
|
return match?.[1] ?? null;
|
|
}
|
|
function normalizeRelativePath(baseDir: string, filePath: string): string {
|
|
return path.relative(baseDir, filePath).replace(/\\/g, "/");
|
|
}
|
|
function groundedMarkdownToDiaryLines(markdown: string): string[] {
|
|
return markdown
|
|
.split(/\r?\n/)
|
|
.map((line) => line.replace(/^##\s+/, "").trimEnd())
|
|
.filter((line, index, lines) => !(line.length === 0 && lines[index - 1]?.length === 0));
|
|
}
|
|
function parseGroundedRef(
|
|
fallbackPath: string,
|
|
ref: string,
|
|
): { path: string; startLine: number; endLine: number } | null {
|
|
const trimmed = ref.trim();
|
|
if (!trimmed) {
|
|
return null;
|
|
}
|
|
const match = trimmed.match(/^(.*?):(\d+)(?:-(\d+))?$/);
|
|
if (!match) {
|
|
return null;
|
|
}
|
|
return {
|
|
path: (match[1] ?? fallbackPath).replaceAll("\\", "/").replace(/^\.\//, ""),
|
|
startLine: Math.max(1, Number(match[2])),
|
|
endLine: Math.max(1, Number(match[3] ?? match[2])),
|
|
};
|
|
}
|
|
function collectGroundedShortTermSeedItems(
|
|
previews: Awaited<ReturnType<typeof previewGroundedRemMarkdown>>["files"],
|
|
): Array<{
|
|
path: string;
|
|
startLine: number;
|
|
endLine: number;
|
|
snippet: string;
|
|
score: number;
|
|
query: string;
|
|
signalCount: number;
|
|
dayBucket?: string;
|
|
}> {
|
|
const items: Array<{
|
|
path: string;
|
|
startLine: number;
|
|
endLine: number;
|
|
snippet: string;
|
|
score: number;
|
|
query: string;
|
|
signalCount: number;
|
|
dayBucket?: string;
|
|
}> = [];
|
|
const seen = new Set<string>();
|
|
for (const file of previews) {
|
|
const dayBucket = extractIsoDayFromPath(file.path) ?? undefined;
|
|
const signals = [
|
|
...file.memoryImplications.map((item) => ({
|
|
text: item.text,
|
|
refs: item.refs,
|
|
score: 0.92,
|
|
query: "__dreaming_grounded_backfill__:lasting-update",
|
|
signalCount: 2,
|
|
})),
|
|
...file.candidates
|
|
.filter((candidate) => candidate.lean === "likely_durable")
|
|
.map((candidate) => ({
|
|
text: candidate.text,
|
|
refs: candidate.refs,
|
|
score: 0.82,
|
|
query: "__dreaming_grounded_backfill__:candidate",
|
|
signalCount: 1,
|
|
})),
|
|
];
|
|
for (const signal of signals) {
|
|
if (!signal.text.trim()) {
|
|
continue;
|
|
}
|
|
const firstRef = signal.refs.find((ref) => ref.trim().length > 0);
|
|
const parsedRef = firstRef ? parseGroundedRef(file.path, firstRef) : null;
|
|
if (!parsedRef) {
|
|
continue;
|
|
}
|
|
const key = `${parsedRef.path}:${parsedRef.startLine}:${parsedRef.endLine}:${signal.query}:${signal.text.toLowerCase()}`;
|
|
if (seen.has(key)) {
|
|
continue;
|
|
}
|
|
seen.add(key);
|
|
items.push({
|
|
path: parsedRef.path,
|
|
startLine: parsedRef.startLine,
|
|
endLine: parsedRef.endLine,
|
|
snippet: signal.text,
|
|
score: signal.score,
|
|
query: signal.query,
|
|
signalCount: signal.signalCount,
|
|
...(dayBucket ? { dayBucket } : {}),
|
|
});
|
|
}
|
|
}
|
|
return items;
|
|
}
|