feat(memory): harden dreaming and multilingual memory promotion (#60697)

* feat(memory): add recall audit and doctor repair flow

* refactor(memory): rename symbolic scoring and harden dreaming

* feat(memory): add multilingual concept vocabulary

* docs(changelog): note dreaming memory follow-up

* docs(changelog): shorten dreaming follow-up entry

* fix(memory): address review follow-ups

* chore(skills): tighten security triage trust model

* Update CHANGELOG.md
This commit is contained in:
Vincent Koc
2026-04-04 15:48:13 +09:00
committed by GitHub
parent 0ab160cda9
commit 0609bf8581
19 changed files with 2308 additions and 56 deletions

View File

@@ -55,6 +55,8 @@ Check in this order:
- Was it fixed before release?
3. Exploit path
- Does the report show a real boundary bypass, not just prompt injection, local same-user control, or helper-level semantics?
- If data only moves between trusted workspace-memory files called out in `SECURITY.md`, do not treat "injection markers" alone as a security bug.
- In that case, frame sanitization as optional hardening only if it preserves expected memory workflows.
4. Functional tradeoff
- If a hardening change would reduce intended user functionality, call that out before proposing it.
- Prefer fixes that preserve user workflows over deny-by-default regressions unless the boundary demands it.
@@ -104,5 +106,6 @@ gh search prs --repo openclaw/openclaw --match title,body,comments -- "<terms>"
- “fixed on main, unreleased” is usually not a close.
- “needs attacker-controlled trusted local state first” is usually out of scope.
- “same-host same-user process can already read/write local state” is usually out of scope.
- “trusted workspace memory promotes/reindexes trusted workspace memory” is usually out of scope unless it crosses a documented boundary.
- “helper function behaves differently than documented config semantics” is usually invalid.
- If only the severity is wrong but the bug is real, keep it open and narrow the impact in the reply.

View File

@@ -25,8 +25,8 @@ Docs: https://docs.openclaw.ai
- Tests/runtime: trim local unit-test import/runtime fan-out across browser, WhatsApp, cron, task, and reply flows so owner suites start faster with lower shared-worker overhead while preserving the same focused behavior coverage. (#60249) Thanks @shakkernerd.
- Tests/secrets runtime: restore split secrets suite cache and env isolation cleanup so broader runs do not leak stale plugin or provider snapshot state. (#60395) Thanks @shakkernerd.
- Memory/dreaming (experimental): add opt-in weighted short-term recall promotion to `MEMORY.md`, managed dreaming modes (`off|core|rem|deep`), and a `/dreaming` command plus Dreams UI so durable memory promotion can run on background cadence without manual scheduling. (#60569) Thanks @vignesh07.
- Memory/dreaming (experimental): add follow-up dreaming hardening, multilingual conceptual tagging, and doctor/status repair support. Thanks @vincentkoc. (#60697)
- Agents/system prompts: add an internal cache-prefix boundary across Anthropic-family, OpenAI-family, Google, and CLI transport shaping so stable system-prompt prefixes stay reusable without leaking internal cache markers to provider payloads. (#59054) Thanks @coletebou and @vincentkoc.
- Docs/memory: add a dedicated Dreaming concept page, expand Memory overview with the Dreaming model, and link Dreaming from further reading to document the experimental opt-in consolidation workflow. Thanks @vignesh07.
- Agents/cache prefixes: route compaction, OpenAI WebSocket HTTP fallback, and later-turn embedded session reuse through the same cache-safe prompt shaping path so Anthropic-family and OpenAI-family requests keep stable prompt bytes across follow-up turns and fallback transport changes. (#60691) Thanks @vincentkoc.
- Agents/Claude CLI: expose OpenClaw tools to background Claude CLI runs through a loopback MCP bridge that reuses gateway tool policy, honors session/account/channel scoping, and only advertises the bridge when the local runtime is actually live. (#35676) Thanks @mylukin.
- Providers/Anthropic: remove setup-token from new onboarding and auth-command setup paths, keep existing configured legacy token profiles runnable, and steer new Anthropic setup to Claude CLI or API keys.

View File

@@ -12,6 +12,11 @@
"placeholder": "0 3 * * *",
"help": "Optional cron cadence override for managed dreaming runs."
},
"dreaming.cron": {
"label": "Dreaming Cron",
"placeholder": "0 3 * * *",
"help": "Optional cron cadence override for managed dreaming runs."
},
"dreaming.timezone": {
"label": "Dreaming Timezone",
"placeholder": "America/Los_Angeles",
@@ -53,6 +58,9 @@
"frequency": {
"type": "string"
},
"cron": {
"type": "string"
},
"timezone": {
"type": "string"
},

View File

@@ -3,4 +3,12 @@ export {
getBuiltinMemoryEmbeddingProviderDoctorMetadata,
listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata,
} from "./src/memory/provider-adapters.js";
export {
auditShortTermPromotionArtifacts,
repairShortTermPromotionArtifacts,
} from "./src/short-term-promotion.js";
export type { BuiltinMemoryEmbeddingProviderDoctorMetadata } from "./src/memory/provider-adapters.js";
export type {
RepairShortTermPromotionArtifactsResult,
ShortTermAuditSummary,
} from "./src/short-term-promotion.js";

View File

@@ -30,11 +30,17 @@ import type {
MemoryPromoteCommandOptions,
MemorySearchCommandOptions,
} from "./cli.types.js";
import { resolveShortTermPromotionDreamingConfig } from "./dreaming.js";
import {
applyShortTermPromotions,
auditShortTermPromotionArtifacts,
repairShortTermPromotionArtifacts,
recordShortTermRecalls,
rankShortTermPromotionCandidates,
resolveShortTermRecallLockPath,
resolveShortTermRecallStorePath,
type RepairShortTermPromotionArtifactsResult,
type ShortTermAuditSummary,
} from "./short-term-promotion.js";
type MemoryManager = NonNullable<Awaited<ReturnType<typeof getMemorySearchManager>>["manager"]>;
@@ -59,6 +65,13 @@ type LoadedMemoryCommandConfig = {
diagnostics: string[];
};
function asRecord(value: unknown): Record<string, unknown> | null {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return null;
}
return value as Record<string, unknown>;
}
function getMemoryCommandSecretTargetIds(): Set<string> {
return new Set([
"agents.defaults.memorySearch.remote.apiKey",
@@ -96,6 +109,57 @@ function emitMemorySecretResolveDiagnostics(
}
}
function resolveMemoryPluginConfig(cfg: OpenClawConfig): Record<string, unknown> {
const entry = asRecord(cfg.plugins?.entries?.["memory-core"]);
return asRecord(entry?.config) ?? {};
}
function formatDreamingSummary(cfg: OpenClawConfig): string {
const pluginConfig = resolveMemoryPluginConfig(cfg);
const dreaming = resolveShortTermPromotionDreamingConfig({ pluginConfig, cfg });
if (!dreaming.enabled) {
return "off";
}
const timezone = dreaming.timezone ? ` (${dreaming.timezone})` : "";
return `${dreaming.cron}${timezone} · limit=${dreaming.limit} · minScore=${dreaming.minScore} · minRecallCount=${dreaming.minRecallCount} · minUniqueQueries=${dreaming.minUniqueQueries}`;
}
function formatAuditCounts(audit: ShortTermAuditSummary): string {
const scriptCoverage = audit.conceptTagScripts
? [
audit.conceptTagScripts.latinEntryCount > 0
? `${audit.conceptTagScripts.latinEntryCount} latin`
: null,
audit.conceptTagScripts.cjkEntryCount > 0
? `${audit.conceptTagScripts.cjkEntryCount} cjk`
: null,
audit.conceptTagScripts.mixedEntryCount > 0
? `${audit.conceptTagScripts.mixedEntryCount} mixed`
: null,
audit.conceptTagScripts.otherEntryCount > 0
? `${audit.conceptTagScripts.otherEntryCount} other`
: null,
]
.filter(Boolean)
.join(", ")
: "";
const suffix = scriptCoverage ? ` · scripts=${scriptCoverage}` : "";
return `${audit.entryCount} entries · ${audit.promotedCount} promoted · ${audit.conceptTaggedEntryCount} concept-tagged · ${audit.spacedEntryCount} spaced${suffix}`;
}
function formatRepairSummary(repair: RepairShortTermPromotionArtifactsResult): string {
const actions: string[] = [];
if (repair.rewroteStore) {
actions.push(
`rewrote store${repair.removedInvalidEntries > 0 ? ` (-${repair.removedInvalidEntries} invalid)` : ""}`,
);
}
if (repair.removedStaleLock) {
actions.push("removed stale lock");
}
return actions.length > 0 ? actions.join(" · ") : "no changes";
}
function formatSourceLabel(source: string, workspaceDir: string, agentId: string): string {
if (source === "memory") {
return shortenHomeInString(
@@ -367,6 +431,8 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
embeddingProbe?: Awaited<ReturnType<MemoryManager["probeEmbeddingAvailability"]>>;
indexError?: string;
scan?: MemorySourceScan;
audit?: ShortTermAuditSummary;
repair?: RepairShortTermPromotionArtifactsResult;
}> = [];
for (const agentId of agentIds) {
@@ -440,7 +506,28 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
extraPaths: status.extraPaths,
})
: undefined;
allResults.push({ agentId, status, embeddingProbe, indexError, scan });
let audit: ShortTermAuditSummary | undefined;
let repair: RepairShortTermPromotionArtifactsResult | undefined;
if (workspaceDir) {
if (opts.fix) {
repair = await repairShortTermPromotionArtifacts({ workspaceDir });
}
const customQmd = asRecord(asRecord(status.custom)?.qmd);
audit = await auditShortTermPromotionArtifacts({
workspaceDir,
qmd:
status.backend === "qmd"
? {
dbPath: status.dbPath,
collections:
typeof customQmd?.collections === "number"
? customQmd.collections
: undefined,
}
: undefined,
});
}
allResults.push({ agentId, status, embeddingProbe, indexError, scan, audit, repair });
},
});
}
@@ -460,7 +547,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
const label = (text: string) => muted(`${text}:`);
for (const result of allResults) {
const { agentId, status, embeddingProbe, indexError, scan } = result;
const { agentId, status, embeddingProbe, indexError, scan, audit, repair } = result;
const filesIndexed = status.files ?? 0;
const chunksIndexed = status.chunks ?? 0;
const totalFiles = scan?.totalFiles ?? null;
@@ -490,6 +577,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
`${label("Dirty")} ${status.dirty ? warn("yes") : muted("no")}`,
`${label("Store")} ${info(storePath)}`,
`${label("Workspace")} ${info(workspacePath)}`,
`${label("Dreaming")} ${info(formatDreamingSummary(cfg))}`,
].filter(Boolean) as string[];
if (embeddingProbe) {
const state = embeddingProbe.ok ? "ready" : "unavailable";
@@ -580,6 +668,24 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
lines.push(`${label("Batch error")} ${warn(status.batch.lastError)}`);
}
}
if (audit) {
lines.push(`${label("Recall store")} ${info(formatAuditCounts(audit))}`);
lines.push(`${label("Recall path")} ${info(shortenHomePath(audit.storePath))}`);
if (audit.updatedAt) {
lines.push(`${label("Recall updated")} ${info(audit.updatedAt)}`);
}
if (status.backend === "qmd" && audit.qmd) {
const qmdBits = [
audit.qmd.dbPath ? shortenHomePath(audit.qmd.dbPath) : "<unknown>",
typeof audit.qmd.dbBytes === "number" ? `${audit.qmd.dbBytes} bytes` : null,
typeof audit.qmd.collections === "number" ? `${audit.qmd.collections} collections` : null,
].filter(Boolean);
lines.push(`${label("QMD audit")} ${info(qmdBits.join(" · "))}`);
}
}
if (repair) {
lines.push(`${label("Repair")} ${info(formatRepairSummary(repair))}`);
}
if (status.fallback?.reason) {
lines.push(muted(status.fallback.reason));
}
@@ -592,6 +698,17 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
lines.push(` ${warn(issue)}`);
}
}
if (audit?.issues.length) {
if (!scan?.issues.length) {
lines.push(label("Issues"));
}
for (const issue of audit.issues) {
lines.push(` ${issue.severity === "error" ? warn(issue.message) : muted(issue.message)}`);
}
if (!opts.fix) {
lines.push(` ${muted(`Fix: openclaw memory status --fix --agent ${agentId}`)}`);
}
}
defaultRuntime.log(lines.join("\n"));
defaultRuntime.log("");
}
@@ -846,11 +963,26 @@ export async function runMemoryPromote(opts: MemoryPromoteCommandOptions) {
}
const storePath = resolveShortTermRecallStorePath(workspaceDir);
const lockPath = resolveShortTermRecallLockPath(workspaceDir);
const customQmd = asRecord(asRecord(status.custom)?.qmd);
const audit = await auditShortTermPromotionArtifacts({
workspaceDir,
qmd:
status.backend === "qmd"
? {
dbPath: status.dbPath,
collections:
typeof customQmd?.collections === "number" ? customQmd.collections : undefined,
}
: undefined,
});
if (opts.json) {
defaultRuntime.writeJson({
workspaceDir,
storePath,
lockPath,
audit,
candidates,
apply: applyResult
? {
@@ -866,6 +998,11 @@ export async function runMemoryPromote(opts: MemoryPromoteCommandOptions) {
if (candidates.length === 0) {
defaultRuntime.log("No short-term recall candidates.");
defaultRuntime.log(`Recall store: ${shortenHomePath(storePath)}`);
if (audit.issues.length > 0) {
for (const issue of audit.issues) {
defaultRuntime.log(issue.message);
}
}
return;
}
@@ -879,6 +1016,7 @@ export async function runMemoryPromote(opts: MemoryPromoteCommandOptions) {
)}`,
);
lines.push(`${colorize(rich, theme.muted, "Recall store:")} ${shortenHomePath(storePath)}`);
lines.push(colorize(rich, theme.muted, `Store health: ${formatAuditCounts(audit)}`));
for (const candidate of candidates) {
lines.push(
`${colorize(rich, theme.success, candidate.score.toFixed(3))} ${colorize(
@@ -891,14 +1029,26 @@ export async function runMemoryPromote(opts: MemoryPromoteCommandOptions) {
colorize(
rich,
theme.muted,
`recalls=${candidate.recallCount} avg=${candidate.avgScore.toFixed(3)} queries=${candidate.uniqueQueries} age=${candidate.ageDays.toFixed(1)}d`,
`recalls=${candidate.recallCount} avg=${candidate.avgScore.toFixed(3)} queries=${candidate.uniqueQueries} age=${candidate.ageDays.toFixed(1)}d consolidate=${candidate.components.consolidation.toFixed(2)} conceptual=${candidate.components.conceptual.toFixed(2)}`,
),
);
if (candidate.conceptTags.length > 0) {
lines.push(colorize(rich, theme.muted, `concepts=${candidate.conceptTags.join(", ")}`));
}
if (candidate.snippet) {
lines.push(colorize(rich, theme.muted, candidate.snippet));
}
lines.push("");
}
if (audit.issues.length > 0) {
lines.push(colorize(rich, theme.warn, "Audit issues:"));
for (const issue of audit.issues) {
lines.push(
colorize(rich, issue.severity === "error" ? theme.warn : theme.muted, issue.message),
);
}
lines.push("");
}
if (applyResult) {
if (applyResult.applied > 0) {
lines.push(

View File

@@ -275,6 +275,8 @@ describe("memory cli", () => {
it("documents memory help examples", () => {
const helpText = getMemoryHelpText();
expect(helpText).toContain("openclaw memory status --fix");
expect(helpText).toContain("Repair stale recall locks and normalize promotion metadata.");
expect(helpText).toContain("openclaw memory status --deep");
expect(helpText).toContain("Probe embedding provider readiness.");
expect(helpText).toContain('openclaw memory search "meeting notes"');
@@ -327,6 +329,134 @@ describe("memory cli", () => {
expect(close).toHaveBeenCalled();
});
it("prints recall-store audit details during status", async () => {
await withTempWorkspace(async (workspaceDir) => {
await recordShortTermRecalls({
workspaceDir,
query: "router vlan",
results: [
{
path: "memory/2026-04-03.md",
startLine: 1,
endLine: 3,
score: 0.93,
snippet: "Configured router VLAN 10 for IoT clients.",
source: "memory",
},
],
});
const close = vi.fn(async () => {});
mockManager({
probeVectorAvailability: vi.fn(async () => true),
status: () => makeMemoryStatus({ workspaceDir }),
close,
});
const log = spyRuntimeLogs(defaultRuntime);
await runMemoryCli(["status"]);
expect(log).toHaveBeenCalledWith(expect.stringContaining("Recall store: 1 entries"));
expect(log).toHaveBeenCalledWith(expect.stringContaining("Dreaming: off"));
expect(close).toHaveBeenCalled();
});
});
it("repairs invalid recall metadata and stale locks with status --fix", async () => {
await withTempWorkspace(async (workspaceDir) => {
const storePath = path.join(workspaceDir, "memory", ".dreams", "short-term-recall.json");
await fs.mkdir(path.dirname(storePath), { recursive: true });
await fs.writeFile(
storePath,
JSON.stringify(
{
version: 1,
updatedAt: "2026-04-04T00:00:00.000Z",
entries: {
good: {
key: "good",
path: "memory/2026-04-03.md",
startLine: 1,
endLine: 2,
source: "memory",
snippet: "QMD router cache note",
recallCount: 1,
totalScore: 0.8,
maxScore: 0.8,
firstRecalledAt: "2026-04-04T00:00:00.000Z",
lastRecalledAt: "2026-04-04T00:00:00.000Z",
queryHashes: ["a"],
},
bad: {
path: "",
},
},
},
null,
2,
),
"utf-8",
);
const lockPath = path.join(workspaceDir, "memory", ".dreams", "short-term-promotion.lock");
await fs.writeFile(lockPath, "999999:0\n", "utf-8");
const staleMtime = new Date(Date.now() - 120_000);
await fs.utimes(lockPath, staleMtime, staleMtime);
const close = vi.fn(async () => {});
mockManager({
probeVectorAvailability: vi.fn(async () => true),
status: () => makeMemoryStatus({ workspaceDir }),
close,
});
const log = spyRuntimeLogs(defaultRuntime);
await runMemoryCli(["status", "--fix"]);
expect(log).toHaveBeenCalledWith(expect.stringContaining("Repair: rewrote store"));
await expect(fs.stat(lockPath)).rejects.toThrow();
const repaired = JSON.parse(await fs.readFile(storePath, "utf-8")) as {
entries: Record<string, { conceptTags?: string[] }>;
};
expect(repaired.entries.good?.conceptTags).toContain("router");
expect(close).toHaveBeenCalled();
});
});
it("shows the fix hint only before --fix has been run", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-cli-fix-hint-"));
try {
const storePath = path.join(workspaceDir, "memory", ".dreams", "short-term-recall.json");
await fs.mkdir(path.dirname(storePath), { recursive: true });
await fs.writeFile(storePath, " \n", "utf-8");
const close = vi.fn(async () => {});
mockManager({
probeVectorAvailability: vi.fn(async () => true),
status: () => makeMemoryStatus({ workspaceDir }),
close,
});
const log = spyRuntimeLogs(defaultRuntime);
await runMemoryCli(["status"]);
expect(log).toHaveBeenCalledWith(
expect.stringContaining("Fix: openclaw memory status --fix --agent main"),
);
log.mockClear();
mockManager({
probeVectorAvailability: vi.fn(async () => true),
status: () => makeMemoryStatus({ workspaceDir }),
close,
});
await runMemoryCli(["status", "--fix"]);
expect(log).not.toHaveBeenCalledWith(
expect.stringContaining("Fix: openclaw memory status --fix --agent main"),
);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("enables verbose logging with --verbose", async () => {
const close = vi.fn(async () => {});
mockManager({
@@ -399,6 +529,36 @@ describe("memory cli", () => {
});
});
it("surfaces qmd audit details in status output", async () => {
const close = vi.fn(async () => {});
await withQmdIndexDb("sqlite-bytes", async (dbPath) => {
mockManager({
probeVectorAvailability: vi.fn(async () => true),
status: () =>
makeMemoryStatus({
backend: "qmd",
provider: "qmd",
model: "qmd",
requestedProvider: "qmd",
dbPath,
custom: {
qmd: {
collections: 2,
},
},
}),
close,
});
const log = spyRuntimeLogs(defaultRuntime);
await runMemoryCli(["status"]);
expect(log).toHaveBeenCalledWith(expect.stringContaining("QMD audit:"));
expect(log).toHaveBeenCalledWith(expect.stringContaining("2 collections"));
expect(close).toHaveBeenCalled();
});
});
it("fails index when qmd db file is empty", async () => {
const close = vi.fn(async () => {});
const sync = vi.fn(async () => {});
@@ -754,4 +914,60 @@ describe("memory cli", () => {
expect(close).toHaveBeenCalled();
});
});
it("prints conceptual promotion signals", async () => {
await withTempWorkspace(async (workspaceDir) => {
await recordShortTermRecalls({
workspaceDir,
query: "router vlan",
nowMs: Date.parse("2026-04-01T00:00:00.000Z"),
results: [
{
path: "memory/2026-04-01.md",
startLine: 4,
endLine: 8,
score: 0.9,
snippet: "Configured router VLAN 10 and Glacier backup notes for QMD.",
source: "memory",
},
],
});
await recordShortTermRecalls({
workspaceDir,
query: "glacier backup",
nowMs: Date.parse("2026-04-03T00:00:00.000Z"),
results: [
{
path: "memory/2026-04-01.md",
startLine: 4,
endLine: 8,
score: 0.88,
snippet: "Configured router VLAN 10 and Glacier backup notes for QMD.",
source: "memory",
},
],
});
const close = vi.fn(async () => {});
mockManager({
status: () => makeMemoryStatus({ workspaceDir }),
close,
});
const log = spyRuntimeLogs(defaultRuntime);
await runMemoryCli([
"promote",
"--min-score",
"0",
"--min-recall-count",
"0",
"--min-unique-queries",
"0",
]);
expect(log).toHaveBeenCalledWith(expect.stringContaining("consolidate="));
expect(log).toHaveBeenCalledWith(expect.stringContaining("concepts="));
expect(close).toHaveBeenCalled();
});
});
});

View File

@@ -53,6 +53,10 @@ export function registerMemoryCli(program: Command) {
() =>
`\n${theme.heading("Examples:")}\n${formatHelpExamples([
["openclaw memory status", "Show index and provider status."],
[
"openclaw memory status --fix",
"Repair stale recall locks and normalize promotion metadata.",
],
["openclaw memory status --deep", "Probe embedding provider readiness."],
["openclaw memory index --force", "Force a full reindex."],
['openclaw memory search "meeting notes"', "Quick search using positional query."],
@@ -79,6 +83,7 @@ export function registerMemoryCli(program: Command) {
.option("--json", "Print JSON")
.option("--deep", "Probe embedding provider availability")
.option("--index", "Reindex if dirty (implies --deep)")
.option("--fix", "Repair stale recall locks and normalize promotion metadata")
.option("--verbose", "Verbose logging", false)
.action(async (opts: MemoryCommandOptions & { force?: boolean }) => {
await runMemoryStatus(opts);

View File

@@ -4,6 +4,7 @@ export type MemoryCommandOptions = {
deep?: boolean;
index?: boolean;
force?: boolean;
fix?: boolean;
verbose?: boolean;
};

View File

@@ -0,0 +1,77 @@
import { describe, expect, it } from "vitest";
import {
classifyConceptTagScript,
deriveConceptTags,
summarizeConceptTagScriptCoverage,
} from "./concept-vocabulary.js";
describe("concept vocabulary", () => {
it("extracts Unicode-aware concept tags for common European languages", () => {
const tags = deriveConceptTags({
path: "memory/2026-04-04.md",
snippet:
"Configuración de gateway, configuration du routeur, Sicherung und Überwachung Glacier.",
});
expect(tags).toEqual(
expect.arrayContaining([
"gateway",
"configuración",
"configuration",
"routeur",
"sicherung",
"überwachung",
"glacier",
]),
);
expect(tags).not.toContain("de");
expect(tags).not.toContain("du");
expect(tags).not.toContain("und");
expect(tags).not.toContain("2026-04-04.md");
});
it("extracts protected and segmented CJK concept tags", () => {
const tags = deriveConceptTags({
path: "memory/2026-04-04.md",
snippet:
"障害対応ルーター設定とバックアップ確認。路由器备份与网关同步。라우터 백업 페일오버 점검.",
});
expect(tags).toEqual(
expect.arrayContaining([
"障害対応",
"ルーター",
"バックアップ",
"路由器",
"备份",
"网关",
"라우터",
"백업",
]),
);
expect(tags).not.toContain("ルー");
expect(tags).not.toContain("ター");
});
it("classifies concept tags by script family", () => {
expect(classifyConceptTagScript("routeur")).toBe("latin");
expect(classifyConceptTagScript("路由器")).toBe("cjk");
expect(classifyConceptTagScript("qmd路由器")).toBe("mixed");
});
it("summarizes entry coverage across latin, cjk, and mixed tags", () => {
expect(
summarizeConceptTagScriptCoverage([
["routeur", "sauvegarde"],
["路由器", "备份"],
["qmd", "路由器"],
["сервер"],
]),
).toEqual({
latinEntryCount: 1,
cjkEntryCount: 1,
mixedEntryCount: 1,
otherEntryCount: 1,
});
});
});

View File

@@ -0,0 +1,471 @@
import path from "node:path";
export const MAX_CONCEPT_TAGS = 8;
export type ConceptTagScriptFamily = "latin" | "cjk" | "mixed" | "other";
export type ConceptTagScriptCoverage = {
latinEntryCount: number;
cjkEntryCount: number;
mixedEntryCount: number;
otherEntryCount: number;
};
const LANGUAGE_STOP_WORDS = {
shared: [
"about",
"after",
"agent",
"again",
"also",
"because",
"before",
"being",
"between",
"build",
"called",
"could",
"daily",
"default",
"deploy",
"during",
"every",
"file",
"files",
"from",
"have",
"into",
"just",
"line",
"lines",
"long",
"main",
"make",
"memory",
"month",
"more",
"most",
"move",
"much",
"next",
"note",
"notes",
"over",
"part",
"past",
"port",
"same",
"score",
"search",
"session",
"sessions",
"short",
"should",
"since",
"some",
"than",
"that",
"their",
"there",
"these",
"they",
"this",
"through",
"today",
"using",
"with",
"work",
"workspace",
"year",
],
english: ["and", "are", "for", "into", "its", "our", "then", "were"],
spanish: [
"al",
"con",
"como",
"de",
"del",
"el",
"en",
"es",
"la",
"las",
"los",
"para",
"por",
"que",
"se",
"sin",
"su",
"sus",
"una",
"uno",
"unos",
"unas",
"y",
],
french: [
"au",
"aux",
"avec",
"dans",
"de",
"des",
"du",
"en",
"est",
"et",
"la",
"le",
"les",
"ou",
"pour",
"que",
"qui",
"sans",
"ses",
"son",
"sur",
"une",
"un",
],
german: [
"auf",
"aus",
"bei",
"das",
"dem",
"den",
"der",
"des",
"die",
"ein",
"eine",
"einem",
"einen",
"einer",
"für",
"im",
"in",
"mit",
"nach",
"oder",
"ohne",
"über",
"und",
"von",
"zu",
"zum",
"zur",
],
cjk: [
"が",
"から",
"する",
"して",
"した",
"で",
"と",
"に",
"の",
"は",
"へ",
"まで",
"も",
"や",
"を",
"与",
"为",
"了",
"及",
"和",
"在",
"将",
"或",
"把",
"是",
"用",
"的",
"과",
"는",
"도",
"로",
"를",
"에",
"에서",
"와",
"은",
"으로",
"을",
"이",
"하다",
"한",
"할",
"해",
"했다",
"했다",
],
pathNoise: [
"cjs",
"cpp",
"cts",
"jsx",
"json",
"md",
"mjs",
"mts",
"text",
"toml",
"ts",
"tsx",
"txt",
"yaml",
"yml",
],
} as const;
const CONCEPT_STOP_WORDS = new Set(
Object.values(LANGUAGE_STOP_WORDS)
.flatMap((words) => words)
.map((word) => word.toLowerCase()),
);
const PROTECTED_GLOSSARY = [
"backup",
"backups",
"embedding",
"embeddings",
"failover",
"gateway",
"glacier",
"gpt",
"kv",
"network",
"openai",
"qmd",
"router",
"s3",
"vlan",
"sauvegarde",
"routeur",
"passerelle",
"konfiguration",
"sicherung",
"überwachung",
"configuración",
"respaldo",
"enrutador",
"puerta-de-enlace",
"バックアップ",
"フェイルオーバー",
"ルーター",
"ネットワーク",
"ゲートウェイ",
"障害対応",
"路由器",
"备份",
"故障转移",
"网络",
"网关",
"라우터",
"백업",
"페일오버",
"네트워크",
"게이트웨이",
"장애대응",
].map((word) => word.normalize("NFKC").toLowerCase());
const COMPOUND_TOKEN_RE = /[\p{L}\p{N}]+(?:[._/-][\p{L}\p{N}]+)+/gu;
const LETTER_OR_NUMBER_RE = /[\p{L}\p{N}]/u;
const LATIN_RE = /\p{Script=Latin}/u;
const HAN_RE = /\p{Script=Han}/u;
const HIRAGANA_RE = /\p{Script=Hiragana}/u;
const KATAKANA_RE = /\p{Script=Katakana}/u;
const HANGUL_RE = /\p{Script=Hangul}/u;
const DEFAULT_WORD_SEGMENTER =
typeof Intl.Segmenter === "function" ? new Intl.Segmenter("und", { granularity: "word" }) : null;
function containsLetterOrNumber(value: string): boolean {
return LETTER_OR_NUMBER_RE.test(value);
}
export function classifyConceptTagScript(tag: string): ConceptTagScriptFamily {
const normalized = tag.normalize("NFKC");
const hasLatin = LATIN_RE.test(normalized);
const hasCjk =
HAN_RE.test(normalized) ||
HIRAGANA_RE.test(normalized) ||
KATAKANA_RE.test(normalized) ||
HANGUL_RE.test(normalized);
if (hasLatin && hasCjk) {
return "mixed";
}
if (hasCjk) {
return "cjk";
}
if (hasLatin) {
return "latin";
}
return "other";
}
function minimumTokenLengthForScript(script: ConceptTagScriptFamily): number {
if (script === "cjk") {
return 2;
}
return 3;
}
function isKanaOnlyToken(value: string): boolean {
return (
!HAN_RE.test(value) &&
!HANGUL_RE.test(value) &&
(HIRAGANA_RE.test(value) || KATAKANA_RE.test(value))
);
}
function normalizeConceptToken(rawToken: string): string | null {
const normalized = rawToken
.normalize("NFKC")
.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")
.replaceAll("_", "-")
.toLowerCase();
if (!normalized || !containsLetterOrNumber(normalized) || normalized.length > 32) {
return null;
}
if (
/^\d+$/.test(normalized) ||
/^\d{4}-\d{2}-\d{2}$/u.test(normalized) ||
/^\d{4}-\d{2}-\d{2}\.[\p{L}\p{N}]+$/u.test(normalized)
) {
return null;
}
const script = classifyConceptTagScript(normalized);
if (normalized.length < minimumTokenLengthForScript(script)) {
return null;
}
if (isKanaOnlyToken(normalized) && normalized.length < 3) {
return null;
}
if (CONCEPT_STOP_WORDS.has(normalized)) {
return null;
}
return normalized;
}
function collectGlossaryMatches(source: string): string[] {
const normalizedSource = source.normalize("NFKC").toLowerCase();
const matches: string[] = [];
for (const entry of PROTECTED_GLOSSARY) {
if (!normalizedSource.includes(entry)) {
continue;
}
matches.push(entry);
}
return matches;
}
function collectCompoundTokens(source: string): string[] {
return source.match(COMPOUND_TOKEN_RE) ?? [];
}
function collectSegmentTokens(source: string): string[] {
if (DEFAULT_WORD_SEGMENTER) {
return Array.from(DEFAULT_WORD_SEGMENTER.segment(source), (part) =>
part.isWordLike ? part.segment : "",
).filter(Boolean);
}
return source.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
}
function pushNormalizedTag(tags: string[], rawToken: string, limit: number): void {
const normalized = normalizeConceptToken(rawToken);
if (!normalized || tags.includes(normalized)) {
return;
}
tags.push(normalized);
if (tags.length > limit) {
tags.splice(limit);
}
}
export function deriveConceptTags(params: {
path: string;
snippet: string;
limit?: number;
}): string[] {
const source = `${path.basename(params.path)} ${params.snippet}`;
const limit = Number.isFinite(params.limit)
? Math.max(0, Math.floor(params.limit as number))
: MAX_CONCEPT_TAGS;
if (limit === 0) {
return [];
}
const tags: string[] = [];
for (const rawToken of [
...collectGlossaryMatches(source),
...collectCompoundTokens(source),
...collectSegmentTokens(source),
]) {
pushNormalizedTag(tags, rawToken, limit);
if (tags.length >= limit) {
break;
}
}
return tags;
}
export function summarizeConceptTagScriptCoverage(
conceptTagsByEntry: string[][],
): ConceptTagScriptCoverage {
const coverage: ConceptTagScriptCoverage = {
latinEntryCount: 0,
cjkEntryCount: 0,
mixedEntryCount: 0,
otherEntryCount: 0,
};
for (const conceptTags of conceptTagsByEntry) {
let hasLatin = false;
let hasCjk = false;
let hasOther = false;
for (const tag of conceptTags) {
const family = classifyConceptTagScript(tag);
if (family === "mixed") {
hasLatin = true;
hasCjk = true;
continue;
}
if (family === "latin") {
hasLatin = true;
continue;
}
if (family === "cjk") {
hasCjk = true;
continue;
}
hasOther = true;
}
if (hasLatin && hasCjk) {
coverage.mixedEntryCount += 1;
} else if (hasCjk) {
coverage.cjkEntryCount += 1;
} else if (hasLatin) {
coverage.latinEntryCount += 1;
} else if (hasOther) {
coverage.otherEntryCount += 1;
}
}
return coverage;
}
export const __testing = {
normalizeConceptToken,
collectGlossaryMatches,
collectCompoundTokens,
collectSegmentTokens,
};

View File

@@ -28,7 +28,7 @@ function createLogger() {
function createCronHarness(
initialJobs: CronJobLike[] = [],
opts?: { removeResult?: "boolean" | "unknown" },
opts?: { removeResult?: "boolean" | "unknown"; removeThrowsForIds?: string[] },
) {
const jobs: CronJobLike[] = [...initialJobs];
const addCalls: CronAddInput[] = [];
@@ -79,6 +79,9 @@ function createCronHarness(
},
async remove(id) {
removeCalls.push(id);
if (opts?.removeThrowsForIds?.includes(id)) {
throw new Error(`remove failed for ${id}`);
}
const index = jobs.findIndex((entry) => entry.id === id);
if (index >= 0) {
jobs.splice(index, 1);
@@ -142,6 +145,51 @@ describe("short-term dreaming config", () => {
});
});
it("accepts cron alias and numeric string thresholds", () => {
const resolved = resolveShortTermPromotionDreamingConfig({
pluginConfig: {
dreaming: {
mode: "deep",
cron: "5 1 * * *",
limit: "4",
minScore: "0.6",
minRecallCount: "2",
minUniqueQueries: "3",
},
},
});
expect(resolved).toEqual({
enabled: true,
cron: "5 1 * * *",
limit: 4,
minScore: 0.6,
minRecallCount: 2,
minUniqueQueries: 3,
});
});
it("treats blank numeric strings as unset and keeps preset defaults", () => {
const resolved = resolveShortTermPromotionDreamingConfig({
pluginConfig: {
dreaming: {
mode: "deep",
limit: " ",
minScore: "",
minRecallCount: " ",
minUniqueQueries: "",
},
},
});
expect(resolved).toEqual({
enabled: true,
cron: constants.DREAMING_PRESET_DEFAULTS.deep.cron,
limit: constants.DREAMING_PRESET_DEFAULTS.deep.limit,
minScore: constants.DREAMING_PRESET_DEFAULTS.deep.minScore,
minRecallCount: constants.DREAMING_PRESET_DEFAULTS.deep.minRecallCount,
minUniqueQueries: constants.DREAMING_PRESET_DEFAULTS.deep.minUniqueQueries,
});
});
it("accepts limit=0 as an explicit no-op promotion cap", () => {
const resolved = resolveShortTermPromotionDreamingConfig({
pluginConfig: {
@@ -376,6 +424,40 @@ describe("short-term dreaming cron reconciliation", () => {
expect(result.removed).toBe(0);
expect(harness.removeCalls).toEqual(["job-managed"]);
});
it("warns and continues when disabling managed jobs hits a remove error", async () => {
const managedJob: CronJobLike = {
id: "job-managed",
name: constants.MANAGED_DREAMING_CRON_NAME,
description: `${constants.MANAGED_DREAMING_CRON_TAG} test`,
enabled: true,
schedule: { kind: "cron", expr: "0 3 * * *" },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: constants.DREAMING_SYSTEM_EVENT_TEXT },
createdAtMs: 10,
};
const harness = createCronHarness([managedJob], { removeThrowsForIds: ["job-managed"] });
const logger = createLogger();
const result = await reconcileShortTermDreamingCronJob({
cron: harness.cron,
config: {
enabled: false,
cron: constants.DEFAULT_DREAMING_CRON_EXPR,
limit: constants.DEFAULT_DREAMING_LIMIT,
minScore: constants.DEFAULT_DREAMING_MIN_SCORE,
minRecallCount: constants.DEFAULT_DREAMING_MIN_RECALL_COUNT,
minUniqueQueries: constants.DEFAULT_DREAMING_MIN_UNIQUE_QUERIES,
},
logger,
});
expect(result).toEqual({ status: "disabled", removed: 0 });
expect(logger.warn).toHaveBeenCalledWith(
expect.stringContaining("failed to remove managed dreaming cron job job-managed"),
);
});
});
describe("short-term dreaming trigger", () => {
@@ -491,4 +573,111 @@ describe("short-term dreaming trigger", () => {
});
expect(result).toBeUndefined();
});
it("skips dreaming promotion cleanly when limit is zero", async () => {
const logger = createLogger();
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-dreaming-limit-zero-"));
tempDirs.push(workspaceDir);
const result = await runShortTermDreamingPromotionIfTriggered({
cleanedBody: constants.DREAMING_SYSTEM_EVENT_TEXT,
trigger: "heartbeat",
workspaceDir,
config: {
enabled: true,
cron: constants.DEFAULT_DREAMING_CRON_EXPR,
limit: 0,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
},
logger,
});
expect(result).toEqual({
handled: true,
reason: "memory-core: short-term dreaming disabled by limit",
});
expect(logger.info).toHaveBeenCalledWith(
"memory-core: dreaming promotion skipped because limit=0.",
);
await expect(fs.access(path.join(workspaceDir, "MEMORY.md"))).rejects.toMatchObject({
code: "ENOENT",
});
});
it("repairs recall artifacts before dreaming promotion runs", async () => {
const logger = createLogger();
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-dreaming-repair-"));
tempDirs.push(workspaceDir);
const storePath = path.join(workspaceDir, "memory", ".dreams", "short-term-recall.json");
await fs.mkdir(path.dirname(storePath), { recursive: true });
await fs.writeFile(
storePath,
`${JSON.stringify(
{
version: 1,
updatedAt: "2026-04-01T00:00:00.000Z",
entries: {
"memory:memory/2026-04-03.md:1:2": {
key: "memory:memory/2026-04-03.md:1:2",
path: "memory/2026-04-03.md",
startLine: 1,
endLine: 2,
source: "memory",
snippet: "Move backups to S3 Glacier and sync router failover notes.",
recallCount: 3,
totalScore: 2.7,
maxScore: 0.95,
firstRecalledAt: "2026-04-01T00:00:00.000Z",
lastRecalledAt: "2026-04-03T00:00:00.000Z",
queryHashes: ["abc", "abc", "def"],
recallDays: ["2026-04-01", "2026-04-01", "2026-04-03"],
conceptTags: [],
},
},
},
null,
2,
)}\n`,
"utf-8",
);
const result = await runShortTermDreamingPromotionIfTriggered({
cleanedBody: constants.DREAMING_SYSTEM_EVENT_TEXT,
trigger: "heartbeat",
workspaceDir,
config: {
enabled: true,
cron: constants.DEFAULT_DREAMING_CRON_EXPR,
limit: 10,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
},
logger,
});
expect(result?.handled).toBe(true);
expect(logger.info).toHaveBeenCalledWith(
expect.stringContaining("normalized recall artifacts before dreaming"),
);
const repaired = JSON.parse(await fs.readFile(storePath, "utf-8")) as {
entries: Record<
string,
{ queryHashes?: string[]; recallDays?: string[]; conceptTags?: string[] }
>;
};
expect(repaired.entries["memory:memory/2026-04-03.md:1:2"]?.queryHashes).toEqual([
"abc",
"def",
]);
expect(repaired.entries["memory:memory/2026-04-03.md:1:2"]?.recallDays).toEqual([
"2026-04-01",
"2026-04-03",
]);
expect(repaired.entries["memory:memory/2026-04-03.md:1:2"]?.conceptTags).toEqual(
expect.arrayContaining(["glacier", "router", "failover"]),
);
});
});

View File

@@ -4,6 +4,7 @@ import {
DEFAULT_PROMOTION_MIN_RECALL_COUNT,
DEFAULT_PROMOTION_MIN_SCORE,
DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES,
repairShortTermPromotionArtifacts,
rankShortTermPromotionCandidates,
} from "./short-term-promotion.js";
@@ -150,10 +151,14 @@ function normalizeDreamingMode(value: unknown): DreamingMode {
}
function normalizeNonNegativeInt(value: unknown, fallback: number): number {
if (typeof value !== "number" || !Number.isFinite(value)) {
if (typeof value === "string" && value.trim().length === 0) {
return fallback;
}
const floored = Math.floor(value);
const num = typeof value === "string" ? Number(value.trim()) : Number(value);
if (!Number.isFinite(num)) {
return fallback;
}
const floored = Math.floor(num);
if (floored < 0) {
return fallback;
}
@@ -161,13 +166,17 @@ function normalizeNonNegativeInt(value: unknown, fallback: number): number {
}
function normalizeScore(value: unknown, fallback: number): number {
if (typeof value !== "number" || !Number.isFinite(value)) {
if (typeof value === "string" && value.trim().length === 0) {
return fallback;
}
if (value < 0 || value > 1) {
const num = typeof value === "string" ? Number(value.trim()) : Number(value);
if (!Number.isFinite(num)) {
return fallback;
}
return value;
if (num < 0 || num > 1) {
return fallback;
}
return num;
}
function formatErrorMessage(err: unknown): string {
@@ -183,6 +192,23 @@ function resolveTimezoneFallback(cfg: OpenClawConfig | undefined): string | unde
return normalizeTrimmedString(defaults?.userTimezone);
}
function formatRepairSummary(repair: {
rewroteStore: boolean;
removedInvalidEntries: number;
removedStaleLock: boolean;
}): string {
const actions: string[] = [];
if (repair.rewroteStore) {
actions.push(
`rewrote recall store${repair.removedInvalidEntries > 0 ? ` (-${repair.removedInvalidEntries} invalid)` : ""}`,
);
}
if (repair.removedStaleLock) {
actions.push("removed stale promotion lock");
}
return actions.join(", ");
}
function resolveManagedCronDescription(config: ShortTermPromotionDreamingConfig): string {
return `${MANAGED_DREAMING_CRON_TAG} Promote weighted short-term recalls into MEMORY.md (limit=${config.limit}, minScore=${config.minScore.toFixed(3)}, minRecallCount=${config.minRecallCount}, minUniqueQueries=${config.minUniqueQueries}).`;
}
@@ -319,7 +345,10 @@ export function resolveShortTermPromotionDreamingConfig(params: {
const enabled = mode !== "off";
const thresholdPreset: DreamingPreset = mode === "off" ? DEFAULT_DREAMING_PRESET : mode;
const thresholdDefaults = DREAMING_PRESET_DEFAULTS[thresholdPreset];
const cron = normalizeTrimmedString(dreaming?.frequency) ?? thresholdDefaults.cron;
const cron =
normalizeTrimmedString(dreaming?.cron) ??
normalizeTrimmedString(dreaming?.frequency) ??
thresholdDefaults.cron;
const timezone =
normalizeTrimmedString(dreaming?.timezone) ?? resolveTimezoneFallback(params.cfg);
const limit = normalizeNonNegativeInt(dreaming?.limit, thresholdDefaults.limit);
@@ -360,9 +389,15 @@ export async function reconcileShortTermDreamingCronJob(params: {
if (!params.config.enabled) {
let removed = 0;
for (const job of managed) {
const result = await cron.remove(job.id);
if (result.removed === true) {
removed += 1;
try {
const result = await cron.remove(job.id);
if (result.removed === true) {
removed += 1;
}
} catch (err) {
params.logger.warn(
`memory-core: failed to remove managed dreaming cron job ${job.id}: ${formatErrorMessage(err)}`,
);
}
}
if (removed > 0) {
@@ -381,9 +416,15 @@ export async function reconcileShortTermDreamingCronJob(params: {
const [primary, ...duplicates] = sortManagedJobs(managed);
let removed = 0;
for (const duplicate of duplicates) {
const result = await cron.remove(duplicate.id);
if (result.removed === true) {
removed += 1;
try {
const result = await cron.remove(duplicate.id);
if (result.removed === true) {
removed += 1;
}
} catch (err) {
params.logger.warn(
`memory-core: failed to prune duplicate managed dreaming cron job ${duplicate.id}: ${formatErrorMessage(err)}`,
);
}
}
@@ -424,8 +465,18 @@ export async function runShortTermDreamingPromotionIfTriggered(params: {
);
return { handled: true, reason: "memory-core: short-term dreaming missing workspace" };
}
if (params.config.limit === 0) {
params.logger.info("memory-core: dreaming promotion skipped because limit=0.");
return { handled: true, reason: "memory-core: short-term dreaming disabled by limit" };
}
try {
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
if (repair.changed) {
params.logger.info(
`memory-core: normalized recall artifacts before dreaming (${formatRepairSummary(repair)}).`,
);
}
const candidates = await rankShortTermPromotionCandidates({
workspaceDir,
limit: params.config.limit,
@@ -455,37 +506,48 @@ export function registerShortTermPromotionDreaming(api: OpenClawPluginApi): void
api.registerHook(
"gateway:startup",
async (event: unknown) => {
const config = resolveShortTermPromotionDreamingConfig({
pluginConfig: api.pluginConfig,
cfg: api.config,
});
const cron = resolveCronServiceFromStartupEvent(event);
if (!cron && config.enabled) {
api.logger.warn(
"memory-core: managed dreaming cron could not be reconciled (cron service unavailable).",
try {
const config = resolveShortTermPromotionDreamingConfig({
pluginConfig: api.pluginConfig,
cfg: api.config,
});
const cron = resolveCronServiceFromStartupEvent(event);
if (!cron && config.enabled) {
api.logger.warn(
"memory-core: managed dreaming cron could not be reconciled (cron service unavailable).",
);
}
await reconcileShortTermDreamingCronJob({
cron,
config,
logger: api.logger,
});
} catch (err) {
api.logger.error(
`memory-core: dreaming startup reconciliation failed: ${formatErrorMessage(err)}`,
);
}
await reconcileShortTermDreamingCronJob({
cron,
config,
logger: api.logger,
});
},
{ name: "memory-core-short-term-dreaming-cron" },
);
api.on("before_agent_reply", async (event, ctx) => {
const config = resolveShortTermPromotionDreamingConfig({
pluginConfig: api.pluginConfig,
cfg: api.config,
});
return await runShortTermDreamingPromotionIfTriggered({
cleanedBody: event.cleanedBody,
trigger: ctx.trigger,
workspaceDir: ctx.workspaceDir,
config,
logger: api.logger,
});
try {
const config = resolveShortTermPromotionDreamingConfig({
pluginConfig: api.pluginConfig,
cfg: api.config,
});
return await runShortTermDreamingPromotionIfTriggered({
cleanedBody: event.cleanedBody,
trigger: ctx.trigger,
workspaceDir: ctx.workspaceDir,
config,
logger: api.logger,
});
} catch (err) {
api.logger.error(`memory-core: dreaming trigger failed: ${formatErrorMessage(err)}`);
return undefined;
}
});
}

View File

@@ -1,13 +1,17 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import {
applyShortTermPromotions,
auditShortTermPromotionArtifacts,
isShortTermMemoryPath,
rankShortTermPromotionCandidates,
recordShortTermRecalls,
repairShortTermPromotionArtifacts,
resolveShortTermRecallLockPath,
resolveShortTermRecallStorePath,
__testing,
} from "./short-term-promotion.js";
describe("short-term promotion", () => {
@@ -79,6 +83,8 @@ describe("short-term promotion", () => {
expect(ranked[0]?.recallCount).toBe(2);
expect(ranked[0]?.uniqueQueries).toBe(2);
expect(ranked[0]?.score).toBeGreaterThan(0);
expect(ranked[0]?.conceptTags).toContain("router");
expect(ranked[0]?.components.conceptual).toBeGreaterThan(0);
const storePath = resolveShortTermRecallStorePath(workspaceDir);
const raw = await fs.readFile(storePath, "utf-8");
@@ -142,6 +148,53 @@ describe("short-term promotion", () => {
});
});
it("rewards spaced recalls as consolidation instead of only raw count", async () => {
await withTempWorkspace(async (workspaceDir) => {
await recordShortTermRecalls({
workspaceDir,
query: "router",
nowMs: Date.parse("2026-04-01T10:00:00.000Z"),
results: [
{
path: "memory/2026-04-01.md",
startLine: 1,
endLine: 2,
score: 0.9,
snippet: "Configured router VLAN 10 and IoT segment.",
source: "memory",
},
],
});
await recordShortTermRecalls({
workspaceDir,
query: "iot segment",
nowMs: Date.parse("2026-04-04T10:00:00.000Z"),
results: [
{
path: "memory/2026-04-01.md",
startLine: 1,
endLine: 2,
score: 0.88,
snippet: "Configured router VLAN 10 and IoT segment.",
source: "memory",
},
],
});
const ranked = await rankShortTermPromotionCandidates({
workspaceDir,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
nowMs: Date.parse("2026-04-05T10:00:00.000Z"),
});
expect(ranked).toHaveLength(1);
expect(ranked[0]?.recallDays).toEqual(["2026-04-01", "2026-04-04"]);
expect(ranked[0]?.components.consolidation).toBeGreaterThan(0.4);
});
});
it("treats negative threshold overrides as invalid and keeps defaults", async () => {
await withTempWorkspace(async (workspaceDir) => {
await recordShortTermRecalls({
@@ -189,11 +242,15 @@ describe("short-term promotion", () => {
lastRecalledAt: new Date().toISOString(),
ageDays: 0,
score: 0.95,
recallDays: [new Date().toISOString().slice(0, 10)],
conceptTags: ["glacier", "backups"],
components: {
frequency: 0.2,
relevance: 0.95,
diversity: 0.2,
recency: 1,
consolidation: 0.2,
conceptual: 0.4,
},
},
],
@@ -305,4 +362,265 @@ describe("short-term promotion", () => {
expect(sectionCount).toBe(1);
});
});
it("audits and repairs invalid store metadata plus stale locks", async () => {
await withTempWorkspace(async (workspaceDir) => {
const storePath = resolveShortTermRecallStorePath(workspaceDir);
await fs.mkdir(path.dirname(storePath), { recursive: true });
await fs.writeFile(
storePath,
JSON.stringify(
{
version: 1,
updatedAt: "2026-04-04T00:00:00.000Z",
entries: {
good: {
key: "good",
path: "memory/2026-04-01.md",
startLine: 1,
endLine: 2,
source: "memory",
snippet: "Gateway host uses qmd vector search for router notes.",
recallCount: 2,
totalScore: 1.8,
maxScore: 0.95,
firstRecalledAt: "2026-04-01T00:00:00.000Z",
lastRecalledAt: "2026-04-04T00:00:00.000Z",
queryHashes: ["a", "b"],
},
bad: {
path: "",
},
},
},
null,
2,
),
"utf-8",
);
const lockPath = path.join(workspaceDir, "memory", ".dreams", "short-term-promotion.lock");
await fs.writeFile(lockPath, "999999:0\n", "utf-8");
const staleMtime = new Date(Date.now() - 120_000);
await fs.utimes(lockPath, staleMtime, staleMtime);
const auditBefore = await auditShortTermPromotionArtifacts({ workspaceDir });
expect(auditBefore.invalidEntryCount).toBe(1);
expect(auditBefore.issues.map((issue) => issue.code)).toEqual(
expect.arrayContaining(["recall-store-invalid", "recall-lock-stale"]),
);
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
expect(repair.changed).toBe(true);
expect(repair.rewroteStore).toBe(true);
expect(repair.removedStaleLock).toBe(true);
const auditAfter = await auditShortTermPromotionArtifacts({ workspaceDir });
expect(auditAfter.invalidEntryCount).toBe(0);
expect(auditAfter.issues.map((issue) => issue.code)).not.toContain("recall-lock-stale");
const repairedRaw = JSON.parse(await fs.readFile(storePath, "utf-8")) as {
entries: Record<string, { conceptTags?: string[]; recallDays?: string[] }>;
};
expect(repairedRaw.entries.good?.conceptTags).toContain("router");
expect(repairedRaw.entries.good?.recallDays).toEqual(["2026-04-04"]);
});
});
it("repairs empty recall-store files without throwing", async () => {
await withTempWorkspace(async (workspaceDir) => {
const storePath = resolveShortTermRecallStorePath(workspaceDir);
await fs.mkdir(path.dirname(storePath), { recursive: true });
await fs.writeFile(storePath, " \n", "utf-8");
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
expect(repair.changed).toBe(true);
expect(repair.rewroteStore).toBe(true);
expect(JSON.parse(await fs.readFile(storePath, "utf-8"))).toMatchObject({
version: 1,
entries: {},
});
});
});
it("does not rewrite an already normalized healthy recall store", async () => {
await withTempWorkspace(async (workspaceDir) => {
const storePath = resolveShortTermRecallStorePath(workspaceDir);
await fs.mkdir(path.dirname(storePath), { recursive: true });
const snippet = "Gateway host uses qmd vector search for router notes.";
const raw = `${JSON.stringify(
{
version: 1,
updatedAt: "2026-04-04T00:00:00.000Z",
entries: {
good: {
key: "good",
path: "memory/2026-04-01.md",
startLine: 1,
endLine: 2,
source: "memory",
snippet,
recallCount: 2,
totalScore: 1.8,
maxScore: 0.95,
firstRecalledAt: "2026-04-01T00:00:00.000Z",
lastRecalledAt: "2026-04-04T00:00:00.000Z",
queryHashes: ["a", "b"],
recallDays: ["2026-04-04"],
conceptTags: __testing.deriveConceptTags({
path: "memory/2026-04-01.md",
snippet,
}),
},
},
},
null,
2,
)}\n`;
await fs.writeFile(storePath, raw, "utf-8");
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
expect(repair.changed).toBe(false);
expect(repair.rewroteStore).toBe(false);
expect(await fs.readFile(storePath, "utf-8")).toBe(raw);
});
});
it("waits for an active short-term lock before repairing", async () => {
await withTempWorkspace(async (workspaceDir) => {
const storePath = resolveShortTermRecallStorePath(workspaceDir);
const lockPath = resolveShortTermRecallLockPath(workspaceDir);
await fs.mkdir(path.dirname(storePath), { recursive: true });
await fs.writeFile(
storePath,
JSON.stringify(
{
version: 1,
updatedAt: "2026-04-04T00:00:00.000Z",
entries: {
bad: {
path: "",
},
},
},
null,
2,
),
"utf-8",
);
await fs.writeFile(lockPath, `${process.pid}:${Date.now()}\n`, "utf-8");
let settled = false;
const repairPromise = repairShortTermPromotionArtifacts({ workspaceDir }).then((result) => {
settled = true;
return result;
});
await new Promise((resolve) => setTimeout(resolve, 80));
expect(settled).toBe(false);
await fs.unlink(lockPath);
const repair = await repairPromise;
expect(repair.changed).toBe(true);
expect(repair.rewroteStore).toBe(true);
expect(repair.removedInvalidEntries).toBe(1);
});
});
it("downgrades lock inspection failures into audit issues", async () => {
await withTempWorkspace(async (workspaceDir) => {
const lockPath = path.join(workspaceDir, "memory", ".dreams", "short-term-promotion.lock");
const stat = vi.spyOn(fs, "stat").mockImplementation(async (target) => {
if (String(target) === lockPath) {
const error = Object.assign(new Error("no access"), { code: "EACCES" });
throw error;
}
return await vi
.importActual<typeof import("node:fs/promises")>("node:fs/promises")
.then((actual) => actual.stat(target));
});
try {
const audit = await auditShortTermPromotionArtifacts({ workspaceDir });
expect(audit.issues).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "recall-lock-unreadable",
fixable: false,
}),
]),
);
} finally {
stat.mockRestore();
}
});
});
it("reports concept tag script coverage for multilingual recalls", async () => {
await withTempWorkspace(async (workspaceDir) => {
await recordShortTermRecalls({
workspaceDir,
query: "routeur glacier",
results: [
{
path: "memory/2026-04-03.md",
startLine: 1,
endLine: 2,
score: 0.93,
snippet: "Configuration du routeur et sauvegarde Glacier.",
source: "memory",
},
],
});
await recordShortTermRecalls({
workspaceDir,
query: "router cjk",
results: [
{
path: "memory/2026-04-04.md",
startLine: 1,
endLine: 2,
score: 0.95,
snippet: "障害対応ルーター設定とバックアップ確認。",
source: "memory",
},
],
});
const audit = await auditShortTermPromotionArtifacts({ workspaceDir });
expect(audit.conceptTaggedEntryCount).toBe(2);
expect(audit.conceptTagScripts).toEqual({
latinEntryCount: 1,
cjkEntryCount: 1,
mixedEntryCount: 0,
otherEntryCount: 0,
});
});
});
it("extracts stable concept tags from snippets and paths", () => {
expect(
__testing.deriveConceptTags({
path: "memory/2026-04-03.md",
snippet: "Move backups to S3 Glacier and sync QMD router notes.",
}),
).toEqual(expect.arrayContaining(["glacier", "router", "backups"]));
});
it("extracts multilingual concept tags across latin and cjk snippets", () => {
expect(
__testing.deriveConceptTags({
path: "memory/2026-04-03.md",
snippet: "Configuración du routeur et sauvegarde Glacier.",
}),
).toEqual(expect.arrayContaining(["configuración", "routeur", "sauvegarde", "glacier"]));
expect(
__testing.deriveConceptTags({
path: "memory/2026-04-03.md",
snippet: "障害対応ルーター設定とバックアップ確認。路由器备份与网关同步。",
}),
).toEqual(expect.arrayContaining(["障害対応", "ルーター", "バックアップ", "路由器", "备份"]));
});
});

View File

@@ -2,6 +2,12 @@ import { createHash, randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
import {
deriveConceptTags,
MAX_CONCEPT_TAGS,
summarizeConceptTagScriptCoverage,
type ConceptTagScriptCoverage,
} from "./concept-vocabulary.js";
const SHORT_TERM_PATH_RE = /(?:^|\/)memory\/(\d{4})-(\d{2})-(\d{2})\.md$/;
const SHORT_TERM_BASENAME_RE = /^(\d{4})-(\d{2})-(\d{2})\.md$/;
@@ -10,6 +16,8 @@ const DEFAULT_RECENCY_HALF_LIFE_DAYS = 14;
export const DEFAULT_PROMOTION_MIN_SCORE = 0.75;
export const DEFAULT_PROMOTION_MIN_RECALL_COUNT = 3;
export const DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES = 2;
const MAX_QUERY_HASHES = 32;
const MAX_RECALL_DAYS = 16;
const SHORT_TERM_STORE_RELATIVE_PATH = path.join("memory", ".dreams", "short-term-recall.json");
const SHORT_TERM_LOCK_RELATIVE_PATH = path.join("memory", ".dreams", "short-term-promotion.lock");
const SHORT_TERM_LOCK_WAIT_TIMEOUT_MS = 10_000;
@@ -21,13 +29,17 @@ export type PromotionWeights = {
relevance: number;
diversity: number;
recency: number;
consolidation: number;
conceptual: number;
};
export const DEFAULT_PROMOTION_WEIGHTS: PromotionWeights = {
frequency: 0.35,
relevance: 0.35,
frequency: 0.24,
relevance: 0.3,
diversity: 0.15,
recency: 0.15,
consolidation: 0.1,
conceptual: 0.06,
};
export type ShortTermRecallEntry = {
@@ -43,6 +55,8 @@ export type ShortTermRecallEntry = {
firstRecalledAt: string;
lastRecalledAt: string;
queryHashes: string[];
recallDays: string[];
conceptTags: string[];
promotedAt?: string;
};
@@ -57,6 +71,8 @@ export type PromotionComponents = {
relevance: number;
diversity: number;
recency: number;
consolidation: number;
conceptual: number;
};
export type PromotionCandidate = {
@@ -75,9 +91,54 @@ export type PromotionCandidate = {
lastRecalledAt: string;
ageDays: number;
score: number;
recallDays: string[];
conceptTags: string[];
components: PromotionComponents;
};
export type ShortTermAuditIssue = {
severity: "warn" | "error";
code:
| "recall-store-unreadable"
| "recall-store-empty"
| "recall-store-invalid"
| "recall-lock-stale"
| "recall-lock-unreadable"
| "qmd-index-missing"
| "qmd-index-empty"
| "qmd-collections-empty";
message: string;
fixable: boolean;
};
export type ShortTermAuditSummary = {
storePath: string;
lockPath: string;
updatedAt?: string;
exists: boolean;
entryCount: number;
promotedCount: number;
spacedEntryCount: number;
conceptTaggedEntryCount: number;
conceptTagScripts?: ConceptTagScriptCoverage;
invalidEntryCount: number;
issues: ShortTermAuditIssue[];
qmd?:
| {
dbPath?: string;
collections?: number;
dbBytes?: number;
}
| undefined;
};
export type RepairShortTermPromotionArtifactsResult = {
changed: boolean;
removedInvalidEntries: number;
rewroteStore: boolean;
removedStaleLock: boolean;
};
export type RankShortTermPromotionOptions = {
workspaceDir: string;
limit?: number;
@@ -153,15 +214,91 @@ function mergeQueryHashes(existing: string[], queryHash: string): string[] {
if (!queryHash) {
return existing;
}
const next = existing.filter(Boolean);
if (!next.includes(queryHash)) {
const seen = new Set<string>();
const next = existing.filter((value) => {
if (!value || seen.has(value)) {
return false;
}
seen.add(value);
return true;
});
if (!seen.has(queryHash)) {
next.push(queryHash);
}
const maxHashes = 32;
if (next.length <= maxHashes) {
if (next.length <= MAX_QUERY_HASHES) {
return next;
}
return next.slice(next.length - maxHashes);
return next.slice(next.length - MAX_QUERY_HASHES);
}
function mergeRecentDistinct(existing: string[], nextValue: string, limit: number): string[] {
const seen = new Set<string>();
const next = existing.filter((value): value is string => {
if (typeof value !== "string" || value.length === 0 || seen.has(value)) {
return false;
}
seen.add(value);
return true;
});
if (nextValue && !next.includes(nextValue)) {
next.push(nextValue);
}
if (next.length <= limit) {
return next;
}
return next.slice(next.length - limit);
}
function normalizeIsoDay(isoLike: string): string | null {
if (typeof isoLike !== "string") {
return null;
}
const match = isoLike.trim().match(/^(\d{4}-\d{2}-\d{2})/);
return match?.[1] ?? null;
}
function normalizeDistinctStrings(values: unknown[], limit: number): string[] {
const seen = new Set<string>();
const normalized: string[] = [];
for (const value of values) {
if (typeof value !== "string") {
continue;
}
const trimmed = value.trim();
if (!trimmed || seen.has(trimmed)) {
continue;
}
seen.add(trimmed);
normalized.push(trimmed);
if (normalized.length >= limit) {
break;
}
}
return normalized;
}
function calculateConsolidationComponent(recallDays: string[]): number {
if (recallDays.length === 0) {
return 0;
}
if (recallDays.length === 1) {
return 0.2;
}
const parsed = recallDays
.map((value) => Date.parse(`${value}T00:00:00.000Z`))
.filter((value) => Number.isFinite(value))
.toSorted((left, right) => left - right);
if (parsed.length <= 1) {
return 0.2;
}
const spanDays = Math.max(0, (parsed.at(-1)! - parsed[0]!) / DAY_MS);
const spacing = clampScore(Math.log1p(parsed.length - 1) / Math.log1p(4));
const span = clampScore(spanDays / 7);
return clampScore(0.55 * spacing + 0.45 * span);
}
function calculateConceptualComponent(conceptTags: string[]): number {
return clampScore(conceptTags.length / 6);
}
function emptyStore(nowIso: string): ShortTermRecallStore {
@@ -204,10 +341,19 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
const promotedAt = typeof entry.promotedAt === "string" ? entry.promotedAt : undefined;
const snippet = typeof entry.snippet === "string" ? normalizeSnippet(entry.snippet) : "";
const queryHashes = Array.isArray(entry.queryHashes)
? entry.queryHashes.filter(
(hash): hash is string => typeof hash === "string" && hash.length > 0,
)
? normalizeDistinctStrings(entry.queryHashes, MAX_QUERY_HASHES)
: [];
const recallDays = Array.isArray(entry.recallDays)
? entry.recallDays
.map((value) => normalizeIsoDay(String(value)))
.filter((value): value is string => value !== null)
: [];
const conceptTags = Array.isArray(entry.conceptTags)
? normalizeDistinctStrings(
entry.conceptTags.map((tag) => (typeof tag === "string" ? tag.toLowerCase() : tag)),
MAX_CONCEPT_TAGS,
)
: deriveConceptTags({ path: entryPath, snippet });
const normalizedKey = key || buildEntryKey({ path: entryPath, startLine, endLine, source });
entries[normalizedKey] = {
@@ -223,6 +369,8 @@ function normalizeStore(raw: unknown, nowIso: string): ShortTermRecallStore {
firstRecalledAt,
lastRecalledAt,
queryHashes,
recallDays: recallDays.slice(-MAX_RECALL_DAYS),
conceptTags,
...(promotedAt ? { promotedAt } : {}),
};
}
@@ -264,7 +412,9 @@ function normalizeWeights(weights?: Partial<PromotionWeights>): PromotionWeights
const relevance = Math.max(0, merged.relevance);
const diversity = Math.max(0, merged.diversity);
const recency = Math.max(0, merged.recency);
const sum = frequency + relevance + diversity + recency;
const consolidation = Math.max(0, merged.consolidation);
const conceptual = Math.max(0, merged.conceptual);
const sum = frequency + relevance + diversity + recency + consolidation + conceptual;
if (sum <= 0) {
return { ...DEFAULT_PROMOTION_WEIGHTS };
}
@@ -273,6 +423,8 @@ function normalizeWeights(weights?: Partial<PromotionWeights>): PromotionWeights
relevance: relevance / sum,
diversity: diversity / sum,
recency: recency / sum,
consolidation: consolidation / sum,
conceptual: conceptual / sum,
};
}
@@ -446,6 +598,12 @@ export async function recordShortTermRecalls(params: {
const totalScore = Math.max(0, (existing?.totalScore ?? 0) + score);
const maxScore = Math.max(existing?.maxScore ?? 0, score);
const queryHashes = mergeQueryHashes(existing?.queryHashes ?? [], queryHash);
const recallDays = mergeRecentDistinct(
existing?.recallDays ?? [],
nowIso.slice(0, 10),
MAX_RECALL_DAYS,
);
const conceptTags = deriveConceptTags({ path: normalizedPath, snippet });
store.entries[key] = {
key,
@@ -460,6 +618,8 @@ export async function recordShortTermRecalls(params: {
firstRecalledAt: existing?.firstRecalledAt ?? nowIso,
lastRecalledAt: nowIso,
queryHashes,
recallDays,
conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []),
...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}),
};
}
@@ -524,12 +684,18 @@ export async function rankShortTermPromotionCandidates(
? Math.max(0, (nowMs - lastRecalledAtMs) / DAY_MS)
: 0;
const recency = clampScore(calculateRecencyComponent(ageDays, halfLifeDays));
const recallDays = entry.recallDays ?? [];
const conceptTags = entry.conceptTags ?? [];
const consolidation = calculateConsolidationComponent(recallDays);
const conceptual = calculateConceptualComponent(conceptTags);
const score =
weights.frequency * frequency +
weights.relevance * avgScore +
weights.diversity * diversity +
weights.recency * recency;
weights.recency * recency +
weights.consolidation * consolidation +
weights.conceptual * conceptual;
if (score < minScore) {
continue;
@@ -551,11 +717,15 @@ export async function rankShortTermPromotionCandidates(
lastRecalledAt: entry.lastRecalledAt,
ageDays,
score: clampScore(score),
recallDays,
conceptTags,
components: {
frequency,
relevance: avgScore,
diversity,
recency,
consolidation,
conceptual,
},
});
}
@@ -688,8 +858,249 @@ export function resolveShortTermRecallStorePath(workspaceDir: string): string {
return resolveStorePath(workspaceDir);
}
export function resolveShortTermRecallLockPath(workspaceDir: string): string {
return resolveLockPath(workspaceDir);
}
export async function auditShortTermPromotionArtifacts(params: {
workspaceDir: string;
qmd?: {
dbPath?: string;
collections?: number;
};
}): Promise<ShortTermAuditSummary> {
const workspaceDir = params.workspaceDir.trim();
const storePath = resolveStorePath(workspaceDir);
const lockPath = resolveLockPath(workspaceDir);
const issues: ShortTermAuditIssue[] = [];
let exists = false;
let entryCount = 0;
let promotedCount = 0;
let spacedEntryCount = 0;
let conceptTaggedEntryCount = 0;
let conceptTagScripts: ConceptTagScriptCoverage | undefined;
let invalidEntryCount = 0;
let updatedAt: string | undefined;
try {
const raw = await fs.readFile(storePath, "utf-8");
exists = true;
if (raw.trim().length === 0) {
issues.push({
severity: "warn",
code: "recall-store-empty",
message: "Short-term recall store is empty.",
fixable: true,
});
} else {
const nowIso = new Date().toISOString();
const parsed = JSON.parse(raw) as unknown;
const store = normalizeStore(parsed, nowIso);
updatedAt = store.updatedAt;
entryCount = Object.keys(store.entries).length;
promotedCount = Object.values(store.entries).filter((entry) =>
Boolean(entry.promotedAt),
).length;
spacedEntryCount = Object.values(store.entries).filter(
(entry) => (entry.recallDays?.length ?? 0) > 1,
).length;
conceptTaggedEntryCount = Object.values(store.entries).filter(
(entry) => (entry.conceptTags?.length ?? 0) > 0,
).length;
conceptTagScripts = summarizeConceptTagScriptCoverage(
Object.values(store.entries)
.filter((entry) => (entry.conceptTags?.length ?? 0) > 0)
.map((entry) => entry.conceptTags ?? []),
);
invalidEntryCount = Object.keys(asRecord(parsed)?.entries ?? {}).length - entryCount;
if (invalidEntryCount > 0) {
issues.push({
severity: "warn",
code: "recall-store-invalid",
message: `Short-term recall store contains ${invalidEntryCount} invalid entr${invalidEntryCount === 1 ? "y" : "ies"}.`,
fixable: true,
});
}
}
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code !== "ENOENT") {
issues.push({
severity: "error",
code: "recall-store-unreadable",
message: `Short-term recall store is unreadable: ${code ?? "error"}.`,
fixable: false,
});
}
}
try {
const stat = await fs.stat(lockPath);
const ageMs = Date.now() - stat.mtimeMs;
if (ageMs > SHORT_TERM_LOCK_STALE_MS && (await canStealStaleLock(lockPath))) {
issues.push({
severity: "warn",
code: "recall-lock-stale",
message: "Short-term promotion lock appears stale.",
fixable: true,
});
}
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code !== "ENOENT") {
issues.push({
severity: "warn",
code: "recall-lock-unreadable",
message: `Short-term promotion lock could not be inspected: ${code ?? "error"}.`,
fixable: false,
});
}
}
let qmd: ShortTermAuditSummary["qmd"];
if (params.qmd) {
qmd = {
dbPath: params.qmd.dbPath,
collections: params.qmd.collections,
};
if (typeof params.qmd.collections === "number" && params.qmd.collections <= 0) {
issues.push({
severity: "warn",
code: "qmd-collections-empty",
message: "QMD reports zero managed collections.",
fixable: false,
});
}
const dbPath = params.qmd.dbPath?.trim();
if (dbPath) {
try {
const stat = await fs.stat(dbPath);
qmd.dbBytes = stat.size;
if (!stat.isFile() || stat.size <= 0) {
issues.push({
severity: "error",
code: "qmd-index-empty",
message: "QMD index file exists but is empty.",
fixable: false,
});
}
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === "ENOENT") {
issues.push({
severity: "error",
code: "qmd-index-missing",
message: "QMD index file is missing.",
fixable: false,
});
} else {
throw err;
}
}
}
}
return {
storePath,
lockPath,
updatedAt,
exists,
entryCount,
promotedCount,
spacedEntryCount,
conceptTaggedEntryCount,
...(conceptTagScripts ? { conceptTagScripts } : {}),
invalidEntryCount,
issues,
...(qmd ? { qmd } : {}),
};
}
function asRecord(value: unknown): Record<string, unknown> | null {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return null;
}
return value as Record<string, unknown>;
}
export async function repairShortTermPromotionArtifacts(params: {
workspaceDir: string;
}): Promise<RepairShortTermPromotionArtifactsResult> {
const workspaceDir = params.workspaceDir.trim();
const nowIso = new Date().toISOString();
let rewroteStore = false;
let removedInvalidEntries = 0;
let removedStaleLock = false;
try {
const lockPath = resolveLockPath(workspaceDir);
const stat = await fs.stat(lockPath);
const ageMs = Date.now() - stat.mtimeMs;
if (ageMs > SHORT_TERM_LOCK_STALE_MS && (await canStealStaleLock(lockPath))) {
await fs.unlink(lockPath).catch(() => undefined);
removedStaleLock = true;
}
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
throw err;
}
}
await withShortTermLock(workspaceDir, async () => {
const storePath = resolveStorePath(workspaceDir);
try {
const raw = await fs.readFile(storePath, "utf-8");
const parsed = raw.trim().length > 0 ? (JSON.parse(raw) as unknown) : emptyStore(nowIso);
const rawEntries = Object.keys(asRecord(parsed)?.entries ?? {}).length;
const normalized = normalizeStore(parsed, nowIso);
removedInvalidEntries = Math.max(0, rawEntries - Object.keys(normalized.entries).length);
const nextEntries = Object.fromEntries(
Object.entries(normalized.entries).map(([key, entry]) => {
const conceptTags = deriveConceptTags({ path: entry.path, snippet: entry.snippet });
const fallbackDay = normalizeIsoDay(entry.lastRecalledAt) ?? nowIso.slice(0, 10);
return [
key,
{
...entry,
queryHashes: (entry.queryHashes ?? []).slice(-MAX_QUERY_HASHES),
recallDays: mergeRecentDistinct(entry.recallDays ?? [], fallbackDay, MAX_RECALL_DAYS),
conceptTags: conceptTags.length > 0 ? conceptTags : (entry.conceptTags ?? []),
} satisfies ShortTermRecallEntry,
];
}),
);
const comparableStore: ShortTermRecallStore = {
version: 1,
updatedAt: normalized.updatedAt,
entries: nextEntries,
};
const comparableRaw = `${JSON.stringify(comparableStore, null, 2)}\n`;
if (comparableRaw !== `${raw.trimEnd()}\n`) {
await writeStore(workspaceDir, {
...comparableStore,
updatedAt: nowIso,
});
rewroteStore = true;
}
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
throw err;
}
}
});
return {
changed: rewroteStore || removedStaleLock,
removedInvalidEntries,
rewroteStore,
removedStaleLock,
};
}
export const __testing = {
parseLockOwnerPid,
canStealStaleLock,
isProcessLikelyAlive,
deriveConceptTags,
calculateConsolidationComponent,
};

View File

@@ -2,6 +2,7 @@ import path from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import type { checkQmdBinaryAvailability as checkQmdBinaryAvailabilityFn } from "../plugin-sdk/memory-core-host-engine-qmd.js";
import type { DoctorPrompter } from "./doctor-prompter.js";
const note = vi.hoisted(() => vi.fn());
const resolveDefaultAgentId = vi.hoisted(() => vi.fn(() => "agent-default"));
@@ -10,10 +11,13 @@ const resolveAgentWorkspaceDir = vi.hoisted(() => vi.fn(() => "/tmp/agent-defaul
const resolveMemorySearchConfig = vi.hoisted(() => vi.fn());
const resolveApiKeyForProvider = vi.hoisted(() => vi.fn());
const resolveActiveMemoryBackendConfig = vi.hoisted(() => vi.fn());
const getActiveMemorySearchManager = vi.hoisted(() => vi.fn());
type CheckQmdBinaryAvailability = typeof checkQmdBinaryAvailabilityFn;
const checkQmdBinaryAvailability = vi.hoisted(() =>
vi.fn<CheckQmdBinaryAvailability>(async () => ({ available: true })),
);
const auditShortTermPromotionArtifacts = vi.hoisted(() => vi.fn());
const repairShortTermPromotionArtifacts = vi.hoisted(() => vi.fn());
vi.mock("../terminal/note.js", () => ({
note,
@@ -35,13 +39,41 @@ vi.mock("../agents/model-auth.js", () => ({
vi.mock("../plugins/memory-runtime.js", () => ({
resolveActiveMemoryBackendConfig,
getActiveMemorySearchManager,
}));
vi.mock("../plugin-sdk/memory-core-host-engine-qmd.js", () => ({
checkQmdBinaryAvailability,
}));
vi.mock("../plugin-sdk/memory-core-engine-runtime.js", () => ({
auditShortTermPromotionArtifacts,
repairShortTermPromotionArtifacts,
getBuiltinMemoryEmbeddingProviderDoctorMetadata: vi.fn((provider: string) => {
if (provider === "gemini") {
return { authProviderId: "google", envVars: ["GEMINI_API_KEY"] };
}
if (provider === "mistral") {
return { authProviderId: "mistral", envVars: ["MISTRAL_API_KEY"] };
}
if (provider === "openai") {
return { authProviderId: "openai", envVars: ["OPENAI_API_KEY"] };
}
return null;
}),
listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata: vi.fn(() => [
{
providerId: "openai",
authProviderId: "openai",
envVars: ["OPENAI_API_KEY"],
transport: "remote",
},
{ providerId: "local", authProviderId: "local", envVars: [], transport: "local" },
]),
}));
import { noteMemorySearchHealth } from "./doctor-memory-search.js";
import { maybeRepairMemoryRecallHealth, noteMemoryRecallHealth } from "./doctor-memory-search.js";
import { detectLegacyWorkspaceDirs } from "./doctor-workspace.js";
describe("noteMemorySearchHealth", () => {
@@ -70,8 +102,34 @@ describe("noteMemorySearchHealth", () => {
resolveApiKeyForProvider.mockRejectedValue(new Error("missing key"));
resolveActiveMemoryBackendConfig.mockReset();
resolveActiveMemoryBackendConfig.mockReturnValue({ backend: "builtin", citations: "auto" });
getActiveMemorySearchManager.mockReset();
getActiveMemorySearchManager.mockResolvedValue({
manager: {
status: () => ({ workspaceDir: "/tmp/agent-default/workspace", backend: "builtin" }),
close: vi.fn(async () => {}),
},
});
checkQmdBinaryAvailability.mockReset();
checkQmdBinaryAvailability.mockResolvedValue({ available: true });
auditShortTermPromotionArtifacts.mockReset();
auditShortTermPromotionArtifacts.mockResolvedValue({
storePath: "/tmp/agent-default/workspace/memory/.dreams/short-term-recall.json",
lockPath: "/tmp/agent-default/workspace/memory/.dreams/short-term-promotion.lock",
exists: true,
entryCount: 1,
promotedCount: 0,
spacedEntryCount: 0,
conceptTaggedEntryCount: 1,
invalidEntryCount: 0,
issues: [],
});
repairShortTermPromotionArtifacts.mockReset();
repairShortTermPromotionArtifacts.mockResolvedValue({
changed: false,
removedInvalidEntries: 0,
rewroteStore: false,
removedStaleLock: false,
});
});
it("does not warn when local provider is set with no explicit modelPath (default model fallback)", async () => {
@@ -369,6 +427,109 @@ describe("noteMemorySearchHealth", () => {
});
});
describe("memory recall doctor integration", () => {
const cfg = {} as OpenClawConfig;
function createPrompter(overrides: Partial<DoctorPrompter> = {}): DoctorPrompter {
return {
confirm: vi.fn(async () => true),
confirmAutoFix: vi.fn(async () => true),
confirmAggressiveAutoFix: vi.fn(async () => true),
confirmRuntimeRepair: vi.fn(async () => true),
select: vi.fn(async (_params, fallback) => fallback),
shouldRepair: true,
shouldForce: false,
repairMode: {
shouldRepair: true,
shouldForce: false,
nonInteractive: false,
canPrompt: true,
updateInProgress: false,
},
...overrides,
};
}
it("notes recall-store audit problems with doctor guidance", async () => {
auditShortTermPromotionArtifacts.mockResolvedValueOnce({
storePath: "/tmp/agent-default/workspace/memory/.dreams/short-term-recall.json",
lockPath: "/tmp/agent-default/workspace/memory/.dreams/short-term-promotion.lock",
exists: true,
entryCount: 12,
promotedCount: 4,
spacedEntryCount: 2,
conceptTaggedEntryCount: 10,
invalidEntryCount: 1,
issues: [
{
severity: "warn",
code: "recall-store-invalid",
message: "Short-term recall store contains 1 invalid entry.",
fixable: true,
},
{
severity: "warn",
code: "recall-lock-stale",
message: "Short-term promotion lock appears stale.",
fixable: true,
},
],
});
await noteMemoryRecallHealth(cfg);
expect(auditShortTermPromotionArtifacts).toHaveBeenCalledWith({
workspaceDir: "/tmp/agent-default/workspace",
qmd: undefined,
});
expect(note).toHaveBeenCalledTimes(1);
const message = String(note.mock.calls[0]?.[0] ?? "");
expect(message).toContain("Memory recall artifacts need attention:");
expect(message).toContain("doctor --fix");
expect(message).toContain("memory status --fix");
});
it("runs memory recall repair during doctor --fix", async () => {
auditShortTermPromotionArtifacts.mockResolvedValueOnce({
storePath: "/tmp/agent-default/workspace/memory/.dreams/short-term-recall.json",
lockPath: "/tmp/agent-default/workspace/memory/.dreams/short-term-promotion.lock",
exists: true,
entryCount: 12,
promotedCount: 4,
spacedEntryCount: 2,
conceptTaggedEntryCount: 10,
invalidEntryCount: 1,
issues: [
{
severity: "warn",
code: "recall-store-invalid",
message: "Short-term recall store contains 1 invalid entry.",
fixable: true,
},
],
});
repairShortTermPromotionArtifacts.mockResolvedValueOnce({
changed: true,
removedInvalidEntries: 1,
rewroteStore: true,
removedStaleLock: true,
});
const prompter = createPrompter();
await maybeRepairMemoryRecallHealth({ cfg, prompter });
expect(prompter.confirmRuntimeRepair).toHaveBeenCalled();
expect(repairShortTermPromotionArtifacts).toHaveBeenCalledWith({
workspaceDir: "/tmp/agent-default/workspace",
});
expect(note).toHaveBeenCalledTimes(1);
const message = String(note.mock.calls[0]?.[0] ?? "");
expect(message).toContain("Memory recall artifacts repaired:");
expect(message).toContain("rewrote recall store");
expect(message).toContain("removed stale promotion lock");
});
});
describe("detectLegacyWorkspaceDirs", () => {
it("returns active workspace and no legacy dirs", () => {
const workspaceDir = "/home/user/openclaw";

View File

@@ -9,15 +9,22 @@ import { resolveApiKeyForProvider } from "../agents/model-auth.js";
import { formatCliCommand } from "../cli/command-format.js";
import type { OpenClawConfig } from "../config/config.js";
import {
auditShortTermPromotionArtifacts,
getBuiltinMemoryEmbeddingProviderDoctorMetadata,
listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata,
repairShortTermPromotionArtifacts,
type ShortTermAuditSummary,
} from "../plugin-sdk/memory-core-engine-runtime.js";
import { DEFAULT_LOCAL_MODEL } from "../plugin-sdk/memory-core-host-engine-embeddings.js";
import { checkQmdBinaryAvailability } from "../plugin-sdk/memory-core-host-engine-qmd.js";
import { hasConfiguredMemorySecretInput } from "../plugin-sdk/memory-core-host-secret.js";
import { resolveActiveMemoryBackendConfig } from "../plugins/memory-runtime.js";
import {
getActiveMemorySearchManager,
resolveActiveMemoryBackendConfig,
} from "../plugins/memory-runtime.js";
import { note } from "../terminal/note.js";
import { resolveUserPath } from "../utils.js";
import type { DoctorPrompter } from "./doctor-prompter.js";
function resolveSuggestedRemoteMemoryProvider(): string | undefined {
return listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata().find(
@@ -25,6 +32,146 @@ function resolveSuggestedRemoteMemoryProvider(): string | undefined {
)?.providerId;
}
type RuntimeMemoryAuditContext = {
workspaceDir?: string;
backend?: string;
dbPath?: string;
qmdCollections?: number;
};
function asRecord(value: unknown): Record<string, unknown> | null {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return null;
}
return value as Record<string, unknown>;
}
async function resolveRuntimeMemoryAuditContext(
cfg: OpenClawConfig,
): Promise<RuntimeMemoryAuditContext | null> {
const agentId = resolveDefaultAgentId(cfg);
const result = await getActiveMemorySearchManager({
cfg,
agentId,
purpose: "status",
});
const manager = result.manager;
if (!manager) {
return null;
}
try {
const status = manager.status();
const customQmd = asRecord(asRecord(status.custom)?.qmd);
return {
workspaceDir: status.workspaceDir?.trim(),
backend: status.backend,
dbPath: status.dbPath,
qmdCollections:
typeof customQmd?.collections === "number" ? customQmd.collections : undefined,
};
} finally {
await manager.close?.().catch(() => undefined);
}
}
function buildMemoryRecallIssueNote(audit: ShortTermAuditSummary): string | null {
if (audit.issues.length === 0) {
return null;
}
const issueLines = audit.issues.map((issue) => `- ${issue.message}`);
const hasFixableIssue = audit.issues.some((issue) => issue.fixable);
const guidance = hasFixableIssue
? `Fix: ${formatCliCommand("openclaw doctor --fix")} or ${formatCliCommand("openclaw memory status --fix")}`
: `Verify: ${formatCliCommand("openclaw memory status --deep")}`;
return [
"Memory recall artifacts need attention:",
...issueLines,
`Recall store: ${audit.storePath}`,
guidance,
].join("\n");
}
export async function noteMemoryRecallHealth(cfg: OpenClawConfig): Promise<void> {
try {
const context = await resolveRuntimeMemoryAuditContext(cfg);
const workspaceDir = context?.workspaceDir?.trim();
if (!workspaceDir) {
return;
}
const audit = await auditShortTermPromotionArtifacts({
workspaceDir,
qmd:
context?.backend === "qmd"
? {
dbPath: context.dbPath,
collections: context.qmdCollections,
}
: undefined,
});
const message = buildMemoryRecallIssueNote(audit);
if (message) {
note(message, "Memory search");
}
} catch (err) {
note(
`Memory recall audit could not be completed: ${err instanceof Error ? err.message : String(err)}`,
"Memory search",
);
}
}
export async function maybeRepairMemoryRecallHealth(params: {
cfg: OpenClawConfig;
prompter: DoctorPrompter;
}): Promise<void> {
try {
const context = await resolveRuntimeMemoryAuditContext(params.cfg);
const workspaceDir = context?.workspaceDir?.trim();
if (!workspaceDir) {
return;
}
const audit = await auditShortTermPromotionArtifacts({
workspaceDir,
qmd:
context?.backend === "qmd"
? {
dbPath: context.dbPath,
collections: context.qmdCollections,
}
: undefined,
});
const hasFixableIssue = audit.issues.some((issue) => issue.fixable);
if (!hasFixableIssue) {
return;
}
const approved = await params.prompter.confirmRuntimeRepair({
message: "Normalize memory recall artifacts and remove stale promotion locks?",
initialValue: true,
});
if (!approved) {
return;
}
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
if (!repair.changed) {
return;
}
const lines = [
"Memory recall artifacts repaired:",
repair.rewroteStore
? `- rewrote recall store${repair.removedInvalidEntries > 0 ? ` (-${repair.removedInvalidEntries} invalid entries)` : ""}`
: null,
repair.removedStaleLock ? "- removed stale promotion lock" : null,
`Verify: ${formatCliCommand("openclaw memory status --deep")}`,
].filter(Boolean);
note(lines.join("\n"), "Doctor changes");
} catch (err) {
note(
`Memory recall repair could not be completed: ${err instanceof Error ? err.message : String(err)}`,
"Memory search",
);
}
}
/**
* Check whether memory search has a usable embedding provider.
* Runs as part of `openclaw doctor` — config-only checks where possible;

View File

@@ -25,7 +25,11 @@ import {
maybeRepairGatewayServiceConfig,
maybeScanExtraGatewayServices,
} from "../commands/doctor-gateway-services.js";
import { noteMemorySearchHealth } from "../commands/doctor-memory-search.js";
import {
maybeRepairMemoryRecallHealth,
noteMemoryRecallHealth,
noteMemorySearchHealth,
} from "../commands/doctor-memory-search.js";
import {
noteMacLaunchAgentOverrides,
noteMacLaunchctlGatewayEnvOverrides,
@@ -416,9 +420,14 @@ async function runGatewayHealthChecks(ctx: DoctorHealthFlowContext): Promise<voi
}
async function runMemorySearchHealthContribution(ctx: DoctorHealthFlowContext): Promise<void> {
await maybeRepairMemoryRecallHealth({
cfg: ctx.cfg,
prompter: ctx.prompter,
});
await noteMemorySearchHealth(ctx.cfg, {
gatewayMemoryProbe: ctx.gatewayMemoryProbe ?? { checked: false, ready: false },
});
await noteMemoryRecallHealth(ctx.cfg);
}
async function runGatewayDaemonHealth(ctx: DoctorHealthFlowContext): Promise<void> {

View File

@@ -214,6 +214,8 @@ export interface PluginSdkFacadeTypeMap {
};
types: {
BuiltinMemoryEmbeddingProviderDoctorMetadata: import("@openclaw/memory-core/runtime-api.js").BuiltinMemoryEmbeddingProviderDoctorMetadata;
RepairShortTermPromotionArtifactsResult: import("@openclaw/memory-core/runtime-api.js").RepairShortTermPromotionArtifactsResult;
ShortTermAuditSummary: import("@openclaw/memory-core/runtime-api.js").ShortTermAuditSummary;
};
};
"mattermost-policy": {

View File

@@ -18,6 +18,12 @@ export const getBuiltinMemoryEmbeddingProviderDoctorMetadata: FacadeModule["getB
loadFacadeModule()["getBuiltinMemoryEmbeddingProviderDoctorMetadata"](
...args,
)) as FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"];
export const auditShortTermPromotionArtifacts: FacadeModule["auditShortTermPromotionArtifacts"] = ((
...args
) =>
loadFacadeModule()["auditShortTermPromotionArtifacts"](
...args,
)) as FacadeModule["auditShortTermPromotionArtifacts"];
export const getMemorySearchManager: FacadeModule["getMemorySearchManager"] = ((...args) =>
loadFacadeModule()["getMemorySearchManager"](...args)) as FacadeModule["getMemorySearchManager"];
export const listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata: FacadeModule["listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata"] =
@@ -28,5 +34,13 @@ export const listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata: FacadeM
export const MemoryIndexManager: FacadeModule["MemoryIndexManager"] = createLazyFacadeObjectValue(
() => loadFacadeModule()["MemoryIndexManager"] as object,
) as FacadeModule["MemoryIndexManager"];
export const repairShortTermPromotionArtifacts: FacadeModule["repairShortTermPromotionArtifacts"] =
((...args) =>
loadFacadeModule()["repairShortTermPromotionArtifacts"](
...args,
)) as FacadeModule["repairShortTermPromotionArtifacts"];
export type BuiltinMemoryEmbeddingProviderDoctorMetadata =
FacadeEntry["types"]["BuiltinMemoryEmbeddingProviderDoctorMetadata"];
export type RepairShortTermPromotionArtifactsResult =
FacadeEntry["types"]["RepairShortTermPromotionArtifactsResult"];
export type ShortTermAuditSummary = FacadeEntry["types"]["ShortTermAuditSummary"];