From b2dec9cd50461205dfb79f4b7c6a99ffbc15fe6b Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Wed, 10 Jun 2026 12:55:23 +0530 Subject: [PATCH] feat(plugin-sdk): expose persistent dedupe import helpers --- src/plugin-sdk/memory-host-events.test.ts | 33 ++++++- src/plugin-sdk/persistent-dedupe.ts | 115 ++++++++++++++++++---- src/plugin-sdk/runtime-doctor.ts | 1 + 3 files changed, 129 insertions(+), 20 deletions(-) diff --git a/src/plugin-sdk/memory-host-events.test.ts b/src/plugin-sdk/memory-host-events.test.ts index 7b7d747f5adc..cc2a214988f5 100644 --- a/src/plugin-sdk/memory-host-events.test.ts +++ b/src/plugin-sdk/memory-host-events.test.ts @@ -10,7 +10,11 @@ import { readMemoryHostEvents, resolveMemoryHostEventLogPath, } from "./memory-host-events.js"; -import { createClaimableDedupe, createPersistentDedupe } from "./persistent-dedupe.js"; +import { + createClaimableDedupe, + createPersistentDedupe, + listPersistentDedupeLegacyJsonFileEntries, +} from "./persistent-dedupe.js"; import { createPluginSdkTestHarness } from "./test-helpers.js"; const { createTempDir } = createPluginSdkTestHarness(); @@ -129,6 +133,33 @@ describe("createPersistentDedupe", () => { await expect(fs.access(legacyPath)).rejects.toThrow(); }); + it("lists retired JSON cache files as persistent dedupe entries", async () => { + const root = await createTempDir("openclaw-legacy-dedupe-"); + const legacyPath = path.join(root, "legacy.json"); + await fs.writeFile( + legacyPath, + JSON.stringify({ + fresh: 1_000, + expired: 100, + invalid: "bad", + }), + ); + + await expect( + listPersistentDedupeLegacyJsonFileEntries({ + filePath: legacyPath, + ttlMs: 500, + now: 1_100, + }), + ).resolves.toStrictEqual([ + { + key: expect.stringMatching(/^k\.[a-f0-9]{32}$/), + value: { key: "fresh", seenAt: 1_000 }, + ttlMs: 400, + }, + ]); + }); + it("warms empty namespaces and ignores retired JSON cache files", async () => { const root = await createTempDir("openclaw-dedupe-"); const emptyReader = createDedupe(root, { ttlMs: 10_000 }); diff --git a/src/plugin-sdk/persistent-dedupe.ts b/src/plugin-sdk/persistent-dedupe.ts index 7aafa6f919dc..0d19b51106eb 100644 --- a/src/plugin-sdk/persistent-dedupe.ts +++ b/src/plugin-sdk/persistent-dedupe.ts @@ -13,7 +13,7 @@ import type { FileLockOptions } from "./file-lock.js"; const LEGACY_PATH_OWNER_ID = "core:persistent-dedupe"; const DEFAULT_NAMESPACE_PREFIX = "persistent-dedupe"; -type PersistentDedupeEntry = { +export type PersistentDedupeEntry = { key: string; seenAt: number; }; @@ -76,6 +76,18 @@ export type PersistentDedupeLegacyJsonMigrationOptions = PersistentDedupePluginS removeFile?: boolean; }; +export type PersistentDedupeLegacyJsonImportEntry = { + key: string; + value: PersistentDedupeEntry; + ttlMs?: number; +}; + +type PersistentDedupeLegacyJsonEntriesResult = { + entries: PersistentDedupeLegacyJsonImportEntry[]; + skippedExpired: number; + skippedInvalid: number; +}; + /** Per-call options used when checking or recording a dedupe key. */ export type PersistentDedupeCheckOptions = { /** Logical bucket for the key; omitted/blank values use `global`. */ @@ -168,6 +180,15 @@ function resolveEntrySeenAt(entry: PersistentDedupeEntry | undefined): number | : undefined; } +function resolveUnknownEntrySeenAt(value: unknown): number | undefined { + if (!value || typeof value !== "object" || !("seenAt" in value)) { + return undefined; + } + return typeof value.seenAt === "number" && Number.isFinite(value.seenAt) + ? value.seenAt + : undefined; +} + function shortHash(value: string): string { return createHash("sha256").update(value).digest("hex").slice(0, 32); } @@ -176,6 +197,18 @@ function resolveEntryKey(key: string): string { return `k.${shortHash(key)}`; } +export function createPersistentDedupeImportEntry(params: { + key: string; + seenAt: number; + ttlMs?: number; +}): PersistentDedupeLegacyJsonImportEntry { + return { + key: resolveEntryKey(params.key), + value: { key: params.key, seenAt: params.seenAt }, + ...(params.ttlMs != null ? { ttlMs: params.ttlMs } : {}), + }; +} + function resolveRemainingTtlMs( seenAt: number, ttlMs: number, @@ -202,6 +235,16 @@ function resolveStateNamespace(prefix: string, namespace: string): string { return `${prefix}.${shortHash(namespace)}`; } +export function resolvePersistentDedupePluginStateNamespace(options: { + namespace: string; + namespacePrefix?: string; +}): string { + return resolveStateNamespace( + normalizeNamespacePrefix(options.namespacePrefix), + resolveNamespace(options.namespace), + ); +} + function hasPluginStateOptions( options: ClaimableDedupeOptions | PersistentDedupeOptions, ): options is PersistentDedupePluginStateOptions { @@ -295,39 +338,74 @@ function parseLegacyDedupeData(raw: string): { return { data, invalidCount }; } -/** Import one retired JSON dedupe cache file into plugin-state SQLite during doctor repair. */ -export async function migratePersistentDedupeLegacyJsonFile( - options: PersistentDedupeLegacyJsonMigrationOptions, -): Promise { +async function readPersistentDedupeLegacyJsonFileEntries(options: { + filePath: string; + ttlMs: number; + now?: number; +}): Promise { const raw = await fs.readFile(options.filePath, "utf8"); const { data, invalidCount } = parseLegacyDedupeData(raw); const ttlMs = resolveNonNegativeIntegerOption(options.ttlMs, 0); const now = options.now ?? Date.now(); - const store = createPersistentStoreResolver(options)(resolveNamespace(options.namespace)); - const result: PersistentDedupeLegacyJsonMigrationResult = { - imported: 0, - skippedExpired: 0, - skippedInvalid: 0, - skippedExisting: 0, - removed: false, - }; + const entries: PersistentDedupeLegacyJsonImportEntry[] = []; + let skippedExpired = 0; for (const [key, seenAt] of Object.entries(data)) { const ttlOption = resolveRemainingTtlMs(seenAt, ttlMs, now); if (ttlOption === null) { - result.skippedExpired++; + skippedExpired++; continue; } + entries.push(createPersistentDedupeImportEntry({ key, seenAt, ...ttlOption })); + } + + return { entries, skippedExpired, skippedInvalid: invalidCount }; +} + +export async function listPersistentDedupeLegacyJsonFileEntries(options: { + filePath: string; + ttlMs: number; + now?: number; +}): Promise { + return (await readPersistentDedupeLegacyJsonFileEntries(options)).entries; +} + +export function shouldReplacePersistentDedupeEntry(params: { + existingValue: unknown; + incomingValue: unknown; +}): boolean { + const incomingSeenAt = resolveUnknownEntrySeenAt(params.incomingValue); + return ( + incomingSeenAt != null && + incomingSeenAt > (resolveUnknownEntrySeenAt(params.existingValue) ?? 0) + ); +} + +/** Import one retired JSON dedupe cache file into plugin-state SQLite during doctor repair. */ +export async function migratePersistentDedupeLegacyJsonFile( + options: PersistentDedupeLegacyJsonMigrationOptions, +): Promise { + const legacy = await readPersistentDedupeLegacyJsonFileEntries(options); + const store = createPersistentStoreResolver(options)(resolveNamespace(options.namespace)); + const result: PersistentDedupeLegacyJsonMigrationResult = { + imported: 0, + skippedExpired: legacy.skippedExpired, + skippedInvalid: legacy.skippedInvalid, + skippedExisting: 0, + removed: false, + }; + + for (const entry of legacy.entries) { const changed = store.update?.( - resolveEntryKey(key), + entry.key, (current) => { const currentSeenAt = resolveEntrySeenAt(current); - if (currentSeenAt != null && currentSeenAt >= seenAt) { + if (currentSeenAt != null && currentSeenAt >= entry.value.seenAt) { return undefined; } - return { key, seenAt }; + return entry.value; }, - ttlOption, + entry.ttlMs != null ? { ttlMs: entry.ttlMs } : undefined, ); if (changed) { result.imported++; @@ -336,7 +414,6 @@ export async function migratePersistentDedupeLegacyJsonFile( } } - result.skippedInvalid = invalidCount; if (options.removeFile !== false) { await fs.rm(options.filePath, { force: true }); result.removed = true; diff --git a/src/plugin-sdk/runtime-doctor.ts b/src/plugin-sdk/runtime-doctor.ts index 96c12e378f94..b8d35ee130f8 100644 --- a/src/plugin-sdk/runtime-doctor.ts +++ b/src/plugin-sdk/runtime-doctor.ts @@ -23,6 +23,7 @@ export type { OpenKeyedStoreOptions, PluginStateKeyedStore, } from "../plugin-state/plugin-state-store.js"; +export { createPluginStateSyncKeyedStore } from "../plugin-state/plugin-state-store.js"; export { removePluginFromConfig } from "../plugins/uninstall.js"; export type { PluginDoctorStateMigration,