test: dedupe hot unit fast coverage

This commit is contained in:
Peter Steinberger
2026-04-27 10:02:42 +01:00
parent edbab0e2db
commit 3f895e5b49
6 changed files with 183 additions and 184 deletions

View File

@@ -1,38 +0,0 @@
import { describe, expect, it } from "vitest";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import { resolveApiKeyFromCredential } from "./models-config.providers.secret-helpers.js";
describe("provider discovery auth marker guardrails", () => {
it("suppresses discovery secrets for marker-backed vLLM credentials", () => {
const resolved = resolveApiKeyFromCredential({
type: "api_key",
provider: "vllm",
keyRef: { source: "file", provider: "vault", id: "/vllm/apiKey" },
});
expect(resolved?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
expect(resolved?.discoveryApiKey).toBeUndefined();
});
it("suppresses discovery secrets for marker-backed Hugging Face credentials", () => {
const resolved = resolveApiKeyFromCredential({
type: "api_key",
provider: "huggingface",
keyRef: { source: "exec", provider: "vault", id: "providers/hf/token" },
});
expect(resolved?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
expect(resolved?.discoveryApiKey).toBeUndefined();
});
it("keeps all-caps plaintext API keys for authenticated discovery", () => {
const resolved = resolveApiKeyFromCredential({
type: "api_key",
provider: "vllm",
key: "ALLCAPS_SAMPLE",
});
expect(resolved?.apiKey).toBe("ALLCAPS_SAMPLE");
expect(resolved?.discoveryApiKey).toBe("ALLCAPS_SAMPLE");
});
});

View File

@@ -1,6 +1,8 @@
import { describe, expect, it } from "vitest";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import type { ProviderConfig } from "./models-config.providers.secret-helpers.js";
import {
resolveApiKeyFromCredential,
resolveAwsSdkApiKeyVarName,
resolveMissingProviderApiKey,
} from "./models-config.providers.secret-helpers.js";
@@ -156,3 +158,38 @@ describe("resolveAwsSdkApiKeyVarName", () => {
).toBe("AWS_PROFILE");
});
});
describe("provider discovery auth marker guardrails", () => {
it("suppresses discovery secrets for marker-backed vLLM credentials", () => {
const resolved = resolveApiKeyFromCredential({
type: "api_key",
provider: "vllm",
keyRef: { source: "file", provider: "vault", id: "/vllm/apiKey" },
});
expect(resolved?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
expect(resolved?.discoveryApiKey).toBeUndefined();
});
it("suppresses discovery secrets for marker-backed Hugging Face credentials", () => {
const resolved = resolveApiKeyFromCredential({
type: "api_key",
provider: "huggingface",
keyRef: { source: "exec", provider: "vault", id: "providers/hf/token" },
});
expect(resolved?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
expect(resolved?.discoveryApiKey).toBeUndefined();
});
it("keeps all-caps plaintext API keys for authenticated discovery", () => {
const resolved = resolveApiKeyFromCredential({
type: "api_key",
provider: "vllm",
key: "ALLCAPS_SAMPLE",
});
expect(resolved?.apiKey).toBe("ALLCAPS_SAMPLE");
expect(resolved?.discoveryApiKey).toBe("ALLCAPS_SAMPLE");
});
});

View File

@@ -6,10 +6,20 @@ import {
readMemoryHostEvents,
resolveMemoryHostEventLogPath,
} from "./memory-host-events.js";
import { createClaimableDedupe, createPersistentDedupe } from "./persistent-dedupe.js";
import { createPluginSdkTestHarness } from "./test-helpers.js";
const { createTempDir } = createPluginSdkTestHarness();
function createDedupe(root: string, overrides?: { ttlMs?: number }) {
return createPersistentDedupe({
ttlMs: overrides?.ttlMs ?? 24 * 60 * 60 * 1000,
memoryMaxSize: 100,
fileMaxEntries: 1000,
resolveFilePath: (namespace) => path.join(root, `${namespace}.json`),
});
}
describe("memory host event journal helpers", () => {
it("appends and reads typed workspace events", async () => {
const workspaceDir = await createTempDir("memory-host-events-");
@@ -53,3 +63,133 @@ describe("memory host event journal helpers", () => {
expect(tail[0]?.type).toBe("memory.dream.completed");
});
});
describe("createPersistentDedupe", () => {
it("deduplicates keys, persists across instances, warms up, and checks recent keys", async () => {
const root = await createTempDir("openclaw-dedupe-");
const first = createDedupe(root);
expect(await first.checkAndRecord("m1", { namespace: "a" })).toBe(true);
expect(await first.checkAndRecord("m1", { namespace: "a" })).toBe(false);
expect(await first.checkAndRecord("m2", { namespace: "a" })).toBe(true);
const second = createDedupe(root);
expect(await second.hasRecent("m1", { namespace: "a" })).toBe(true);
expect(await second.hasRecent("missing", { namespace: "a" })).toBe(false);
expect(await second.warmup("a")).toBe(2);
expect(await second.checkAndRecord("m1", { namespace: "a" })).toBe(false);
expect(await second.checkAndRecord("m2", { namespace: "a" })).toBe(false);
expect(await second.checkAndRecord("m3", { namespace: "a" })).toBe(true);
expect(await second.checkAndRecord("m1", { namespace: "b" })).toBe(true);
const raceDedupe = createDedupe(root, { ttlMs: 10_000 });
const [raceFirst, raceSecond] = await Promise.all([
raceDedupe.checkAndRecord("race-key", { namespace: "feishu" }),
raceDedupe.checkAndRecord("race-key", { namespace: "feishu" }),
]);
expect(raceFirst).toBe(true);
expect(raceSecond).toBe(false);
});
it("falls back to memory-only behavior on disk errors", async () => {
const dedupe = createPersistentDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
fileMaxEntries: 1000,
resolveFilePath: () => path.join("/dev/null", "dedupe.json"),
});
expect(await dedupe.checkAndRecord("memory-only", { namespace: "x" })).toBe(true);
expect(await dedupe.checkAndRecord("memory-only", { namespace: "x" })).toBe(false);
});
it("warms empty namespaces and skips expired disk entries", async () => {
const root = await createTempDir("openclaw-dedupe-");
const emptyReader = createDedupe(root, { ttlMs: 10_000 });
expect(await emptyReader.warmup("nonexistent")).toBe(0);
const writer = createDedupe(root, { ttlMs: 1000 });
const oldNow = Date.now() - 2000;
expect(await writer.checkAndRecord("old-msg", { namespace: "acct", now: oldNow })).toBe(true);
expect(await writer.checkAndRecord("new-msg", { namespace: "acct" })).toBe(true);
const reader = createDedupe(root, { ttlMs: 1000 });
expect(await reader.warmup("acct")).toBe(1);
expect(await reader.checkAndRecord("old-msg", { namespace: "acct" })).toBe(true);
expect(await reader.checkAndRecord("new-msg", { namespace: "acct" })).toBe(false);
});
});
describe("createClaimableDedupe", () => {
it("mirrors in-flight duplicates, serializes races, and records on commit", async () => {
const dedupe = createClaimableDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
});
await expect(dedupe.claim("line:evt-1")).resolves.toEqual({ kind: "claimed" });
const duplicate = await dedupe.claim("line:evt-1");
expect(duplicate.kind).toBe("inflight");
const commit = dedupe.commit("line:evt-1");
await expect(commit).resolves.toBe(true);
if (duplicate.kind === "inflight") {
await expect(duplicate.pending).resolves.toBe(true);
}
await expect(dedupe.claim("line:evt-1")).resolves.toEqual({ kind: "duplicate" });
const claims = await Promise.all([dedupe.claim("line:race-1"), dedupe.claim("line:race-1")]);
expect(claims.filter((claim) => claim.kind === "claimed")).toHaveLength(1);
expect(claims.filter((claim) => claim.kind === "inflight")).toHaveLength(1);
const waitingClaim = claims.find((claim) => claim.kind === "inflight");
await expect(dedupe.commit("line:race-1")).resolves.toBe(true);
if (waitingClaim?.kind === "inflight") {
await expect(waitingClaim.pending).resolves.toBe(true);
}
await expect(dedupe.claim("line:race-1")).resolves.toEqual({ kind: "duplicate" });
});
it("rejects waiting duplicates when the active claim releases with an error", async () => {
const dedupe = createClaimableDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
});
await expect(dedupe.claim("line:evt-2")).resolves.toEqual({ kind: "claimed" });
const duplicate = await dedupe.claim("line:evt-2");
expect(duplicate.kind).toBe("inflight");
const failure = new Error("transient failure");
dedupe.release("line:evt-2", { error: failure });
if (duplicate.kind === "inflight") {
await expect(duplicate.pending).rejects.toThrow("transient failure");
}
await expect(dedupe.claim("line:evt-2")).resolves.toEqual({ kind: "claimed" });
});
it("supports persistent-backed recent checks and warmup", async () => {
const root = await createTempDir("openclaw-claimable-dedupe-");
const writer = createClaimableDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
fileMaxEntries: 1000,
resolveFilePath: (namespace) => path.join(root, `${namespace}.json`),
});
await expect(writer.claim("m1", { namespace: "acct" })).resolves.toEqual({ kind: "claimed" });
await expect(writer.commit("m1", { namespace: "acct" })).resolves.toBe(true);
const reader = createClaimableDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
fileMaxEntries: 1000,
resolveFilePath: (namespace) => path.join(root, `${namespace}.json`),
});
expect(await reader.hasRecent("m1", { namespace: "acct" })).toBe(true);
expect(await reader.warmup("acct")).toBe(1);
await expect(reader.claim("m1", { namespace: "acct" })).resolves.toEqual({
kind: "duplicate",
});
});
});

View File

@@ -1,145 +0,0 @@
import path from "node:path";
import { describe, expect, it } from "vitest";
import { createClaimableDedupe, createPersistentDedupe } from "./persistent-dedupe.js";
import { createPluginSdkTestHarness } from "./test-helpers.js";
const { createTempDir } = createPluginSdkTestHarness();
function createDedupe(root: string, overrides?: { ttlMs?: number }) {
return createPersistentDedupe({
ttlMs: overrides?.ttlMs ?? 24 * 60 * 60 * 1000,
memoryMaxSize: 100,
fileMaxEntries: 1000,
resolveFilePath: (namespace) => path.join(root, `${namespace}.json`),
});
}
describe("createPersistentDedupe", () => {
it("deduplicates keys, persists across instances, warms up, and checks recent keys", async () => {
const root = await createTempDir("openclaw-dedupe-");
const first = createDedupe(root);
expect(await first.checkAndRecord("m1", { namespace: "a" })).toBe(true);
expect(await first.checkAndRecord("m1", { namespace: "a" })).toBe(false);
expect(await first.checkAndRecord("m2", { namespace: "a" })).toBe(true);
const second = createDedupe(root);
expect(await second.hasRecent("m1", { namespace: "a" })).toBe(true);
expect(await second.hasRecent("missing", { namespace: "a" })).toBe(false);
expect(await second.warmup("a")).toBe(2);
expect(await second.checkAndRecord("m1", { namespace: "a" })).toBe(false);
expect(await second.checkAndRecord("m2", { namespace: "a" })).toBe(false);
expect(await second.checkAndRecord("m3", { namespace: "a" })).toBe(true);
expect(await second.checkAndRecord("m1", { namespace: "b" })).toBe(true);
const raceDedupe = createDedupe(root, { ttlMs: 10_000 });
const [raceFirst, raceSecond] = await Promise.all([
raceDedupe.checkAndRecord("race-key", { namespace: "feishu" }),
raceDedupe.checkAndRecord("race-key", { namespace: "feishu" }),
]);
expect(raceFirst).toBe(true);
expect(raceSecond).toBe(false);
});
it("falls back to memory-only behavior on disk errors", async () => {
const dedupe = createPersistentDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
fileMaxEntries: 1000,
resolveFilePath: () => path.join("/dev/null", "dedupe.json"),
});
expect(await dedupe.checkAndRecord("memory-only", { namespace: "x" })).toBe(true);
expect(await dedupe.checkAndRecord("memory-only", { namespace: "x" })).toBe(false);
});
it("warms empty namespaces and skips expired disk entries", async () => {
const root = await createTempDir("openclaw-dedupe-");
const emptyReader = createDedupe(root, { ttlMs: 10_000 });
expect(await emptyReader.warmup("nonexistent")).toBe(0);
const writer = createDedupe(root, { ttlMs: 1000 });
const oldNow = Date.now() - 2000;
expect(await writer.checkAndRecord("old-msg", { namespace: "acct", now: oldNow })).toBe(true);
expect(await writer.checkAndRecord("new-msg", { namespace: "acct" })).toBe(true);
const reader = createDedupe(root, { ttlMs: 1000 });
expect(await reader.warmup("acct")).toBe(1);
expect(await reader.checkAndRecord("old-msg", { namespace: "acct" })).toBe(true);
expect(await reader.checkAndRecord("new-msg", { namespace: "acct" })).toBe(false);
});
});
describe("createClaimableDedupe", () => {
it("mirrors in-flight duplicates, serializes races, and records on commit", async () => {
const dedupe = createClaimableDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
});
await expect(dedupe.claim("line:evt-1")).resolves.toEqual({ kind: "claimed" });
const duplicate = await dedupe.claim("line:evt-1");
expect(duplicate.kind).toBe("inflight");
const commit = dedupe.commit("line:evt-1");
await expect(commit).resolves.toBe(true);
if (duplicate.kind === "inflight") {
await expect(duplicate.pending).resolves.toBe(true);
}
await expect(dedupe.claim("line:evt-1")).resolves.toEqual({ kind: "duplicate" });
const claims = await Promise.all([dedupe.claim("line:race-1"), dedupe.claim("line:race-1")]);
expect(claims.filter((claim) => claim.kind === "claimed")).toHaveLength(1);
expect(claims.filter((claim) => claim.kind === "inflight")).toHaveLength(1);
const waitingClaim = claims.find((claim) => claim.kind === "inflight");
await expect(dedupe.commit("line:race-1")).resolves.toBe(true);
if (waitingClaim?.kind === "inflight") {
await expect(waitingClaim.pending).resolves.toBe(true);
}
await expect(dedupe.claim("line:race-1")).resolves.toEqual({ kind: "duplicate" });
});
it("rejects waiting duplicates when the active claim releases with an error", async () => {
const dedupe = createClaimableDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
});
await expect(dedupe.claim("line:evt-2")).resolves.toEqual({ kind: "claimed" });
const duplicate = await dedupe.claim("line:evt-2");
expect(duplicate.kind).toBe("inflight");
const failure = new Error("transient failure");
dedupe.release("line:evt-2", { error: failure });
if (duplicate.kind === "inflight") {
await expect(duplicate.pending).rejects.toThrow("transient failure");
}
await expect(dedupe.claim("line:evt-2")).resolves.toEqual({ kind: "claimed" });
});
it("supports persistent-backed recent checks and warmup", async () => {
const root = await createTempDir("openclaw-claimable-dedupe-");
const writer = createClaimableDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
fileMaxEntries: 1000,
resolveFilePath: (namespace) => path.join(root, `${namespace}.json`),
});
await expect(writer.claim("m1", { namespace: "acct" })).resolves.toEqual({ kind: "claimed" });
await expect(writer.commit("m1", { namespace: "acct" })).resolves.toBe(true);
const reader = createClaimableDedupe({
ttlMs: 10_000,
memoryMaxSize: 100,
fileMaxEntries: 1000,
resolveFilePath: (namespace) => path.join(root, `${namespace}.json`),
});
expect(await reader.hasRecent("m1", { namespace: "acct" })).toBe(true);
expect(await reader.warmup("acct")).toBe(1);
await expect(reader.claim("m1", { namespace: "acct" })).resolves.toEqual({
kind: "duplicate",
});
});
});

View File

@@ -8,9 +8,13 @@ const pluginSdkLightEntries = [
test: "src/plugin-sdk/keyed-async-queue.test.ts",
},
{ source: "src/plugin-sdk/lazy-value.ts", test: "src/plugin-sdk/lazy-value.test.ts" },
{
source: "src/plugin-sdk/memory-host-events.ts",
test: "src/plugin-sdk/memory-host-events.test.ts",
},
{
source: "src/plugin-sdk/persistent-dedupe.ts",
test: "src/plugin-sdk/persistent-dedupe.test.ts",
test: "src/plugin-sdk/memory-host-events.test.ts",
},
{ source: "src/plugin-sdk/provider-entry.ts", test: "src/plugin-sdk/provider-entry.test.ts" },
{

View File

@@ -59,6 +59,7 @@ export const forcedUnitFastTestFiles = [
"src/node-host/invoke-system-run-plan.test.ts",
"src/node-host/invoke-system-run.test.ts",
"src/pairing/pairing-store.test.ts",
"src/plugin-sdk/memory-host-events.test.ts",
];
const forcedUnitFastTestFileSet = new Set(forcedUnitFastTestFiles);
const unitFastCandidateExactFiles = [...pluginSdkLightTestFiles, ...commandsLightTestFiles];