fix(gateway): bound costUsageCache with MAX + FIFO eviction

Regression: `costUsageCache` in `src/gateway/server-methods/usage.ts` had no
delete/prune/evict path. The TTL check at L310 only gates stale reads — on a
miss after expiry, `set()` overwrites the same key but never removes stale
keys. `parseDateRange` derives cacheKey from `getTodayStartMs`, so cacheKey
rolls at every UTC 00:00, and additional axes (days / startDate / endDate /
utcOffset) multiply cardinality. The macOS menu polls `usage.cost` every ~45s
with no params, exercising `parseDateRange`'s default branch every day. Over
gateway uptime the map grows monotonically.

Three sibling caches in the same subsystem already implement MAX + FIFO
eviction (resolvedSessionKeyByRunId, TRANSCRIPT_SESSION_KEY_CACHE,
sessionTitleFieldsCache). This change mirrors their pattern:

- `COST_USAGE_CACHE_MAX = 256` (matches RUN_LOOKUP_CACHE_LIMIT and
  TRANSCRIPT_SESSION_KEY_CACHE_MAX).
- New `setCostUsageCache(cacheKey, entry)` helper checks size + evicts
  `keys().next().value` when adding a new key would exceed the cap.
- The three existing `costUsageCache.set(...)` call sites now route through
  the helper. TTL-on-read, in-flight dedup, and overwrite-on-same-key
  semantics are preserved.

Adds `src/gateway/server-methods/usage.cost-usage-cache.test.ts` which drives
growth through `__test.loadCostUsageSummaryCached` with 600 distinct
(startMs, endMs) pairs (mirrors day rollover + range switches). Pre-fix the
Map grows to 600; post-fix it plateaus, the last key is retained, and the
first key is evicted (FIFO).

AI-assisted (fully tested). 432 server-methods tests pass, pnpm check +
pnpm build clean.
This commit is contained in:
Feelw00
2026-04-19 15:29:25 +09:00
committed by Peter Steinberger
parent 0d305839e5
commit 8bf57e8bde
2 changed files with 108 additions and 3 deletions

View File

@@ -0,0 +1,90 @@
// Regression: costUsageCache (usage.ts:65) has no production delete/prune/evict
// path. The TTL at L310 is read-only — on a miss after expiry, set() overwrites
// the same key but never removes stale keys. parseDateRange derives cacheKey
// from getTodayStartMs so cacheKey rolls at every UTC 00:00, and additional
// axes (days, startDate, endDate, utcOffset) multiply cardinality.
//
// The same file has three sibling caches that implement MAX + FIFO eviction
// (resolvedSessionKeyByRunId, TRANSCRIPT_SESSION_KEY_CACHE,
// sessionTitleFieldsCache); costUsageCache alone lacked the pattern.
//
// Production trigger: MenuSessionsInjector polls usage.cost every ~45s with
// no params, exercising parseDateRange's default branch on every UTC day
// rollover. The Control UI adds more key variance via explicit startDate /
// endDate / utcTimeZone combinations.
//
// CAL-003 compliance: no mock of internal branches. Growth is driven through
// the __test.loadCostUsageSummaryCached seam (same entry point usage.test.ts
// already exercises) with distinct (startMs, endMs) pairs. Only the external
// loadCostUsageSummary dependency is stubbed.
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
vi.mock("../../infra/session-cost-usage.js", async () => {
const actual = await vi.importActual<typeof import("../../infra/session-cost-usage.js")>(
"../../infra/session-cost-usage.js",
);
return {
...actual,
loadCostUsageSummary: vi.fn(async () => ({
updatedAt: Date.now(),
startDate: "2026-02-01",
endDate: "2026-02-02",
daily: [],
totals: {
totalTokens: 1,
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalCost: 0,
},
})),
};
});
import { __test } from "./usage.js";
describe("costUsageCache bounded growth", () => {
const DAY_MS = 24 * 60 * 60 * 1000;
beforeEach(() => {
__test.costUsageCache.clear();
vi.useRealTimers();
vi.clearAllMocks();
});
it("does not grow without bound when (startMs, endMs) varies across day rollover and range switches", async () => {
const config = {} as OpenClawConfig;
// 600 distinct (startMs, endMs) pairs — larger than the 256 caps used by
// the smallest sibling caches (RUN_LOOKUP_CACHE_LIMIT,
// TRANSCRIPT_SESSION_KEY_CACHE_MAX) and small enough that the test runs
// quickly.
const ITERATIONS = 600;
for (let i = 0; i < ITERATIONS; i++) {
const startMs = Date.UTC(2026, 0, 1) + i * DAY_MS;
const endMs = startMs + (i % 3 === 0 ? DAY_MS : 7 * DAY_MS) - 1;
await __test.loadCostUsageSummaryCached({ startMs, endMs, config });
}
// Primary: map must be bounded. Pre-fix this equals ITERATIONS (600).
expect(__test.costUsageCache.size).toBeLessThan(ITERATIONS);
// Secondary: the most recent entry must still be present. FIFO evicts
// oldest-first, never the newest.
const lastStartMs = Date.UTC(2026, 0, 1) + (ITERATIONS - 1) * DAY_MS;
const lastEndMs = lastStartMs + ((ITERATIONS - 1) % 3 === 0 ? DAY_MS : 7 * DAY_MS) - 1;
const lastCacheKey = `${lastStartMs}-${lastEndMs}`;
expect(__test.costUsageCache.has(lastCacheKey)).toBe(true);
// Tertiary: the oldest entry must have been evicted once the cap was
// exceeded. Pre-fix all 600 entries remain and this fails too.
const firstStartMs = Date.UTC(2026, 0, 1);
const firstEndMs = firstStartMs + DAY_MS - 1;
const firstCacheKey = `${firstStartMs}-${firstEndMs}`;
expect(__test.costUsageCache.has(firstCacheKey)).toBe(false);
});
});

View File

@@ -49,6 +49,7 @@ import {
import type { GatewayRequestHandlers, RespondFn } from "./types.js";
const COST_USAGE_CACHE_TTL_MS = 30_000;
const COST_USAGE_CACHE_MAX = 256;
const DAY_MS = 24 * 60 * 60 * 1000;
type DateRange = { startMs: number; endMs: number };
@@ -64,6 +65,20 @@ type CostUsageCacheEntry = {
const costUsageCache = new Map<string, CostUsageCacheEntry>();
// Store an entry with FIFO eviction when adding a new key would exceed the
// cap. Mirrors the pattern in session-transcript-key.ts and server-session-key.ts
// so the cache stays bounded under sliding-window usage queries (each
// day/range combination produces a distinct key).
function setCostUsageCache(cacheKey: string, entry: CostUsageCacheEntry): void {
if (!costUsageCache.has(cacheKey) && costUsageCache.size >= COST_USAGE_CACHE_MAX) {
const oldest = costUsageCache.keys().next().value;
if (oldest !== undefined) {
costUsageCache.delete(oldest);
}
}
costUsageCache.set(cacheKey, entry);
}
function resolveSessionUsageFileOrRespond(
key: string,
respond: RespondFn,
@@ -325,7 +340,7 @@ async function loadCostUsageSummaryCached(params: {
config: params.config,
})
.then((summary) => {
costUsageCache.set(cacheKey, { summary, updatedAt: Date.now() });
setCostUsageCache(cacheKey, { summary, updatedAt: Date.now() });
return summary;
})
.catch((err) => {
@@ -338,12 +353,12 @@ async function loadCostUsageSummaryCached(params: {
const current = costUsageCache.get(cacheKey);
if (current?.inFlight === inFlight) {
current.inFlight = undefined;
costUsageCache.set(cacheKey, current);
setCostUsageCache(cacheKey, current);
}
});
entry.inFlight = inFlight;
costUsageCache.set(cacheKey, entry);
setCostUsageCache(cacheKey, entry);
if (entry.summary) {
return entry.summary;