mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 07:50:43 +00:00
fix(cost): snapshot estimatedCostUsd instead of accumulating (#69347)
The bug: three persist sites accumulated cost instead of snapshotting it like tokens. This caused cost to be inflated 1x-72x on multi-persist sessions because the same cumulative usage was added repeatedly. Root cause: persistSessionUsageUpdate, updateSessionStoreAfterAgentRun, and the cron isolated-agent run path all used: estimatedCostUsd = existingCost + runCost But runCost was already computed from cumulative run usage, so this added the same cost repeatedly on redundant persists. Fix: snapshot cost directly like tokens already do: estimatedCostUsd = runCost Files affected: - src/auto-reply/reply/session-usage.ts - src/agents/command/session-store.ts - src/cron/isolated-agent/run.ts Tests added: - session-store.test.ts: verify cost is snapshotted, not accumulated - session.test.ts: updated existing test to verify snapshot behavior Fixes #69347
This commit is contained in:
committed by
Peter Steinberger
parent
5bc9d9cc5c
commit
47bb5ddece
@@ -15,6 +15,42 @@ vi.mock("../model-selection.js", () => ({
|
||||
normalizeProviderId: (provider: string) => provider.trim().toLowerCase(),
|
||||
}));
|
||||
|
||||
vi.mock("../../utils/usage-format.js", () => ({
|
||||
estimateUsageCost: (params: { usage?: { input?: number; output?: number }; cost?: { input?: number; output?: number } }) => {
|
||||
if (!params.usage || !params.cost) {
|
||||
return undefined;
|
||||
}
|
||||
const input = params.usage.input ?? 0;
|
||||
const output = params.usage.output ?? 0;
|
||||
const costInput = params.cost.input ?? 0;
|
||||
const costOutput = params.cost.output ?? 0;
|
||||
const total = input * costInput + output * costOutput;
|
||||
if (!Number.isFinite(total)) {
|
||||
return undefined;
|
||||
}
|
||||
return total / 1e6;
|
||||
},
|
||||
resolveModelCostConfig: (params: { provider?: string; model?: string; config?: unknown }) => {
|
||||
// Look up cost from config.models.providers[provider][model].cost
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const providers = params.config?.models?.providers as Record<string, unknown> | undefined;
|
||||
if (!providers) {
|
||||
return undefined;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const provider = providers[params.provider ?? ""] as Record<string, unknown> | undefined;
|
||||
if (!provider) {
|
||||
return undefined;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const model = provider[params.model ?? ""] as { cost?: { input: number; output: number } } | undefined;
|
||||
if (!model?.cost) {
|
||||
return undefined;
|
||||
}
|
||||
return model.cost;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../../config/sessions.js", async () => {
|
||||
const fsSync = await import("node:fs");
|
||||
const fs = await import("node:fs/promises");
|
||||
@@ -383,4 +419,84 @@ describe("updateSessionStoreAfterAgentRun", () => {
|
||||
expect(persisted[sessionKey]?.totalTokensFresh).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it("snapshots cost instead of accumulating (fixes #69347)", async () => {
|
||||
await withTempSessionStore(async ({ storePath }) => {
|
||||
const cfg = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
"gpt-4": {
|
||||
cost: {
|
||||
input: 10, // $10 per million input tokens
|
||||
output: 30, // $30 per million output tokens
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
const sessionKey = "agent:main:explicit:test-cost-snapshot";
|
||||
const sessionId = "test-cost-snapshot-session";
|
||||
|
||||
const sessionStore: Record<string, SessionEntry> = {
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: 1,
|
||||
},
|
||||
};
|
||||
await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2));
|
||||
|
||||
// Simulate a run with 10k input + 5k output tokens
|
||||
// Cost = (10000 * 10 + 5000 * 30) / 1e6 = $0.25
|
||||
const result: EmbeddedPiRunResult = {
|
||||
meta: {
|
||||
durationMs: 500,
|
||||
agentMeta: {
|
||||
sessionId,
|
||||
provider: "openai",
|
||||
model: "gpt-4",
|
||||
usage: {
|
||||
input: 10000,
|
||||
output: 5000,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await updateSessionStoreAfterAgentRun({
|
||||
cfg,
|
||||
sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
sessionStore,
|
||||
defaultProvider: "openai",
|
||||
defaultModel: "gpt-4",
|
||||
result,
|
||||
});
|
||||
|
||||
// First run: cost should be $0.25
|
||||
expect(sessionStore[sessionKey]?.estimatedCostUsd).toBeCloseTo(0.25, 4);
|
||||
|
||||
// Simulate a second persist with the SAME cumulative usage (e.g., from a heartbeat or
|
||||
// redundant persist). Before the fix, this would double the cost.
|
||||
// After the fix, cost should remain the same because it's snapshotted.
|
||||
await updateSessionStoreAfterAgentRun({
|
||||
cfg,
|
||||
sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
sessionStore,
|
||||
defaultProvider: "openai",
|
||||
defaultModel: "gpt-4",
|
||||
result, // Same usage again
|
||||
});
|
||||
|
||||
// After second persist with same usage, cost should STILL be $0.25 (not $0.50)
|
||||
expect(sessionStore[sessionKey]?.estimatedCostUsd).toBeCloseTo(0.25, 4);
|
||||
|
||||
const persisted = loadSessionStore(storePath);
|
||||
expect(persisted[sessionKey]?.estimatedCostUsd).toBeCloseTo(0.25, 4);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -130,9 +130,11 @@ export async function updateSessionStoreAfterAgentRun(params: {
|
||||
}
|
||||
next.cacheRead = usage.cacheRead ?? 0;
|
||||
next.cacheWrite = usage.cacheWrite ?? 0;
|
||||
// Snapshot cost like tokens (runEstimatedCostUsd is already computed from
|
||||
// cumulative run usage, so assign directly instead of accumulating).
|
||||
// Fixes #69347: cost was inflated 1x-72x by accumulating on every persist.
|
||||
if (runEstimatedCostUsd !== undefined) {
|
||||
next.estimatedCostUsd =
|
||||
(resolveNonNegativeNumber(entry.estimatedCostUsd) ?? 0) + runEstimatedCostUsd;
|
||||
next.estimatedCostUsd = runEstimatedCostUsd;
|
||||
}
|
||||
} else if (
|
||||
typeof entry.totalTokens === "number" &&
|
||||
|
||||
@@ -149,10 +149,11 @@ export async function persistSessionUsageUpdate(params: {
|
||||
patch.cacheRead = cacheUsage?.cacheRead ?? 0;
|
||||
patch.cacheWrite = cacheUsage?.cacheWrite ?? 0;
|
||||
}
|
||||
// Snapshot cost like tokens (runEstimatedCostUsd is already computed from
|
||||
// cumulative run usage, so assign directly instead of accumulating).
|
||||
// Fixes #69347: cost was inflated 1x-72x by accumulating on every persist.
|
||||
if (runEstimatedCostUsd !== undefined) {
|
||||
patch.estimatedCostUsd = existingEstimatedCostUsd + runEstimatedCostUsd;
|
||||
} else if (entry.estimatedCostUsd !== undefined) {
|
||||
patch.estimatedCostUsd = entry.estimatedCostUsd;
|
||||
patch.estimatedCostUsd = runEstimatedCostUsd;
|
||||
}
|
||||
// Missing a last-call snapshot (and promptTokens fallback) means
|
||||
// context utilization is stale/unknown.
|
||||
|
||||
@@ -2424,7 +2424,7 @@ describe("persistSessionUsageUpdate", () => {
|
||||
expect(stored[sessionKey].totalTokensFresh).toBe(true);
|
||||
});
|
||||
|
||||
it("accumulates estimatedCostUsd across persisted usage updates", async () => {
|
||||
it("snapshots estimatedCostUsd instead of accumulating (fixes #69347)", async () => {
|
||||
const storePath = await createStorePath("openclaw-usage-cost-");
|
||||
const sessionKey = "main";
|
||||
await seedSessionStore({
|
||||
@@ -2433,33 +2433,36 @@ describe("persistSessionUsageUpdate", () => {
|
||||
entry: {
|
||||
sessionId: "s1",
|
||||
updatedAt: Date.now(),
|
||||
estimatedCostUsd: 0.0015,
|
||||
},
|
||||
});
|
||||
|
||||
const cfg: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
models: [
|
||||
{
|
||||
id: "gpt-5.4",
|
||||
name: "GPT 5.4",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 0.5 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8_192,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// First persist: 2000 input + 500 output + 1000 cacheRead + 200 cacheWrite tokens
|
||||
// Cost = (2000*1.25 + 500*10 + 1000*0.125 + 200*0.5) / 1e6 = $0.007725
|
||||
await persistSessionUsageUpdate({
|
||||
storePath,
|
||||
sessionKey,
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
models: [
|
||||
{
|
||||
id: "gpt-5.4",
|
||||
name: "GPT 5.4",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 0.5 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8_192,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig,
|
||||
cfg,
|
||||
usage: { input: 2_000, output: 500, cacheRead: 1_000, cacheWrite: 200 },
|
||||
lastCallUsage: { input: 800, output: 200, cacheRead: 300, cacheWrite: 50 },
|
||||
providerUsed: "openai",
|
||||
@@ -2467,8 +2470,26 @@ describe("persistSessionUsageUpdate", () => {
|
||||
contextTokensUsed: 200_000,
|
||||
});
|
||||
|
||||
const stored = JSON.parse(await fs.readFile(storePath, "utf-8"));
|
||||
expect(stored[sessionKey].estimatedCostUsd).toBeCloseTo(0.009225, 8);
|
||||
const stored1 = JSON.parse(await fs.readFile(storePath, "utf-8"));
|
||||
expect(stored1[sessionKey].estimatedCostUsd).toBeCloseTo(0.007725, 8);
|
||||
|
||||
// Second persist with SAME cumulative usage (e.g., heartbeat or redundant persist)
|
||||
// Before fix: cost would accumulate to $0.0155 (2x)
|
||||
// After fix: cost stays $0.00775 (snapshotted)
|
||||
await persistSessionUsageUpdate({
|
||||
storePath,
|
||||
sessionKey,
|
||||
cfg,
|
||||
usage: { input: 2_000, output: 500, cacheRead: 1_000, cacheWrite: 200 },
|
||||
lastCallUsage: { input: 800, output: 200, cacheRead: 300, cacheWrite: 50 },
|
||||
providerUsed: "openai",
|
||||
modelUsed: "gpt-5.4",
|
||||
contextTokensUsed: 200_000,
|
||||
});
|
||||
|
||||
const stored2 = JSON.parse(await fs.readFile(storePath, "utf-8"));
|
||||
// Cost should still be $0.007725, NOT $0.01545
|
||||
expect(stored2[sessionKey].estimatedCostUsd).toBeCloseTo(0.007725, 8);
|
||||
});
|
||||
|
||||
it("persists zero estimatedCostUsd for free priced models", async () => {
|
||||
|
||||
@@ -596,10 +596,11 @@ async function finalizeCronRun(params: {
|
||||
}
|
||||
prepared.cronSession.sessionEntry.cacheRead = usage.cacheRead ?? 0;
|
||||
prepared.cronSession.sessionEntry.cacheWrite = usage.cacheWrite ?? 0;
|
||||
// Snapshot cost like tokens (runEstimatedCostUsd is already computed from
|
||||
// cumulative run usage, so assign directly instead of accumulating).
|
||||
// Fixes #69347: cost was inflated 1x-72x by accumulating on every persist.
|
||||
if (runEstimatedCostUsd !== undefined) {
|
||||
prepared.cronSession.sessionEntry.estimatedCostUsd =
|
||||
(resolveNonNegativeNumber(prepared.cronSession.sessionEntry.estimatedCostUsd) ?? 0) +
|
||||
runEstimatedCostUsd;
|
||||
prepared.cronSession.sessionEntry.estimatedCostUsd = runEstimatedCostUsd;
|
||||
}
|
||||
telemetry = {
|
||||
model: modelUsed,
|
||||
|
||||
Reference in New Issue
Block a user