diff --git a/extensions/ollama/src/stream-runtime.test.ts b/extensions/ollama/src/stream-runtime.test.ts index 81a7a5f20337..c33b4ad491b3 100644 --- a/extensions/ollama/src/stream-runtime.test.ts +++ b/extensions/ollama/src/stream-runtime.test.ts @@ -1202,6 +1202,19 @@ describe("buildAssistantMessage", () => { total: 0, }); }); + + it("records unavailable cache telemetry when Ollama omits the cache split", () => { + const response = { + model: "qwen3:32b", + created_at: "2026-01-01T00:00:00Z", + message: { role: "assistant" as const, content: "ok" }, + done: true, + prompt_eval_count: 10, + eval_count: 2, + }; + const result = buildAssistantMessage(response, modelInfo); + expect(result.usage.cacheTelemetry).toEqual({ state: "unavailable" }); + }); }); // Helper: build a ReadableStreamDefaultReader from NDJSON lines diff --git a/extensions/ollama/src/stream.ts b/extensions/ollama/src/stream.ts index 5e855fd4acfe..58236b6de570 100644 --- a/extensions/ollama/src/stream.ts +++ b/extensions/ollama/src/stream.ts @@ -561,17 +561,24 @@ function buildUsageWithNoCost(params: { output?: number; cacheRead?: number; cacheWrite?: number; + cacheTelemetry?: Usage["cacheTelemetry"]; totalTokens?: number; }): Usage { const input = params.input ?? 0; const output = params.output ?? 0; const cacheRead = params.cacheRead ?? 0; const cacheWrite = params.cacheWrite ?? 0; + const cacheTelemetry = + params.cacheTelemetry ?? + (params.cacheRead !== undefined && params.cacheWrite !== undefined + ? { state: "available" as const } + : { state: "unavailable" as const }); return { input, output, cacheRead, cacheWrite, + cacheTelemetry, totalTokens: params.totalTokens ?? input + output, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }; diff --git a/extensions/qa-lab/src/runtime-parity-cache-diagnostics.test.ts b/extensions/qa-lab/src/runtime-parity-cache-diagnostics.test.ts index c157fa9c09c8..c3092eed9511 100644 --- a/extensions/qa-lab/src/runtime-parity-cache-diagnostics.test.ts +++ b/extensions/qa-lab/src/runtime-parity-cache-diagnostics.test.ts @@ -98,6 +98,114 @@ describe("runtime parity prompt-cache diagnostics", () => { }); }); + it("treats Ollama adapter cache zeros as unavailable telemetry", async () => { + const tempRoot = await seedRuntimeParityCacheTranscript([ + { role: "user", content: "Inspect cache telemetry." }, + { + role: "assistant", + content: "done", + api: "ollama", + provider: "ollama", + model: "qwen3.5:9b", + usage: { + input: 22, + output: 7, + total: 29, + cacheRead: 0, + cacheWrite: 0, + cacheTelemetry: { state: "unavailable" }, + }, + }, + ]); + + const cell = await captureRuntimeParityCell({ + runtime: "openclaw", + gateway: { tempRoot }, + scenarioResult: { status: "pass" }, + wallClockMs: 10, + }); + + expect(cell.usage).toEqual({ + inputTokens: 22, + outputTokens: 7, + totalTokens: 29, + }); + expect(cell.cacheDiagnostics).toEqual({ + assistantTurns: 1, + cacheTelemetryTurns: 0, + cacheHitTurns: 0, + cacheWriteTurns: 0, + cacheMisses: [], + cacheMissInputTokens: 0, + unmeasuredPostWarmTurns: [], + }); + }); + + it("preserves unavailable cache telemetry from pre-marker Ollama transcripts", async () => { + const tempRoot = await seedRuntimeParityCacheTranscript([ + { role: "user", content: "Inspect cache telemetry." }, + { + role: "assistant", + content: "done", + api: "ollama", + provider: "ollama", + model: "qwen3.5:9b", + usage: { input: 22, output: 7, total: 29, cacheRead: 0, cacheWrite: 0 }, + }, + ]); + + const cell = await captureRuntimeParityCell({ + runtime: "openclaw", + gateway: { tempRoot }, + scenarioResult: { status: "pass" }, + wallClockMs: 10, + }); + + expect(cell.usage).toEqual({ + inputTokens: 22, + outputTokens: 7, + totalTokens: 29, + }); + expect(cell.cacheDiagnostics?.cacheTelemetryTurns).toBe(0); + }); + + it("preserves an explicitly measured zero-cache result", async () => { + const tempRoot = await seedRuntimeParityCacheTranscript([ + { role: "user", content: "Inspect cache telemetry." }, + { + role: "assistant", + content: "done", + api: "ollama", + provider: "ollama", + model: "future-ollama", + usage: { + input: 22, + output: 7, + total: 29, + cacheRead: 0, + cacheWrite: 0, + cacheTelemetry: { state: "available" }, + }, + }, + ]); + + const cell = await captureRuntimeParityCell({ + runtime: "openclaw", + gateway: { tempRoot }, + scenarioResult: { status: "pass" }, + wallClockMs: 10, + }); + + expect(cell.usage).toEqual({ + inputTokens: 22, + outputTokens: 7, + totalTokens: 29, + cacheRead: 0, + cacheWrite: 0, + }); + expect(cell.cacheDiagnostics?.cacheTelemetryTurns).toBe(1); + }); + it("identifies the first complete cache miss after a cache-warming write", () => { const diagnostics = buildRuntimeParityCacheDiagnostics([ usage(3, { cacheRead: 0, cacheWrite: 24_407 }), diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index 2f0dfe4a2e71..841015d0ba28 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -241,9 +241,14 @@ function readUsageTotals(raw: unknown): RuntimeParityUsage { readFiniteNumber(usage.outputTokens) ?? readFiniteNumber(usage.output_tokens) ?? 0; - const cacheRead = readFiniteNumber(usage.cacheRead) ?? readFiniteNumber(usage.cache_read_tokens); - const cacheWrite = - readFiniteNumber(usage.cacheWrite) ?? readFiniteNumber(usage.cache_write_tokens); + const cacheTelemetryUnavailable = + isMessageRecord(usage.cacheTelemetry) && usage.cacheTelemetry.state === "unavailable"; + const cacheRead = cacheTelemetryUnavailable + ? undefined + : (readFiniteNumber(usage.cacheRead) ?? readFiniteNumber(usage.cache_read_tokens)); + const cacheWrite = cacheTelemetryUnavailable + ? undefined + : (readFiniteNumber(usage.cacheWrite) ?? readFiniteNumber(usage.cache_write_tokens)); const componentTotal = inputTokens + outputTokens + (cacheRead ?? 0) + (cacheWrite ?? 0); const totalTokens = readFiniteNumber(usage.total) ?? @@ -259,6 +264,26 @@ function readUsageTotals(raw: unknown): RuntimeParityUsage { }; } +function readAssistantUsage(message: Record): RuntimeParityUsage { + const usage = readUsageTotals(message.usage ?? null); + if (!isMessageRecord(message.usage) || isMessageRecord(message.usage.cacheTelemetry)) { + return usage; + } + const provider = readNonEmptyString(message.provider)?.toLowerCase(); + const api = readNonEmptyString(message.api)?.toLowerCase(); + if ( + (provider === "ollama" || api === "ollama") && + usage.cacheRead === 0 && + usage.cacheWrite === 0 + ) { + // Transcripts written before cacheTelemetry was added contain Ollama's + // required placeholder zeros. Explicit current provenance always wins. + delete usage.cacheRead; + delete usage.cacheWrite; + } + return usage; +} + function addUsage(target: RuntimeParityUsage, next: RuntimeParityUsage) { target.inputTokens += next.inputTokens; target.outputTokens += next.outputTokens; @@ -1002,7 +1027,7 @@ function aggregateUsage(records: RuntimeParityTranscriptRecord[]): RuntimeParity if (record.role !== "assistant") { continue; } - const usage = readUsageTotals(record.message.usage ?? null); + const usage = readAssistantUsage(record.message); addUsage(totals, usage); } return totals; @@ -1474,7 +1499,7 @@ export async function captureRuntimeParityCell( cacheDiagnostics: buildRuntimeParityCacheDiagnostics( transcriptRecords .filter((record) => record.role === "assistant") - .map((record) => readUsageTotals(record.message.usage ?? null)), + .map((record) => readAssistantUsage(record.message)), ), wallClockMs: params.wallClockMs, ...(params.bootstrapWallClockMs === undefined diff --git a/packages/llm-core/src/types.ts b/packages/llm-core/src/types.ts index 1e12b4a8c1d2..d81f897b6236 100644 --- a/packages/llm-core/src/types.ts +++ b/packages/llm-core/src/types.ts @@ -273,6 +273,8 @@ export interface Usage { output: number; cacheRead: number; cacheWrite: number; + /** Whether the provider reported a cache-read/write token split. */ + cacheTelemetry?: { state: "available" | "unavailable" }; /** Subset of `cacheWrite` written with 1-hour retention when reported. */ cacheWrite1h?: number; /** Exact context snapshot for the final provider iteration. */