From 5de0366d9567ffd45c4a104adb3e682a0dd2a35e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Jul 2026 13:26:04 -0400 Subject: [PATCH] fix(codex): stabilize prompts and expose real cache regressions (#115238) * fix(codex): stabilize prompt caching and measure uncached work * test(codex): refresh stable dynamic-tool prompt snapshots * fix(qa): account for cache rewrites and incomplete telemetry * fix(qa): reject inconsistent measured cache totals * fix(qa): preserve live profile eligibility and parity topology * fix(qa): extract acyclic runtime parity usage contract --- .../src/app-server/dynamic-tools.test.ts | 51 +- .../codex/src/app-server/dynamic-tools.ts | 8 +- .../src/agentic-parity-report.cache.test.ts | 81 +++ .../qa-lab/src/agentic-parity-report.ts | 6 + .../src/agentic-parity-runtime-markdown.ts | 26 + .../agentic-parity-runtime-report-contract.ts | 3 + extensions/qa-lab/src/profile-planning.ts | 15 +- extensions/qa-lab/src/run-config.test.ts | 38 ++ .../runtime-parity-cache-diagnostics.test.ts | 251 ++++++++ .../src/runtime-parity-cache-diagnostics.ts | 69 +++ extensions/qa-lab/src/runtime-parity-usage.ts | 7 + extensions/qa-lab/src/runtime-parity.ts | 21 +- .../src/token-efficiency-report.test.ts | 440 +++++++++++++++ .../qa-lab/src/token-efficiency-report.ts | 239 ++++++-- .../codex-dynamic-tools.discord-group.json | 534 +++++++++--------- .../codex-dynamic-tools.heartbeat-turn.json | 534 +++++++++--------- .../codex-dynamic-tools.telegram-direct.json | 534 +++++++++--------- .../discord-group-codex-message-tool.md | 24 +- .../telegram-direct-codex-message-tool.md | 24 +- .../telegram-heartbeat-codex-tool.md | 24 +- 20 files changed, 2048 insertions(+), 881 deletions(-) create mode 100644 extensions/qa-lab/src/runtime-parity-cache-diagnostics.test.ts create mode 100644 extensions/qa-lab/src/runtime-parity-cache-diagnostics.ts create mode 100644 extensions/qa-lab/src/runtime-parity-usage.ts diff --git a/extensions/codex/src/app-server/dynamic-tools.test.ts b/extensions/codex/src/app-server/dynamic-tools.test.ts index 0e1bc128ad65..42912b75ea51 100644 --- a/extensions/codex/src/app-server/dynamic-tools.test.ts +++ b/extensions/codex/src/app-server/dynamic-tools.test.ts @@ -282,6 +282,55 @@ describe("createCodexDynamicToolBridge", () => { expectNoNamespace(specs.find((tool) => tool.name === "message")); }); + it("keeps model-visible tools stable when plugin discovery order changes", () => { + const tools = [ + createTool({ name: "web_search" }), + createTool({ name: "sessions_yield" }), + createTool({ name: "message" }), + createTool({ name: "computer", catalogMode: "direct-only" }), + createTool({ name: "agents_list" }), + createTool({ name: "browser", catalogMode: "direct-only" }), + createTool({ name: "openclaw" }), + ]; + const createBridge = (orderedTools: AnyAgentTool[]) => + createCodexDynamicToolBridge({ + tools: orderedTools, + registeredTools: orderedTools, + signal: new AbortController().signal, + directToolNames: ["openclaw"], + }); + const forward = createBridge(tools); + const reversed = createBridge(tools.toReversed()); + + expect(forward.availableSpecs).toEqual(reversed.availableSpecs); + expect(forward.specs).toEqual(reversed.specs); + expect(specNames(forward.specs)).toEqual([ + "agents_list", + "openclaw", + "sessions_yield", + "message", + "web_search", + "browser", + "computer", + ]); + expect(forward.specs.filter((spec) => spec.type === "namespace")).toEqual([ + expect.objectContaining({ + name: CODEX_OPENCLAW_DYNAMIC_TOOL_NAMESPACE, + tools: [ + expect.objectContaining({ name: "message", deferLoading: true }), + expect.objectContaining({ name: "web_search", deferLoading: true }), + ], + }), + expect.objectContaining({ + name: CODEX_OPENCLAW_DIRECT_DYNAMIC_TOOL_NAMESPACE, + tools: [ + expect.objectContaining({ name: "browser" }), + expect.objectContaining({ name: "computer" }), + ], + }), + ]); + }); + it("can register a durable tool schema while denying execution for the current turn", async () => { const heartbeatExecute = vi.fn(async () => textToolResult("heartbeat recorded")); const onAgentToolResult = vi.fn(); @@ -297,7 +346,7 @@ describe("createCodexDynamicToolBridge", () => { }); expect(specNames(bridge.availableSpecs)).toEqual(["message"]); - expect(specNames(bridge.specs)).toEqual(["message", HEARTBEAT_RESPONSE_TOOL_NAME]); + expect(specNames(bridge.specs)).toEqual([HEARTBEAT_RESPONSE_TOOL_NAME, "message"]); const result = await bridge.handleToolCall( { diff --git a/extensions/codex/src/app-server/dynamic-tools.ts b/extensions/codex/src/app-server/dynamic-tools.ts index 8af6bec716c3..e57dbf81cd58 100644 --- a/extensions/codex/src/app-server/dynamic-tools.ts +++ b/extensions/codex/src/app-server/dynamic-tools.ts @@ -963,7 +963,13 @@ function createCodexDynamicToolSpecs(params: { const specs: CodexDynamicToolSpec[] = []; const namespaceTools: CodexDynamicToolFunctionSpec[] = []; const directOnlyNamespaceTools: CodexDynamicToolFunctionSpec[] = []; - for (const entry of params.entries) { + // Codex reuses its incremental websocket request only when the complete + // searchable surface is unchanged. Direct mode retains its compatibility order. + const entries = + params.loading === "direct" + ? params.entries + : params.entries.toSorted((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { const functionSpec = createCodexDynamicToolFunctionSpec({ entry }); if (entry.name === "openclaw" && params.directToolNames.has(entry.name)) { // OpenClaw is ring-zero and its whole turn surface. Keep its canonical diff --git a/extensions/qa-lab/src/agentic-parity-report.cache.test.ts b/extensions/qa-lab/src/agentic-parity-report.cache.test.ts index ed9b9c179bcd..428c34983263 100644 --- a/extensions/qa-lab/src/agentic-parity-report.cache.test.ts +++ b/extensions/qa-lab/src/agentic-parity-report.cache.test.ts @@ -4,6 +4,7 @@ import { buildQaRuntimeParityReport, renderQaRuntimeParityMarkdownReport, } from "./agentic-parity-report.js"; +import { buildRuntimeParityCacheDiagnostics } from "./runtime-parity-cache-diagnostics.js"; function makeMeasuredRuntimeParitySummary() { const summary = makeRuntimeParitySummary(); @@ -18,6 +19,86 @@ function makeMeasuredRuntimeParitySummary() { } describe("qa runtime parity prompt-cache reporting", () => { + it("reports the exact turn of a measured cache miss without inventing missing telemetry", () => { + const summary = makeMeasuredRuntimeParitySummary(); + const scenario = summary.scenarios[0]; + if (!scenario?.runtimeParity) { + throw new Error("runtime parity fixture missing"); + } + scenario.runtimeParity.cells.codex.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([ + { + inputTokens: 3, + outputTokens: 11, + totalTokens: 24_421, + cacheRead: 0, + cacheWrite: 24_407, + }, + { + inputTokens: 24_448, + outputTokens: 11, + totalTokens: 24_459, + cacheRead: 0, + cacheWrite: 0, + }, + ]); + + const report = buildQaRuntimeParityReport({ summary }); + + expect(report.scenarios[0]?.codexCacheDiagnostics).toMatchObject({ + cacheMisses: [{ turn: 2, inputTokens: 24_448, cacheRead: 0, cacheWrite: 0 }], + cacheMissInputTokens: 24_448, + }); + expect(report.scenarios[0]?.openclawCacheDiagnostics).toBeUndefined(); + expect(renderQaRuntimeParityMarkdownReport(report)).toContain( + "post-warm cache misses: openclaw N/A; codex turn 2 (24448 uncached input)", + ); + }); + + it("reports unknown post-warm turns without hiding measured cache misses", () => { + const summary = makeMeasuredRuntimeParitySummary(); + const scenario = summary.scenarios[0]; + if (!scenario?.runtimeParity) { + throw new Error("runtime parity fixture missing"); + } + scenario.runtimeParity.cells.codex.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([ + { inputTokens: 3, outputTokens: 11, totalTokens: 1_014, cacheRead: 0, cacheWrite: 1_000 }, + { inputTokens: 1_050, outputTokens: 11, totalTokens: 1_061, cacheRead: 0, cacheWrite: 0 }, + { inputTokens: 0, outputTokens: 11, totalTokens: 11 }, + ]); + + const report = buildQaRuntimeParityReport({ summary }); + + expect(report.scenarios[0]?.codexCacheDiagnostics).toMatchObject({ + cacheMisses: [{ turn: 2, inputTokens: 1_050, cacheRead: 0, cacheWrite: 0 }], + unmeasuredPostWarmTurns: [3], + }); + expect(renderQaRuntimeParityMarkdownReport(report)).toContain( + "post-warm cache misses: openclaw N/A; codex turn 2 (1050 uncached input); unmeasured turns 3", + ); + }); + + it("preserves unknown warm turns when no turn has complete cache telemetry", () => { + const summary = makeMeasuredRuntimeParitySummary(); + const scenario = summary.scenarios[0]; + if (!scenario?.runtimeParity) { + throw new Error("runtime parity fixture missing"); + } + scenario.runtimeParity.cells.codex.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([ + { inputTokens: 3, outputTokens: 11, totalTokens: 1_014, cacheWrite: 1_000 }, + { inputTokens: 100, outputTokens: 11, totalTokens: 111 }, + ]); + + const report = buildQaRuntimeParityReport({ summary }); + + expect(report.scenarios[0]?.codexCacheDiagnostics).toMatchObject({ + cacheTelemetryTurns: 0, + unmeasuredPostWarmTurns: [2], + }); + expect(renderQaRuntimeParityMarkdownReport(report)).toContain( + "post-warm cache misses: openclaw N/A; codex N/A (unmeasured turns 2)", + ); + }); + it("reports cached, uncached, and cache-write input without counting output as cacheable", () => { const summary = makeMeasuredRuntimeParitySummary(); const scenario = summary.scenarios[0]; diff --git a/extensions/qa-lab/src/agentic-parity-report.ts b/extensions/qa-lab/src/agentic-parity-report.ts index 325359f1e7fa..a05e6d324275 100644 --- a/extensions/qa-lab/src/agentic-parity-report.ts +++ b/extensions/qa-lab/src/agentic-parity-report.ts @@ -681,6 +681,12 @@ export function buildQaRuntimeParityReport(params: { runtimeParityUsage.expectation === "not-applicable" ? null : summarizeRuntimeParityCacheUsage(codexCell.usage), + ...(openclawCell.cacheDiagnostics === undefined + ? {} + : { openclawCacheDiagnostics: openclawCell.cacheDiagnostics }), + ...(codexCell.cacheDiagnostics === undefined + ? {} + : { codexCacheDiagnostics: codexCell.cacheDiagnostics }), openclawToolCalls: openclawCell.toolCalls.length, codexToolCalls: codexCell.toolCalls.length, openclawWallClockMs: openclawCell.wallClockMs, diff --git a/extensions/qa-lab/src/agentic-parity-runtime-markdown.ts b/extensions/qa-lab/src/agentic-parity-runtime-markdown.ts index 48d7f4e2648d..a0630c953e19 100644 --- a/extensions/qa-lab/src/agentic-parity-runtime-markdown.ts +++ b/extensions/qa-lab/src/agentic-parity-runtime-markdown.ts @@ -3,8 +3,31 @@ import { formatRuntimeCacheHitPercent, } from "./agentic-parity-cache-usage.js"; import type { QaRuntimeParityReport } from "./agentic-parity-runtime-report-contract.js"; +import type { RuntimeParityCacheDiagnostics } from "./runtime-parity-cache-diagnostics.js"; import { formatRuntimeSpeedComparison, formatRuntimeWallClockMs } from "./runtime-parity-timing.js"; +function formatRuntimeCacheMisses(diagnostics: RuntimeParityCacheDiagnostics | undefined): string { + if (!diagnostics) { + return "N/A"; + } + if (diagnostics.cacheTelemetryTurns === 0) { + return diagnostics.unmeasuredPostWarmTurns.length > 0 + ? `N/A (unmeasured turns ${diagnostics.unmeasuredPostWarmTurns.join(", ")})` + : "N/A"; + } + const measuredMisses = + diagnostics.cacheMisses.length === 0 + ? "none" + : diagnostics.cacheMisses + .map((miss) => `turn ${miss.turn} (${miss.inputTokens} uncached input)`) + .join(", "); + if (diagnostics.unmeasuredPostWarmTurns.length === 0) { + return measuredMisses; + } + const unknownTurns = `unmeasured turns ${diagnostics.unmeasuredPostWarmTurns.join(", ")}`; + return measuredMisses === "none" ? `N/A (${unknownTurns})` : `${measuredMisses}; ${unknownTurns}`; +} + export function renderQaRuntimeParityMarkdownReport(report: QaRuntimeParityReport): string { const lines = [ `# OpenClaw Runtime Parity Report — ${report.runtimePair[0]} vs ${report.runtimePair[1]}`, @@ -91,6 +114,9 @@ export function renderQaRuntimeParityMarkdownReport(report: QaRuntimeParityRepor lines.push( `- prompt cache: openclaw ${formatRuntimeCacheHitPercent(scenario.openclawUsage?.cacheHitPercent)} (${formatRuntimeCacheCount(scenario.openclawUsage?.cachedInputTokens)} cached, ${formatRuntimeCacheCount(scenario.openclawUsage?.uncachedInputTokens)} uncached input); codex ${formatRuntimeCacheHitPercent(scenario.codexUsage?.cacheHitPercent)} (${formatRuntimeCacheCount(scenario.codexUsage?.cachedInputTokens)} cached, ${formatRuntimeCacheCount(scenario.codexUsage?.uncachedInputTokens)} uncached input)`, ); + lines.push( + `- post-warm cache misses: openclaw ${formatRuntimeCacheMisses(scenario.openclawCacheDiagnostics)}; codex ${formatRuntimeCacheMisses(scenario.codexCacheDiagnostics)}`, + ); if (scenario.runtimeParityUsage.expectation === "not-applicable") { lines.push(`- assistant-message usage: N/A (${scenario.runtimeParityUsage.reason})`); } diff --git a/extensions/qa-lab/src/agentic-parity-runtime-report-contract.ts b/extensions/qa-lab/src/agentic-parity-runtime-report-contract.ts index 6471cc3558f6..7e7b9b0227dd 100644 --- a/extensions/qa-lab/src/agentic-parity-runtime-report-contract.ts +++ b/extensions/qa-lab/src/agentic-parity-runtime-report-contract.ts @@ -1,4 +1,5 @@ import type { QaRuntimeParityCacheUsage } from "./agentic-parity-cache-usage.js"; +import type { RuntimeParityCacheDiagnostics } from "./runtime-parity-cache-diagnostics.js"; import type { QaRuntimeTiming } from "./runtime-parity-timing.js"; import type { RuntimeId, RuntimeParityDrift, RuntimeParityUsagePolicy } from "./runtime-parity.js"; @@ -14,6 +15,8 @@ export type QaRuntimeParityScenarioReport = { codexTokens: number; openclawUsage: QaRuntimeParityCacheUsage | null; codexUsage: QaRuntimeParityCacheUsage | null; + openclawCacheDiagnostics?: RuntimeParityCacheDiagnostics; + codexCacheDiagnostics?: RuntimeParityCacheDiagnostics; openclawToolCalls: number; codexToolCalls: number; openclawWallClockMs: number | null; diff --git a/extensions/qa-lab/src/profile-planning.ts b/extensions/qa-lab/src/profile-planning.ts index c417f3b04d83..18e5284a765c 100644 --- a/extensions/qa-lab/src/profile-planning.ts +++ b/extensions/qa-lab/src/profile-planning.ts @@ -131,10 +131,23 @@ export function resolveQaRunProfileExecutionSelection(params: { if (scenario.execution.channel === "qa-channel" && params.channelDriver !== "qa-channel") { reasons.push("channelDriver=qa-channel"); } + // Unpinned live profiles must resolve a declared real transport before checking a + // portable scenario; qa-channel is the built-in harness, not a live adapter. + const declaredLiveChannel = + params.channelDriver === "live" && scenario.execution.kind === "flow" + ? (scenario.execution.channels?.find( + (channel) => + channel !== "qa-channel" && + (!params.supportsChannel || params.supportsChannel(channel)), + ) ?? scenario.execution.channels?.find((channel) => channel !== "qa-channel")) + : undefined; const effectiveChannel = params.channelDriver === "qa-channel" ? "qa-channel" - : (params.channel ?? scenario.execution.channel ?? params.defaultChannel); + : (params.channel ?? + scenario.execution.channel ?? + declaredLiveChannel ?? + params.defaultChannel); reasons.push( ...describeQaProviderLaneMismatches({ scenario, diff --git a/extensions/qa-lab/src/run-config.test.ts b/extensions/qa-lab/src/run-config.test.ts index 43e37234d128..c37a9a092294 100644 --- a/extensions/qa-lab/src/run-config.test.ts +++ b/extensions/qa-lab/src/run-config.test.ts @@ -314,6 +314,44 @@ describe("qa run config", () => { ); }); + it("keeps portable thread scenarios in unpinned live profiles", () => { + const catalog = readQaScenarioPack(); + const scenarioIds = new Set(["thread-follow-up", "thread-isolation"]); + const selected = catalog.scenarios.filter((scenario) => scenarioIds.has(scenario.id)); + + const execution = resolveQaRunProfileExecutionSelection({ + scenarios: selected, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + channelDriver: "live", + }); + + expect(execution.selectedScenarios.map((scenario) => scenario.id)).toEqual([ + "thread-follow-up", + "thread-isolation", + ]); + expect(execution.excludedScenarios).toEqual([]); + }); + + it("selects a supported declared transport for portable live scenarios", () => { + const catalog = readQaScenarioPack(); + const scenario = catalog.scenarios.find((entry) => entry.id === "thread-follow-up"); + if (!scenario) { + throw new Error("thread-follow-up scenario is missing from the QA catalog"); + } + + const execution = resolveQaRunProfileExecutionSelection({ + scenarios: [scenario], + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + channelDriver: "live", + supportsChannel: (channel) => channel === "matrix", + }); + + expect(execution.selectedScenarios).toEqual([scenario]); + expect(execution.excludedScenarios).toEqual([]); + }); + it("excludes live-only and unsupported thread scenarios from Crabline plans", () => { const catalog = readQaScenarioPack(); const scenarioIds = new Set([ diff --git a/extensions/qa-lab/src/runtime-parity-cache-diagnostics.test.ts b/extensions/qa-lab/src/runtime-parity-cache-diagnostics.test.ts new file mode 100644 index 000000000000..c157fa9c09c8 --- /dev/null +++ b/extensions/qa-lab/src/runtime-parity-cache-diagnostics.test.ts @@ -0,0 +1,251 @@ +import path from "node:path"; +import { resolveStorePath, upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { formatSqliteSessionFileMarker } from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildRuntimeParityCacheDiagnostics } from "./runtime-parity-cache-diagnostics.js"; +import { captureRuntimeParityCell, type RuntimeParityUsage } from "./runtime-parity.js"; +import { createTempDirHarness } from "./temp-dir.test-helper.js"; + +const tempDirs = createTempDirHarness(); + +afterEach(async () => { + await tempDirs.cleanup(); +}); + +async function seedRuntimeParityCacheTranscript(messages: Array>) { + const tempRoot = await tempDirs.makeTempDir("openclaw-qa-runtime-parity-cache-"); + const agentId = "qa"; + const sessionId = "runtime-parity-cache-miss"; + const sessionKey = "agent:qa:runtime-parity-cache-miss"; + const env = { ...process.env, OPENCLAW_STATE_DIR: path.join(tempRoot, "state") }; + const storePath = resolveStorePath(undefined, { agentId, env }); + await upsertSessionEntry({ + agentId, + env, + sessionKey, + storePath, + entry: { + sessionId, + sessionFile: formatSqliteSessionFileMarker({ agentId, sessionId, storePath }), + updatedAt: 100, + }, + }); + for (const [index, message] of messages.entries()) { + await appendSessionTranscriptMessageByIdentity({ + agentId, + env, + sessionId, + sessionKey, + storePath, + now: index + 1, + message: message as never, + }); + } + return tempRoot; +} + +function usage( + inputTokens: number, + cache?: { cacheRead: number; cacheWrite: number }, +): RuntimeParityUsage { + return { + inputTokens, + outputTokens: 11, + totalTokens: inputTokens + 11 + (cache?.cacheRead ?? 0) + (cache?.cacheWrite ?? 0), + ...cache, + }; +} + +describe("runtime parity prompt-cache diagnostics", () => { + it("captures post-warm cache losses from the canonical SQLite assistant transcript", async () => { + const tempRoot = await seedRuntimeParityCacheTranscript([ + { role: "user", content: "Warm the native conversation." }, + { + role: "assistant", + content: "warm", + usage: { input: 3, output: 11, total: 24_421, cacheRead: 0, cacheWrite: 24_407 }, + }, + { role: "user", content: "Continue that conversation." }, + { + role: "assistant", + content: "continued", + usage: { input: 24_448, output: 11, total: 24_459, cacheRead: 0, cacheWrite: 0 }, + }, + ]); + + const cell = await captureRuntimeParityCell({ + runtime: "codex", + gateway: { tempRoot }, + scenarioResult: { status: "pass" }, + wallClockMs: 10, + }); + + expect(cell.cacheDiagnostics).toEqual({ + assistantTurns: 2, + cacheTelemetryTurns: 2, + cacheHitTurns: 0, + cacheWriteTurns: 1, + cacheMisses: [{ turn: 2, inputTokens: 24_448, cacheRead: 0, cacheWrite: 0 }], + cacheMissInputTokens: 24_448, + unmeasuredPostWarmTurns: [], + }); + expect(cell.usage).toMatchObject({ + inputTokens: 24_451, + outputTokens: 22, + cacheRead: 0, + cacheWrite: 24_407, + }); + }); + + it("identifies the first complete cache miss after a cache-warming write", () => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + usage(3, { cacheRead: 0, cacheWrite: 24_407 }), + usage(24_448, { cacheRead: 0, cacheWrite: 0 }), + usage(41, { cacheRead: 24_445, cacheWrite: 0 }), + ]); + + expect(diagnostics).toEqual({ + assistantTurns: 3, + cacheTelemetryTurns: 3, + cacheHitTurns: 1, + cacheWriteTurns: 1, + cacheMisses: [{ turn: 2, inputTokens: 24_448, cacheRead: 0, cacheWrite: 0 }], + cacheMissInputTokens: 24_448, + unmeasuredPostWarmTurns: [], + }); + }); + + it("identifies complete cache losses later in an already cached conversation", () => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + usage(5, { cacheRead: 0, cacheWrite: 1_000 }), + usage(8, { cacheRead: 1_000, cacheWrite: 0 }), + usage(1_050, { cacheRead: 0, cacheWrite: 0 }), + usage(12, { cacheRead: 1_050, cacheWrite: 0 }), + usage(1_100, { cacheRead: 0, cacheWrite: 0 }), + ]); + + expect(diagnostics.cacheMisses).toEqual([ + { turn: 3, inputTokens: 1_050, cacheRead: 0, cacheWrite: 0 }, + { turn: 5, inputTokens: 1_100, cacheRead: 0, cacheWrite: 0 }, + ]); + expect(diagnostics.cacheMissInputTokens).toBe(2_150); + }); + + it("counts a zero-read post-warm cache rewrite as a complete miss", () => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + usage(3, { cacheRead: 0, cacheWrite: 1_000 }), + usage(200, { cacheRead: 800, cacheWrite: 0 }), + usage(20, { cacheRead: 0, cacheWrite: 1_100 }), + ]); + + expect(diagnostics.cacheMisses).toEqual([ + { turn: 3, inputTokens: 1_120, cacheRead: 0, cacheWrite: 1_100 }, + ]); + expect(diagnostics.cacheMissInputTokens).toBe(1_120); + expect(diagnostics.cacheHitTurns).toBe(1); + expect(diagnostics.cacheWriteTurns).toBe(2); + }); + + it("counts write-only processed input when a post-warm miss rewrites the cache", () => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + usage(3, { cacheRead: 0, cacheWrite: 1_000 }), + usage(0, { cacheRead: 0, cacheWrite: 1_100 }), + ]); + + expect(diagnostics.cacheMisses).toEqual([ + { turn: 2, inputTokens: 1_100, cacheRead: 0, cacheWrite: 1_100 }, + ]); + expect(diagnostics.cacheMissInputTokens).toBe(1_100); + }); + + it("does not mistake unavailable cache telemetry for a measured cache miss", () => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + usage(3, { cacheRead: 0, cacheWrite: 1_000 }), + usage(1_050), + usage(8, { cacheRead: 1_000, cacheWrite: 0 }), + ]); + + expect(diagnostics.assistantTurns).toBe(3); + expect(diagnostics.cacheTelemetryTurns).toBe(2); + expect(diagnostics.cacheMisses).toEqual([]); + expect(diagnostics.unmeasuredPostWarmTurns).toEqual([2]); + }); + + it.each([ + { + name: "a measured cache hit without cache-write telemetry", + partialUsage: { cacheRead: 1_000 }, + cacheHitTurns: 1, + cacheWriteTurns: 0, + }, + { + name: "a measured cache write without cache-hit telemetry", + partialUsage: { cacheWrite: 1_000 }, + cacheHitTurns: 0, + cacheWriteTurns: 1, + }, + ])("recognizes $name as proof that the conversation is warm", (testCase) => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + { ...usage(3), ...testCase.partialUsage }, + usage(1_050, { cacheRead: 0, cacheWrite: 0 }), + ]); + + expect(diagnostics).toEqual({ + assistantTurns: 2, + cacheTelemetryTurns: 1, + cacheHitTurns: testCase.cacheHitTurns, + cacheWriteTurns: testCase.cacheWriteTurns, + cacheMisses: [{ turn: 2, inputTokens: 1_050, cacheRead: 0, cacheWrite: 0 }], + cacheMissInputTokens: 1_050, + unmeasuredPostWarmTurns: [], + }); + }); + + it("preserves measured misses without concealing later unmeasured warm turns", () => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + usage(3, { cacheRead: 0, cacheWrite: 1_000 }), + usage(1_050, { cacheRead: 0, cacheWrite: 0 }), + usage(1_100), + { ...usage(12), cacheRead: 1_000 }, + ]); + + expect(diagnostics.cacheMisses).toEqual([ + { turn: 2, inputTokens: 1_050, cacheRead: 0, cacheWrite: 0 }, + ]); + expect(diagnostics.cacheMissInputTokens).toBe(1_050); + expect(diagnostics.unmeasuredPostWarmTurns).toEqual([3, 4]); + expect(diagnostics.cacheHitTurns).toBe(1); + }); + + it("does not flag a cold conversation that never established a cache", () => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + usage(1_000, { cacheRead: 0, cacheWrite: 0 }), + usage(1_050, { cacheRead: 0, cacheWrite: 0 }), + ]); + + expect(diagnostics.cacheMisses).toEqual([]); + expect(diagnostics.cacheMissInputTokens).toBe(0); + }); + + it("does not classify output-only turns as input-cache misses", () => { + const diagnostics = buildRuntimeParityCacheDiagnostics([ + usage(3, { cacheRead: 0, cacheWrite: 1_000 }), + usage(0, { cacheRead: 0, cacheWrite: 0 }), + ]); + + expect(diagnostics.cacheMisses).toEqual([]); + }); + + it("reports an empty transcript without inventing cache telemetry", () => { + expect(buildRuntimeParityCacheDiagnostics([])).toEqual({ + assistantTurns: 0, + cacheTelemetryTurns: 0, + cacheHitTurns: 0, + cacheWriteTurns: 0, + cacheMisses: [], + cacheMissInputTokens: 0, + unmeasuredPostWarmTurns: [], + }); + }); +}); diff --git a/extensions/qa-lab/src/runtime-parity-cache-diagnostics.ts b/extensions/qa-lab/src/runtime-parity-cache-diagnostics.ts new file mode 100644 index 000000000000..38737550b7ce --- /dev/null +++ b/extensions/qa-lab/src/runtime-parity-cache-diagnostics.ts @@ -0,0 +1,69 @@ +import type { RuntimeParityUsage } from "./runtime-parity-usage.js"; + +export type RuntimeParityCacheMiss = { + turn: number; + inputTokens: number; + cacheRead: number; + cacheWrite: number; +}; + +export type RuntimeParityCacheDiagnostics = { + assistantTurns: number; + cacheTelemetryTurns: number; + cacheHitTurns: number; + cacheWriteTurns: number; + cacheMisses: RuntimeParityCacheMiss[]; + cacheMissInputTokens: number; + unmeasuredPostWarmTurns: number[]; +}; + +/** Detect complete cache losses only after this conversation has actually warmed its cache. */ +export function buildRuntimeParityCacheDiagnostics( + turns: readonly RuntimeParityUsage[], +): RuntimeParityCacheDiagnostics { + const diagnostics: RuntimeParityCacheDiagnostics = { + assistantTurns: turns.length, + cacheTelemetryTurns: 0, + cacheHitTurns: 0, + cacheWriteTurns: 0, + cacheMisses: [], + cacheMissInputTokens: 0, + unmeasuredPostWarmTurns: [], + }; + let cacheWasWarmed = false; + + for (const [index, usage] of turns.entries()) { + const cacheRead = usage.cacheRead; + const cacheWrite = usage.cacheWrite; + const cacheWasWarmBeforeTurn = cacheWasWarmed; + if (cacheRead !== undefined && cacheRead > 0) { + diagnostics.cacheHitTurns += 1; + cacheWasWarmed = true; + } + if (cacheWrite !== undefined && cacheWrite > 0) { + diagnostics.cacheWriteTurns += 1; + cacheWasWarmed = true; + } + if (cacheRead === undefined || cacheWrite === undefined) { + if (cacheWasWarmBeforeTurn) { + diagnostics.unmeasuredPostWarmTurns.push(index + 1); + } + continue; + } + diagnostics.cacheTelemetryTurns += 1; + const missedInputTokens = usage.inputTokens + cacheWrite; + // A zero-read rewrite still reprocesses the entire prefix; the write only + // repopulates the cache for a later turn and must not conceal this miss. + if (cacheWasWarmBeforeTurn && cacheRead === 0 && missedInputTokens > 0) { + diagnostics.cacheMisses.push({ + turn: index + 1, + inputTokens: missedInputTokens, + cacheRead, + cacheWrite, + }); + diagnostics.cacheMissInputTokens += missedInputTokens; + } + } + + return diagnostics; +} diff --git a/extensions/qa-lab/src/runtime-parity-usage.ts b/extensions/qa-lab/src/runtime-parity-usage.ts new file mode 100644 index 000000000000..f4cbfcd41b35 --- /dev/null +++ b/extensions/qa-lab/src/runtime-parity-usage.ts @@ -0,0 +1,7 @@ +export type RuntimeParityUsage = { + inputTokens: number; + outputTokens: number; + totalTokens: number; + cacheRead?: number; + cacheWrite?: number; +}; diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index 2ed9a83bf640..f2cb5eaf302f 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -20,8 +20,15 @@ import { } from "./gateway-log-sentinel.js"; import { discardIgnoredResponseBody } from "./ignored-response-body.js"; import * as parity from "./parity-shared.js"; +import { + buildRuntimeParityCacheDiagnostics, + type RuntimeParityCacheDiagnostics, +} from "./runtime-parity-cache-diagnostics.js"; +import type { RuntimeParityUsage } from "./runtime-parity-usage.js"; import { readRawQaSessionStore } from "./suite-runtime-agent-session.js"; +export type { RuntimeParityUsage } from "./runtime-parity-usage.js"; + // These are the canonical QA comparison cells, not the extensible product // AgentHarness registry. Broader harness coverage needs its own explicit lane. export type RuntimeId = "openclaw" | "codex"; @@ -37,14 +44,6 @@ export type RuntimeParityToolCall = { errorClass?: string; }; -export type RuntimeParityUsage = { - inputTokens: number; - outputTokens: number; - totalTokens: number; - cacheRead?: number; - cacheWrite?: number; -}; - export type RuntimeParityUsagePolicy = | { expectation: "assistant-message-required" } | { expectation: "not-applicable"; reason: string }; @@ -56,6 +55,7 @@ export type RuntimeParityCell = { providerPlanToolCalls?: RuntimeParityToolCall[]; finalText: string; usage: RuntimeParityUsage; + cacheDiagnostics?: RuntimeParityCacheDiagnostics; wallClockMs: number; bootstrapWallClockMs?: number; transportErrorClass?: string; @@ -1471,6 +1471,11 @@ export async function captureRuntimeParityCell( ...(mockToolCalls ? { providerPlanToolCalls: mockToolCalls } : {}), finalText: extractFinalAssistantText(transcriptRecords), usage: aggregateUsage(transcriptRecords), + cacheDiagnostics: buildRuntimeParityCacheDiagnostics( + transcriptRecords + .filter((record) => record.role === "assistant") + .map((record) => readUsageTotals(record.message.usage ?? null)), + ), wallClockMs: params.wallClockMs, ...(params.bootstrapWallClockMs === undefined ? {} diff --git a/extensions/qa-lab/src/token-efficiency-report.test.ts b/extensions/qa-lab/src/token-efficiency-report.test.ts index b91be3aca326..e2b903165502 100644 --- a/extensions/qa-lab/src/token-efficiency-report.test.ts +++ b/extensions/qa-lab/src/token-efficiency-report.test.ts @@ -1,5 +1,6 @@ // Qa Lab tests cover token efficiency report plugin behavior. import { describe, expect, it } from "vitest"; +import { buildRuntimeParityCacheDiagnostics } from "./runtime-parity-cache-diagnostics.js"; import type { RuntimeId, RuntimeParityCell, @@ -122,6 +123,445 @@ describe("token efficiency report", () => { ]); }); + it("detects a real cache regression even when cached-inclusive totals hide it", () => { + const openclaw = makeCell("openclaw", { + inputTokens: 7_403, + outputTokens: 220, + totalTokens: 435_810, + cacheRead: 415_492, + cacheWrite: 12_695, + }); + const codex = makeCell("codex", { + inputTokens: 25_894, + outputTokens: 220, + totalTokens: 495_710, + cacheRead: 445_189, + cacheWrite: 24_407, + }); + codex.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([ + { + inputTokens: 3, + outputTokens: 11, + totalTokens: 24_421, + cacheRead: 0, + cacheWrite: 24_407, + }, + { + inputTokens: 24_448, + outputTokens: 11, + totalTokens: 24_459, + cacheRead: 0, + cacheWrite: 0, + }, + ]); + + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([makeRuntimeParity("first-hour-cache-miss", openclaw, codex)]), + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]).toMatchObject({ + classification: "regression", + flagged: true, + openclaw: { processedTokens: 20_318, cacheReadTokens: 415_492 }, + codex: { + processedTokens: 50_521, + cacheReadTokens: 445_189, + cacheWriteTokens: 24_407, + cacheMisses: [{ turn: 2, inputTokens: 24_448, cacheRead: 0, cacheWrite: 0 }], + }, + }); + expect(report.rows[0]?.deltaPercent).toBeGreaterThan(145); + expect(report.aggregate.codex).toMatchObject({ + processedTokens: 50_521, + cacheMissCount: 1, + cacheMissInputTokens: 24_448, + }); + expect(renderTokenEfficiencyMarkdownReport(report)).toContain("turn 2 (24448 input)"); + }); + + it("does not treat additional reused cached input as newly processed work", () => { + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([ + makeRuntimeParity( + "different-cache-hit-totals", + makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 1_000, + cacheRead: 880, + cacheWrite: 0, + }), + makeCell("codex", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 5_000, + cacheRead: 4_880, + cacheWrite: 0, + }), + ), + ]), + }); + + expect(report.pass).toBe(true); + expect(report.rows[0]).toMatchObject({ + deltaPercent: 0, + classification: "neutral", + flagged: false, + openclaw: { processedTokens: 120 }, + codex: { processedTokens: 120 }, + }); + }); + + it("counts newly written cache input as genuinely processed work", () => { + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([ + makeRuntimeParity( + "cache-write-regression", + makeCell("openclaw", { + inputTokens: 10, + outputTokens: 5, + totalTokens: 105, + cacheRead: 90, + cacheWrite: 0, + }), + makeCell("codex", { + inputTokens: 10, + outputTokens: 5, + totalTokens: 105, + cacheRead: 0, + cacheWrite: 90, + }), + ), + ]), + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]).toMatchObject({ + classification: "regression", + flagged: true, + deltaPercent: 600, + openclaw: { processedTokens: 15 }, + codex: { processedTokens: 105 }, + }); + }); + + it("reports unavailable cache-miss telemetry as unknown rather than zero", () => { + const openclaw = makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }); + const codex = makeCell("codex", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }); + for (const cell of [openclaw, codex]) { + cell.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([cell.usage]); + } + + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([makeRuntimeParity("unknown-cache-telemetry", openclaw, codex)]), + }); + + expect(report.pass).toBe(true); + expect(report.rows[0]).toMatchObject({ + openclaw: { cacheReadTokens: null, cacheWriteTokens: null, cacheMisses: null }, + codex: { cacheReadTokens: null, cacheWriteTokens: null, cacheMisses: null }, + }); + expect(report.aggregate.openclaw).toMatchObject({ + cacheMissCount: null, + cacheMissInputTokens: null, + }); + expect(report.aggregate.codex).toMatchObject({ + cacheMissCount: null, + cacheMissInputTokens: null, + }); + expect(renderTokenEfficiencyMarkdownReport(report)).toContain("| N/A | N/A |"); + }); + + it("derives missing cache writes only from coherent measured cache reads", () => { + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([ + makeRuntimeParity( + "derived-cache-writes", + makeCell("openclaw", { + inputTokens: 10, + outputTokens: 5, + totalTokens: 115, + cacheRead: 100, + }), + makeCell("codex", { + inputTokens: 10, + outputTokens: 5, + totalTokens: 135, + cacheRead: 100, + }), + ), + ]), + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]).toMatchObject({ + classification: "regression", + flagged: true, + openclaw: { + processedTokens: 15, + processedTokenEvidence: "derived", + cacheWriteTokens: null, + }, + codex: { + processedTokens: 35, + processedTokenEvidence: "derived", + cacheWriteTokens: null, + }, + }); + }); + + it("fails live proof when processed tokens cannot be derived from partial cache telemetry", () => { + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([ + makeRuntimeParity( + "unverifiable-cache-writes", + makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }), + makeCell("codex", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 1_000, + }), + ), + ]), + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]).toMatchObject({ + classification: "neutral", + flagged: false, + openclaw: { processedTokenEvidence: "derived" }, + codex: { processedTokenEvidence: "unavailable" }, + }); + expect(report.failures).toEqual([ + "unverifiable-cache-writes codex live processed-token usage cannot be verified from cache-write telemetry or coherent cache-read totals", + ]); + expect(report.aggregate.codex).toMatchObject({ + processedTokenEvidence: "unavailable", + p50PerScenario: null, + p90PerScenario: null, + }); + expect(renderTokenEfficiencyMarkdownReport(report)).toContain("| delta | N/A |"); + expect(renderTokenEfficiencyMarkdownReport(report)).toContain( + "| codex | N/A | 1000 | N/A | N/A | N/A | N/A | N/A |", + ); + }); + + it("fails incoherent cache-read totals instead of fabricating derived cache writes", () => { + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([ + makeRuntimeParity( + "incoherent-cache-totals", + makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }), + makeCell("codex", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 125, + cacheRead: 100, + }), + ), + ]), + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]?.codex).toMatchObject({ + cacheReadTokens: 100, + cacheWriteTokens: null, + processedTokenEvidence: "unavailable", + }); + expect(report.failures).toEqual([ + "incoherent-cache-totals codex live processed-token usage cannot be verified from cache-write telemetry or coherent cache-read totals", + ]); + }); + + it("fails unexplained totals when both cache counters have already been measured", () => { + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([ + makeRuntimeParity( + "unexplained-cache-totals", + makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }), + makeCell("codex", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 1_000, + cacheRead: 100, + cacheWrite: 0, + }), + ), + ]), + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]?.codex).toMatchObject({ + cacheReadTokens: 100, + cacheWriteTokens: 0, + processedTokenEvidence: "unavailable", + }); + expect(report.failures).toEqual([ + "unexplained-cache-totals codex live processed-token usage cannot be verified from cache-write telemetry or coherent cache-read totals", + ]); + }); + + it("does not derive cache writes from partially observed post-warm cache reads", () => { + const openclaw = makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }); + const codex = makeCell("codex", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 1_120, + cacheRead: 100, + }); + codex.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([ + { inputTokens: 3, outputTokens: 11, totalTokens: 114, cacheRead: 100, cacheWrite: 0 }, + { inputTokens: 97, outputTokens: 9, totalTokens: 1_006 }, + ]); + + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([ + makeRuntimeParity("incomplete-cache-read-telemetry", openclaw, codex), + ]), + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]?.codex).toMatchObject({ + cacheReadTokens: 100, + cacheWriteTokens: null, + processedTokenEvidence: "unavailable", + unmeasuredPostWarmTurns: [2], + }); + expect(report.rows[0]).toMatchObject({ classification: "neutral", flagged: false }); + expect(report.failures).toEqual([ + "incomplete-cache-read-telemetry codex live processed-token usage cannot be verified from cache-write telemetry or coherent cache-read totals", + ]); + }); + + it("does not certify incomplete cache-write telemetry with unaccounted cache input", () => { + const openclaw = makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }); + const codex = makeCell("codex", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 1_120, + cacheRead: 100, + cacheWrite: 200, + }); + codex.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([ + { inputTokens: 3, outputTokens: 11, totalTokens: 314, cacheRead: 100, cacheWrite: 200 }, + { inputTokens: 97, outputTokens: 9, totalTokens: 806 }, + ]); + + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([ + makeRuntimeParity("incomplete-cache-write-telemetry", openclaw, codex), + ]), + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]?.codex).toMatchObject({ + cacheReadTokens: 100, + cacheWriteTokens: 200, + processedTokenEvidence: "unavailable", + unmeasuredPostWarmTurns: [2], + }); + expect(report.failures).toEqual([ + "incomplete-cache-write-telemetry codex live processed-token usage cannot be verified from cache-write telemetry or coherent cache-read totals", + ]); + }); + + it("keeps mixed post-warm telemetry unknown without discarding measured misses", () => { + const openclaw = makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }); + const codex = makeCell("codex", { + inputTokens: 1_053, + outputTokens: 33, + totalTokens: 2_086, + cacheRead: 0, + cacheWrite: 1_000, + }); + codex.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([ + { inputTokens: 3, outputTokens: 11, totalTokens: 1_014, cacheRead: 0, cacheWrite: 1_000 }, + { inputTokens: 1_050, outputTokens: 11, totalTokens: 1_061, cacheRead: 0, cacheWrite: 0 }, + { inputTokens: 0, outputTokens: 11, totalTokens: 11 }, + ]); + + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([makeRuntimeParity("mixed-cache-telemetry", openclaw, codex)]), + }); + + expect(report.rows[0]?.codex).toMatchObject({ + cacheMisses: [{ turn: 2, inputTokens: 1_050, cacheRead: 0, cacheWrite: 0 }], + unmeasuredPostWarmTurns: [3], + }); + expect(report.aggregate.codex).toMatchObject({ + cacheMissCount: null, + cacheMissInputTokens: null, + }); + expect(renderTokenEfficiencyMarkdownReport(report)).toContain( + "turn 2 (1050 input); unmeasured turns 3", + ); + }); + + it("preserves unmeasured warm turns when only partial cache telemetry is available", () => { + const openclaw = makeCell("openclaw", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }); + const codex = makeCell("codex", { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + }); + codex.cacheDiagnostics = buildRuntimeParityCacheDiagnostics([ + { inputTokens: 3, outputTokens: 11, totalTokens: 1_014, cacheWrite: 1_000 }, + { inputTokens: 100, outputTokens: 11, totalTokens: 111 }, + ]); + + const report = buildTokenEfficiencyReport({ + summary: makeLiveSummary([makeRuntimeParity("partial-warm-telemetry", openclaw, codex)]), + }); + + expect(report.pass).toBe(true); + expect(report.rows[0]?.codex).toMatchObject({ + cacheMisses: null, + unmeasuredPostWarmTurns: [2], + }); + expect(report.aggregate.codex).toMatchObject({ + cacheMissCount: null, + cacheMissInputTokens: null, + }); + expect(renderTokenEfficiencyMarkdownReport(report)).toContain("N/A (unmeasured turns 2)"); + }); + it("keeps live zero-usage rows failing instead of passing as neutral", () => { const report = buildTokenEfficiencyReport({ summary: makeLiveSummary([ diff --git a/extensions/qa-lab/src/token-efficiency-report.ts b/extensions/qa-lab/src/token-efficiency-report.ts index c5fb27886e80..ba7d8749c5ab 100644 --- a/extensions/qa-lab/src/token-efficiency-report.ts +++ b/extensions/qa-lab/src/token-efficiency-report.ts @@ -1,14 +1,35 @@ // Qa Lab plugin module implements token efficiency report behavior. +import type { RuntimeParityCacheMiss } from "./runtime-parity-cache-diagnostics.js"; import type { RuntimeId, RuntimeParityCell, RuntimeParityResult } from "./runtime-parity.js"; import { resolveRuntimeParityUsagePolicy } from "./runtime-parity.js"; +type ProcessedTokenEvidence = "measured" | "derived" | "unavailable"; + type TokenEfficiencyRuntimeUsage = { inputTokens: number; outputTokens: number; totalTokens: number; + processedTokens: number; + processedTokenEvidence: ProcessedTokenEvidence; + cacheReadTokens: number | null; + cacheWriteTokens: number | null; + cacheMisses: RuntimeParityCacheMiss[] | null; + unmeasuredPostWarmTurns: number[] | null; toolCallCount: number; }; +type TokenEfficiencyAggregateRuntimeUsage = { + totalTokens: number; + processedTokens: number; + processedTokenEvidence: ProcessedTokenEvidence; + cacheReadTokens: number | null; + cacheWriteTokens: number | null; + cacheMissCount: number | null; + cacheMissInputTokens: number | null; + p50PerScenario: number | null; + p90PerScenario: number | null; +}; + type TokenEfficiencyRow = { scenarioId: string; usageSource: "live-usage" | "mock-estimate"; @@ -29,8 +50,8 @@ type TokenEfficiencyReport = { rows: TokenEfficiencyRow[]; notApplicableScenarios: Array<{ scenarioId: string; reason: string }>; aggregate: { - openclaw: { totalTokens: number; p50PerScenario: number; p90PerScenario: number }; - codex: { totalTokens: number; p50PerScenario: number; p90PerScenario: number }; + openclaw: TokenEfficiencyAggregateRuntimeUsage; + codex: TokenEfficiencyAggregateRuntimeUsage; deltaPercent: number; flaggedScenarios: string[]; savingsScenarios: string[]; @@ -60,9 +81,20 @@ type BuildTokenEfficiencyReportParams = { }; const DEFAULT_THRESHOLD_PERCENT = 15; +const ZERO_AGGREGATE_RUNTIME: TokenEfficiencyAggregateRuntimeUsage = { + totalTokens: 0, + processedTokens: 0, + processedTokenEvidence: "unavailable", + cacheReadTokens: null, + cacheWriteTokens: null, + cacheMissCount: null, + cacheMissInputTokens: null, + p50PerScenario: 0, + p90PerScenario: 0, +}; const ZERO_AGGREGATE: TokenEfficiencyReport["aggregate"] = { - openclaw: { totalTokens: 0, p50PerScenario: 0, p90PerScenario: 0 }, - codex: { totalTokens: 0, p50PerScenario: 0, p90PerScenario: 0 }, + openclaw: { ...ZERO_AGGREGATE_RUNTIME }, + codex: { ...ZERO_AGGREGATE_RUNTIME }, deltaPercent: 0, flaggedScenarios: [], savingsScenarios: [], @@ -106,11 +138,97 @@ function formatPercent(value: number) { return `${sign}${value.toFixed(1)}%`; } +function formatOptionalCount(value: number | null): string { + return value === null ? "N/A" : String(value); +} + +function formatProcessedCount( + usage: Pick, +): string { + return usage.processedTokenEvidence === "unavailable" ? "N/A" : String(usage.processedTokens); +} + +function formatProcessedDelta(params: { + deltaPercent: number; + openclaw: Pick; + codex: Pick; +}): string { + return params.openclaw.processedTokenEvidence === "unavailable" || + params.codex.processedTokenEvidence === "unavailable" + ? "N/A" + : formatPercent(params.deltaPercent); +} + +function formatCacheMisses( + misses: readonly RuntimeParityCacheMiss[] | null, + unmeasuredPostWarmTurns: readonly number[] | null, +): string { + if (misses === null) { + return unmeasuredPostWarmTurns?.length + ? `N/A (unmeasured turns ${unmeasuredPostWarmTurns.join(", ")})` + : "N/A"; + } + const measuredMisses = + misses.length === 0 + ? "none" + : misses.map((miss) => `turn ${miss.turn} (${miss.inputTokens} input)`).join(", "); + if (!unmeasuredPostWarmTurns?.length) { + return measuredMisses; + } + const unknownTurns = `unmeasured turns ${unmeasuredPostWarmTurns.join(", ")}`; + return measuredMisses === "none" ? `N/A (${unknownTurns})` : `${measuredMisses}; ${unknownTurns}`; +} + function runtimeUsage(cell: RuntimeParityCell): TokenEfficiencyRuntimeUsage { + const inputTokens = normalizeTokenCount(cell.usage.inputTokens); + const outputTokens = normalizeTokenCount(cell.usage.outputTokens); + const totalTokens = normalizeTokenCount(cell.usage.totalTokens); + const cacheReadTokens = + cell.usage.cacheRead === undefined ? null : normalizeTokenCount(cell.usage.cacheRead); + const cacheWriteTokens = + cell.usage.cacheWrite === undefined ? null : normalizeTokenCount(cell.usage.cacheWrite); + const cacheDiagnostics = cell.cacheDiagnostics; + const baseProcessedTokens = inputTokens + outputTokens; + const completeCacheTelemetry = + cacheDiagnostics === undefined || + cacheDiagnostics.cacheTelemetryTurns === cacheDiagnostics.assistantTurns; + const unaccountedCacheTokens = + totalTokens - baseProcessedTokens - (cacheReadTokens ?? 0) - (cacheWriteTokens ?? 0); + let processedTokens = baseProcessedTokens; + let processedTokenEvidence: ProcessedTokenEvidence = "unavailable"; + // Aggregate counters can omit an unmeasured turn. Only exact accounting + // proves that omitted nonnegative cache reads and writes were both zero. + if ( + cacheWriteTokens !== null && + unaccountedCacheTokens >= 0 && + (cacheReadTokens === null || unaccountedCacheTokens === 0) && + (completeCacheTelemetry || unaccountedCacheTokens === 0) + ) { + processedTokens += cacheWriteTokens; + processedTokenEvidence = "measured"; + } else if (cacheReadTokens !== null && cacheWriteTokens === null && completeCacheTelemetry) { + const derivedCacheWriteTokens = totalTokens - baseProcessedTokens - cacheReadTokens; + if (derivedCacheWriteTokens >= 0) { + processedTokens += derivedCacheWriteTokens; + processedTokenEvidence = "derived"; + } + } else if (totalTokens === baseProcessedTokens) { + // Nonnegative usage components prove both missing cache counters are zero. + processedTokenEvidence = "derived"; + } return { - inputTokens: normalizeTokenCount(cell.usage.inputTokens), - outputTokens: normalizeTokenCount(cell.usage.outputTokens), - totalTokens: normalizeTokenCount(cell.usage.totalTokens), + inputTokens, + outputTokens, + totalTokens, + processedTokens, + processedTokenEvidence, + cacheReadTokens, + cacheWriteTokens, + cacheMisses: + cacheDiagnostics && cacheDiagnostics.cacheTelemetryTurns > 0 + ? cacheDiagnostics.cacheMisses + : null, + unmeasuredPostWarmTurns: cacheDiagnostics?.unmeasuredPostWarmTurns ?? null, toolCallCount: cell.toolCalls.length, }; } @@ -128,7 +246,10 @@ function buildRow(params: { }): TokenEfficiencyRow { const openclaw = runtimeUsage(params.result.cells.openclaw); const codex = runtimeUsage(params.result.cells.codex); - const delta = deltaPercent(openclaw.totalTokens, codex.totalTokens); + const comparable = + openclaw.processedTokenEvidence !== "unavailable" && + codex.processedTokenEvidence !== "unavailable"; + const delta = comparable ? deltaPercent(openclaw.processedTokens, codex.processedTokens) : 0; const flagged = params.usageSource === "live-usage" && delta > params.thresholdPercent; const classification = delta > params.thresholdPercent @@ -148,23 +269,65 @@ function buildRow(params: { }; } -function buildAggregate(rows: readonly TokenEfficiencyRow[]): TokenEfficiencyReport["aggregate"] { - const openclawTotals = rows.map((row) => row.openclaw.totalTokens); - const codexTotals = rows.map((row) => row.codex.totalTokens); - const openclawTotalTokens = openclawTotals.reduce((sum, value) => sum + value, 0); - const codexTotalTokens = codexTotals.reduce((sum, value) => sum + value, 0); +function sumKnownCounts(values: readonly (number | null)[]): number | null { + let total = 0; + for (const value of values) { + if (value === null) { + return null; + } + total += value; + } + return total; +} + +function buildAggregateRuntime( + rows: readonly TokenEfficiencyRow[], + runtime: RuntimeId, +): TokenEfficiencyAggregateRuntimeUsage { + const usages = rows.map((row) => row[runtime]); + const processedTotals = usages.map((usage) => usage.processedTokens); + const processedTokenEvidence: ProcessedTokenEvidence = usages.some( + (usage) => usage.processedTokenEvidence === "unavailable", + ) + ? "unavailable" + : usages.some((usage) => usage.processedTokenEvidence === "derived") + ? "derived" + : "measured"; return { - openclaw: { - totalTokens: openclawTotalTokens, - p50PerScenario: percentile(openclawTotals, 50), - p90PerScenario: percentile(openclawTotals, 90), - }, - codex: { - totalTokens: codexTotalTokens, - p50PerScenario: percentile(codexTotals, 50), - p90PerScenario: percentile(codexTotals, 90), - }, - deltaPercent: deltaPercent(openclawTotalTokens, codexTotalTokens), + totalTokens: usages.reduce((sum, usage) => sum + usage.totalTokens, 0), + processedTokens: processedTotals.reduce((sum, value) => sum + value, 0), + processedTokenEvidence, + cacheReadTokens: sumKnownCounts(usages.map((usage) => usage.cacheReadTokens)), + cacheWriteTokens: sumKnownCounts(usages.map((usage) => usage.cacheWriteTokens)), + cacheMissCount: sumKnownCounts( + usages.map((usage) => + usage.unmeasuredPostWarmTurns?.length ? null : (usage.cacheMisses?.length ?? null), + ), + ), + cacheMissInputTokens: sumKnownCounts( + usages.map((usage) => + usage.unmeasuredPostWarmTurns?.length + ? null + : (usage.cacheMisses?.reduce((sum, cacheMiss) => sum + cacheMiss.inputTokens, 0) ?? null), + ), + ), + p50PerScenario: + processedTokenEvidence === "unavailable" ? null : percentile(processedTotals, 50), + p90PerScenario: + processedTokenEvidence === "unavailable" ? null : percentile(processedTotals, 90), + }; +} + +function buildAggregate(rows: readonly TokenEfficiencyRow[]): TokenEfficiencyReport["aggregate"] { + const openclaw = buildAggregateRuntime(rows, "openclaw"); + const codex = buildAggregateRuntime(rows, "codex"); + const comparable = + openclaw.processedTokenEvidence !== "unavailable" && + codex.processedTokenEvidence !== "unavailable"; + return { + openclaw, + codex, + deltaPercent: comparable ? deltaPercent(openclaw.processedTokens, codex.processedTokens) : 0, flaggedScenarios: rows.filter((row) => row.flagged).map((row) => row.scenarioId), savingsScenarios: rows .filter((row) => row.classification === "savings") @@ -180,6 +343,13 @@ function liveEvidenceFailures(row: TokenEfficiencyRow): string[] { if (row.codex.totalTokens <= 0) { failures.push(`${row.scenarioId} codex live usage totalTokens=${row.codex.totalTokens}`); } + for (const runtime of ["openclaw", "codex"] as const) { + if (row[runtime].processedTokenEvidence === "unavailable") { + failures.push( + `${row.scenarioId} ${runtime} live processed-token usage cannot be verified from cache-write telemetry or coherent cache-read totals`, + ); + } + } return failures; } @@ -302,6 +472,9 @@ export function buildTokenEfficiencyReport( failures, notes: [ "Token totals are read from RuntimeParityCell.usage, which is captured from normalized AssistantMessage.usage.", + "Efficiency deltas and percentiles compare newly processed uncached input, cache-write input, and output; reused cached input remains separately reported and never masks a regression.", + "Missing cache-write counts are derived only when measured cache reads and coherent usage totals prove the exact processed input; otherwise live efficiency proof fails.", + "Post-warm cache misses require measured zero cache reads and newly processed input after the same conversation has already established a cache; cache rewrites are included and unavailable telemetry is N/A.", "Codex savings are reported as savings and do not fail the gate; only positive Codex-over-OpenClaw live deltas exceed the threshold.", usageSource === "mock-estimate" ? "Mock-provider token totals are labeled as estimates and do not block the token-efficiency gate." @@ -318,7 +491,7 @@ export function renderTokenEfficiencyMarkdownReport(report: TokenEfficiencyRepor ...(report.providerMode ? [`- Provider mode: ${report.providerMode}`] : []), `- Verdict: ${report.status === "skipped" ? "skipped" : report.pass ? "pass" : "fail"}`, `- Usage source: ${report.rows[0]?.usageSource ?? "none"}`, - `- Threshold: Codex token increase > ${report.thresholdPercent.toFixed(1)}%`, + `- Threshold: Codex processed-token increase > ${report.thresholdPercent.toFixed(1)}%`, "", ]; @@ -329,11 +502,11 @@ export function renderTokenEfficiencyMarkdownReport(report: TokenEfficiencyRepor lines.push( "## Aggregate Metrics", "", - "| Runtime | Total tokens | p50 per scenario | p90 per scenario |", - "| --- | ---: | ---: | ---: |", - `| openclaw | ${report.aggregate.openclaw.totalTokens} | ${report.aggregate.openclaw.p50PerScenario} | ${report.aggregate.openclaw.p90PerScenario} |`, - `| codex | ${report.aggregate.codex.totalTokens} | ${report.aggregate.codex.p50PerScenario} | ${report.aggregate.codex.p90PerScenario} |`, - `| delta | ${formatPercent(report.aggregate.deltaPercent)} | | |`, + "| Runtime | Processed tokens | Total tokens | Cached input | Cache writes | Post-warm cache misses | p50 per scenario | p90 per scenario |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + `| openclaw | ${formatProcessedCount(report.aggregate.openclaw)} | ${report.aggregate.openclaw.totalTokens} | ${formatOptionalCount(report.aggregate.openclaw.cacheReadTokens)} | ${formatOptionalCount(report.aggregate.openclaw.cacheWriteTokens)} | ${formatOptionalCount(report.aggregate.openclaw.cacheMissCount)} | ${formatOptionalCount(report.aggregate.openclaw.p50PerScenario)} | ${formatOptionalCount(report.aggregate.openclaw.p90PerScenario)} |`, + `| codex | ${formatProcessedCount(report.aggregate.codex)} | ${report.aggregate.codex.totalTokens} | ${formatOptionalCount(report.aggregate.codex.cacheReadTokens)} | ${formatOptionalCount(report.aggregate.codex.cacheWriteTokens)} | ${formatOptionalCount(report.aggregate.codex.cacheMissCount)} | ${formatOptionalCount(report.aggregate.codex.p50PerScenario)} | ${formatOptionalCount(report.aggregate.codex.p90PerScenario)} |`, + `| delta | ${formatProcessedDelta({ deltaPercent: report.aggregate.deltaPercent, openclaw: report.aggregate.openclaw, codex: report.aggregate.codex })} | | | | | | |`, "", ); @@ -341,12 +514,12 @@ export function renderTokenEfficiencyMarkdownReport(report: TokenEfficiencyRepor lines.push( "## Scenario Efficiency", "", - "| Scenario | Source | OpenClaw in/out/total/tools | Codex in/out/total/tools | Token delta | Classification | Flagged | Tools used |", - "| --- | --- | ---: | ---: | ---: | --- | --- | --- |", + "| Scenario | Source | OpenClaw processed/in/out/cached/written/total/tools | Codex processed/in/out/cached/written/total/tools | Processed-token delta | Classification | Flagged | OpenClaw cache misses | Codex cache misses | Tools used |", + "| --- | --- | ---: | ---: | ---: | --- | --- | --- | --- | --- |", ); for (const row of report.rows) { lines.push( - `| ${row.scenarioId} | ${row.usageSource} | ${row.openclaw.inputTokens}/${row.openclaw.outputTokens}/${row.openclaw.totalTokens}/${row.openclaw.toolCallCount} | ${row.codex.inputTokens}/${row.codex.outputTokens}/${row.codex.totalTokens}/${row.codex.toolCallCount} | ${formatPercent(row.deltaPercent)} | ${row.classification} | ${row.flagged ? "yes" : "no"} | ${row.toolsUsed.join(", ")} |`, + `| ${row.scenarioId} | ${row.usageSource} | ${formatProcessedCount(row.openclaw)}/${row.openclaw.inputTokens}/${row.openclaw.outputTokens}/${formatOptionalCount(row.openclaw.cacheReadTokens)}/${formatOptionalCount(row.openclaw.cacheWriteTokens)}/${row.openclaw.totalTokens}/${row.openclaw.toolCallCount} | ${formatProcessedCount(row.codex)}/${row.codex.inputTokens}/${row.codex.outputTokens}/${formatOptionalCount(row.codex.cacheReadTokens)}/${formatOptionalCount(row.codex.cacheWriteTokens)}/${row.codex.totalTokens}/${row.codex.toolCallCount} | ${formatProcessedDelta({ deltaPercent: row.deltaPercent, openclaw: row.openclaw, codex: row.codex })} | ${row.classification} | ${row.flagged ? "yes" : "no"} | ${formatCacheMisses(row.openclaw.cacheMisses, row.openclaw.unmeasuredPostWarmTurns)} | ${formatCacheMisses(row.codex.cacheMisses, row.codex.unmeasuredPostWarmTurns)} | ${row.toolsUsed.join(", ")} |`, ); } lines.push(""); diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.discord-group.json b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.discord-group.json index c0242dc9f9c9..73f3ee9d42f2 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.discord-group.json +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.discord-group.json @@ -1,4 +1,15 @@ [ + { + "description": "List configured agent ids with name/model/runtime metadata, allowed as `sessions_spawn(runtime:\"subagent\")` targets.", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "required": [], + "type": "object" + }, + "name": "agents_list", + "type": "function" + }, { "description": "Send/manage channel messages. Supports actions: send.", "inputSchema": { @@ -126,17 +137,6 @@ "name": "message", "type": "function" }, - { - "description": "List configured agent ids with name/model/runtime metadata, allowed as `sessions_spawn(runtime:\"subagent\")` targets.", - "inputSchema": { - "additionalProperties": false, - "properties": {}, - "required": [], - "type": "object" - }, - "name": "agents_list", - "type": "function" - }, { "description": "Spawn clean child; default `runtime=\"subagent\"`. `mode=\"run\"` one-shot; `mode=\"session\"` persistent/thread-bound only on supporting requester channel. `agentId` targets a configured agent (see agents_list); `model` overrides its model; `cleanup` delete|keep hidden child session; `sandbox` inherit|require. `visible=true`: persistent dashboard session; subagent only; omit `mode` (no `mode=\"run\"`), `thread`, `thinking`, `lightContext`, `attachments`, `attachAs`; inherited tool allow/denylist blocks it at spawn with no config override; may check out a git worktree via `worktree`/`worktreeName`/`worktreeBaseRef`. Session listing/addressing obeys `tools.sessions.visibility` (`tree` default: current + own spawn subtree). Inherits parent workspace. Native task arrives as first `[Subagent Task]`. Native transcript needed: `context=\"fork\"`; else omit/isolated. Use fresh child for sidecar/parallel batch reads, multi-step search, data collection; avoid quick lookup/single read unless policy prefers. After spawn, do non-overlap work. Run result returns; session output stays thread.", "inputSchema": { @@ -263,168 +263,6 @@ "description": "", "name": "openclaw", "tools": [ - { - "deferLoading": true, - "description": "Paired nodes: status/list with active-computer presence; pass node to describe/control. Pairing lifecycle (pending/approve/reject), notify, camera_snap/camera_list/camera_clip (with audio), photos_latest, screen_snapshot, screen_record video, location_get, notifications_list + notifications_action (open/dismiss/reply), device_status/device_info/device_permissions/device_health, executable lookup (which + bins), generic invoke. Files: file_fetch.", - "inputSchema": { - "properties": { - "action": { - "enum": [ - "status", - "describe", - "pending", - "approve", - "reject", - "notify", - "camera_snap", - "camera_list", - "camera_clip", - "photos_latest", - "screen_record", - "screen_snapshot", - "location_get", - "notifications_list", - "notifications_action", - "device_status", - "device_info", - "device_permissions", - "device_health", - "which", - "invoke" - ], - "type": "string" - }, - "bins": { - "description": "which: executable names to resolve on the selected node.", - "items": { - "minLength": 1, - "type": "string" - }, - "maxItems": 64, - "minItems": 1, - "type": "array" - }, - "body": { - "type": "string" - }, - "delayMs": { - "minimum": 0, - "type": "integer" - }, - "delivery": { - "enum": ["system", "overlay", "auto"], - "type": "string" - }, - "desiredAccuracy": { - "enum": ["coarse", "balanced", "precise"], - "type": "string" - }, - "deviceId": { - "type": "string" - }, - "duration": { - "type": "string" - }, - "durationMs": { - "maximum": 300000, - "minimum": 1, - "type": "integer" - }, - "facing": { - "description": "camera_snap: front/back/both; camera_clip: front/back only.", - "enum": ["front", "back", "both"], - "type": "string" - }, - "fps": { - "exclusiveMinimum": 0, - "type": "number" - }, - "gatewayToken": { - "type": "string" - }, - "gatewayUrl": { - "type": "string" - }, - "includeAudio": { - "type": "boolean" - }, - "invokeCommand": { - "type": "string" - }, - "invokeParamsJson": { - "type": "string" - }, - "invokeTimeoutMs": { - "minimum": 1, - "type": "integer" - }, - "limit": { - "maximum": 20, - "minimum": 1, - "type": "integer" - }, - "locationTimeoutMs": { - "minimum": 1, - "type": "integer" - }, - "maxAgeMs": { - "minimum": 0, - "type": "integer" - }, - "maxWidth": { - "minimum": 1, - "type": "integer" - }, - "node": { - "description": "Node ID, name, or IP. Required for describe and node-targeted actions; use status to discover nodes.", - "type": "string" - }, - "notificationAction": { - "enum": ["open", "dismiss", "reply"], - "type": "string" - }, - "notificationKey": { - "type": "string" - }, - "notificationReplyText": { - "type": "string" - }, - "outPath": { - "type": "string" - }, - "priority": { - "enum": ["passive", "active", "timeSensitive"], - "type": "string" - }, - "quality": { - "maximum": 1, - "minimum": 0, - "type": "number" - }, - "requestId": { - "type": "string" - }, - "screenIndex": { - "minimum": 0, - "type": "integer" - }, - "sound": { - "type": "string" - }, - "timeoutMs": { - "minimum": 1, - "type": "integer" - }, - "title": { - "type": "string" - } - }, - "required": ["action"], - "type": "object" - }, - "name": "nodes", - "type": "function" - }, { "deferLoading": true, "description": "Gateway scheduler: reminders, delayed self-wakeups, loops, recurring work, event watchers. Never exec sleep/poll as timer.\n\nACTIONS: status | list [includeDisabled,limit?,offset?] (use nextOffset for the next page) | get jobId | add job | update jobId patch | remove jobId | run jobId (runMode \"force\"=now) | runs jobId = history | next_check in:\"30m\" (own paced run only) | wake text mode?:\"now\"|\"next-heartbeat\"(default) nudges a caller-owned lane (sessionKey/agentId to pick another).\n\nADD: {name?,schedule,payload,sessionTarget?,pacing?,trigger?,delivery?,enabled?}. Required: schedule+payload.\n\nSCHEDULE:\n- {kind:\"at\",at:\"ISO-8601\"} one-shot; no tz=UTC; auto-deletes after run.\n- {kind:\"every\",everyMs}.\n- {kind:\"cron\",expr,tz?:\"IANA\"}: expr is wall time in tz; never pre-convert to UTC; no tz=gateway host local. 18:00 Shanghai => {expr:\"0 18 * * *\",tz:\"Asia/Shanghai\"}.\n- {kind:\"stream\",command:[argv],mode?:\"line\"|\"match\",match?}: fires on supervised process output; needs cron.triggers.enabled.\n\nTARGET+PAYLOAD:\n- \"current\" (agentTurn default) = this conversation: run carries this chat's context, result lands here. Self-wakeup/\"continue later\"/loop = at|every + agentTurn + current.\n- \"isolated\" = fresh detached session (shows in `openclaw tasks`); standalone background work.\n- \"main\" = heartbeat lane; payload {kind:\"systemEvent\",text} (systemEvent default target).\n- \"session:\" = named session.\n- agentTurn {kind:\"agentTurn\",message,model?,thinking?,timeoutSeconds?}; timeoutSeconds 0=none.\n- script {kind:\"script\",script,timeoutSeconds?,toolBudget?}: main|isolated only; needs cron.triggers.enabled.\n\nPACED LOOP: recurring job + pacing{min?,max?} durations (\"15m\",\"4h\"; at least one). Inside its run, job calls next_check in:\"\" to set the next delay (clamped to bounds, measured from run end; failed runs keep normal backoff). Adaptive polling: tighten when active, back off when quiet.\n\nTRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call(\"exec\",{command:\"...\"}).\n\nDELIVERY {mode:\"none\"|\"announce\"|\"webhook\",channel?,to?,threadId?,bestEffort?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:\"none\". webhook posts finished-run event to URL in `to`.\n\nJob wakeMode (main jobs): \"now\"(default)|\"next-heartbeat\". Restricted cron-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.", @@ -1265,31 +1103,6 @@ "name": "cron", "type": "function" }, - { - "deferLoading": true, - "description": "Convert text to spoken audio (TTS) with the configured voice provider. Only explicit voice/speech/TTS intent or active TTS config; never ordinary text reply. Audio auto-delivered. After success follow reply instructions; no duplicate text/audio.", - "inputSchema": { - "properties": { - "channel": { - "description": "Channel id; output-format hint.", - "type": "string" - }, - "text": { - "description": "Text to speak.", - "type": "string" - }, - "timeoutMs": { - "description": "Provider timeout ms.", - "minimum": 1, - "type": "integer" - } - }, - "required": ["text"], - "type": "object" - }, - "name": "tts", - "type": "function" - }, { "deferLoading": true, "description": "Read gateway config + schema. Writes/restart: use openclaw tool.", @@ -1319,6 +1132,223 @@ "name": "gateway", "type": "function" }, + { + "deferLoading": true, + "description": "Paired nodes: status/list with active-computer presence; pass node to describe/control. Pairing lifecycle (pending/approve/reject), notify, camera_snap/camera_list/camera_clip (with audio), photos_latest, screen_snapshot, screen_record video, location_get, notifications_list + notifications_action (open/dismiss/reply), device_status/device_info/device_permissions/device_health, executable lookup (which + bins), generic invoke. Files: file_fetch.", + "inputSchema": { + "properties": { + "action": { + "enum": [ + "status", + "describe", + "pending", + "approve", + "reject", + "notify", + "camera_snap", + "camera_list", + "camera_clip", + "photos_latest", + "screen_record", + "screen_snapshot", + "location_get", + "notifications_list", + "notifications_action", + "device_status", + "device_info", + "device_permissions", + "device_health", + "which", + "invoke" + ], + "type": "string" + }, + "bins": { + "description": "which: executable names to resolve on the selected node.", + "items": { + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "type": "array" + }, + "body": { + "type": "string" + }, + "delayMs": { + "minimum": 0, + "type": "integer" + }, + "delivery": { + "enum": ["system", "overlay", "auto"], + "type": "string" + }, + "desiredAccuracy": { + "enum": ["coarse", "balanced", "precise"], + "type": "string" + }, + "deviceId": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "durationMs": { + "maximum": 300000, + "minimum": 1, + "type": "integer" + }, + "facing": { + "description": "camera_snap: front/back/both; camera_clip: front/back only.", + "enum": ["front", "back", "both"], + "type": "string" + }, + "fps": { + "exclusiveMinimum": 0, + "type": "number" + }, + "gatewayToken": { + "type": "string" + }, + "gatewayUrl": { + "type": "string" + }, + "includeAudio": { + "type": "boolean" + }, + "invokeCommand": { + "type": "string" + }, + "invokeParamsJson": { + "type": "string" + }, + "invokeTimeoutMs": { + "minimum": 1, + "type": "integer" + }, + "limit": { + "maximum": 20, + "minimum": 1, + "type": "integer" + }, + "locationTimeoutMs": { + "minimum": 1, + "type": "integer" + }, + "maxAgeMs": { + "minimum": 0, + "type": "integer" + }, + "maxWidth": { + "minimum": 1, + "type": "integer" + }, + "node": { + "description": "Node ID, name, or IP. Required for describe and node-targeted actions; use status to discover nodes.", + "type": "string" + }, + "notificationAction": { + "enum": ["open", "dismiss", "reply"], + "type": "string" + }, + "notificationKey": { + "type": "string" + }, + "notificationReplyText": { + "type": "string" + }, + "outPath": { + "type": "string" + }, + "priority": { + "enum": ["passive", "active", "timeSensitive"], + "type": "string" + }, + "quality": { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + "requestId": { + "type": "string" + }, + "screenIndex": { + "minimum": 0, + "type": "integer" + }, + "sound": { + "type": "string" + }, + "timeoutMs": { + "minimum": 1, + "type": "integer" + }, + "title": { + "type": "string" + } + }, + "required": ["action"], + "type": "object" + }, + "name": "nodes", + "type": "function" + }, + { + "deferLoading": true, + "description": "Show visible-session model/usage/time/cost/tasks. `sessionKey=\"current\"` for current; UI labels are not keys. `model` overrides; `model=default` resets. Use for active model/session questions.", + "inputSchema": { + "properties": { + "changesSince": { + "minimum": 0, + "type": "integer" + }, + "model": { + "type": "string" + }, + "sessionKey": { + "type": "string" + } + }, + "type": "object" + }, + "name": "session_status", + "type": "function" + }, + { + "deferLoading": true, + "description": "Read sanitized visible-session history. Before reply/debug/resume. Supports limit, offset, search-result sessionId/messageId anchors, and tool messages.", + "inputSchema": { + "properties": { + "includeTools": { + "type": "boolean" + }, + "limit": { + "minimum": 1, + "type": "integer" + }, + "messageId": { + "minLength": 1, + "type": "string" + }, + "offset": { + "minimum": 0, + "type": "integer" + }, + "sessionId": { + "minLength": 1, + "type": "string" + }, + "sessionKey": { + "type": "string" + } + }, + "required": ["sessionKey"], + "type": "object" + }, + "name": "sessions_history", + "type": "function" + }, { "deferLoading": true, "description": "List visible sessions; filter kind/label/agentId/search/activity/archive. Preview recent messages inline via includeLastMessage/messageLimit; includeDerivedTitles adds derived titles. Use before history/send target selection.", @@ -1370,40 +1400,6 @@ "name": "sessions_list", "type": "function" }, - { - "deferLoading": true, - "description": "Read sanitized visible-session history. Before reply/debug/resume. Supports limit, offset, search-result sessionId/messageId anchors, and tool messages.", - "inputSchema": { - "properties": { - "includeTools": { - "type": "boolean" - }, - "limit": { - "minimum": 1, - "type": "integer" - }, - "messageId": { - "minLength": 1, - "type": "string" - }, - "offset": { - "minimum": 0, - "type": "integer" - }, - "sessionId": { - "minLength": 1, - "type": "string" - }, - "sessionKey": { - "type": "string" - } - }, - "required": ["sessionKey"], - "type": "object" - }, - "name": "sessions_history", - "type": "function" - }, { "deferLoading": true, "description": "Search your own past sessions for matching user and assistant text. Follow up with sessions_history using a returned sessionKey, sessionId, and messageId for neighboring context.", @@ -1488,23 +1484,54 @@ }, { "deferLoading": true, - "description": "Show visible-session model/usage/time/cost/tasks. `sessionKey=\"current\"` for current; UI labels are not keys. `model` overrides; `model=default` resets. Use for active model/session questions.", + "description": "Convert text to spoken audio (TTS) with the configured voice provider. Only explicit voice/speech/TTS intent or active TTS config; never ordinary text reply. Audio auto-delivered. After success follow reply instructions; no duplicate text/audio.", "inputSchema": { "properties": { - "changesSince": { - "minimum": 0, - "type": "integer" - }, - "model": { + "channel": { + "description": "Channel id; output-format hint.", "type": "string" }, - "sessionKey": { + "text": { + "description": "Text to speak.", + "type": "string" + }, + "timeoutMs": { + "description": "Provider timeout ms.", + "minimum": 1, + "type": "integer" + } + }, + "required": ["text"], + "type": "object" + }, + "name": "tts", + "type": "function" + }, + { + "deferLoading": true, + "description": "Fetch URL; extract readable markdown/text. Lightweight; no browser automation.", + "inputSchema": { + "properties": { + "extractMode": { + "default": "markdown", + "description": "Extract as markdown/text.", + "enum": ["markdown", "text"], + "type": "string" + }, + "maxChars": { + "description": "Max chars returned; truncates.", + "minimum": 100, + "type": "integer" + }, + "url": { + "description": "HTTP(S) URL.", "type": "string" } }, + "required": ["url"], "type": "object" }, - "name": "session_status", + "name": "web_fetch", "type": "function" }, { @@ -1574,33 +1601,6 @@ }, "name": "web_search", "type": "function" - }, - { - "deferLoading": true, - "description": "Fetch URL; extract readable markdown/text. Lightweight; no browser automation.", - "inputSchema": { - "properties": { - "extractMode": { - "default": "markdown", - "description": "Extract as markdown/text.", - "enum": ["markdown", "text"], - "type": "string" - }, - "maxChars": { - "description": "Max chars returned; truncates.", - "minimum": 100, - "type": "integer" - }, - "url": { - "description": "HTTP(S) URL.", - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "web_fetch", - "type": "function" } ], "type": "namespace" diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.heartbeat-turn.json b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.heartbeat-turn.json index 2277fc515dbb..f8ce73fa1b2e 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.heartbeat-turn.json +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.heartbeat-turn.json @@ -1,4 +1,15 @@ [ + { + "description": "List configured agent ids with name/model/runtime metadata, allowed as `sessions_spawn(runtime:\"subagent\")` targets.", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "required": [], + "type": "object" + }, + "name": "agents_list", + "type": "function" + }, { "description": "Send/manage channel messages. Supports actions: send.", "inputSchema": { @@ -126,17 +137,6 @@ "name": "message", "type": "function" }, - { - "description": "List configured agent ids with name/model/runtime metadata, allowed as `sessions_spawn(runtime:\"subagent\")` targets.", - "inputSchema": { - "additionalProperties": false, - "properties": {}, - "required": [], - "type": "object" - }, - "name": "agents_list", - "type": "function" - }, { "description": "Spawn clean child; default `runtime=\"subagent\"`. `mode=\"run\"` one-shot background. `agentId` targets a configured agent (see agents_list); `model` overrides its model; `cleanup` delete|keep hidden child session; `sandbox` inherit|require. `visible=true`: persistent dashboard session; subagent only; omit `mode` (no `mode=\"run\"`), `thread`, `thinking`, `lightContext`, `attachments`, `attachAs`; inherited tool allow/denylist blocks it at spawn with no config override; may check out a git worktree via `worktree`/`worktreeName`/`worktreeBaseRef`. Session listing/addressing obeys `tools.sessions.visibility` (`tree` default: current + own spawn subtree). Inherits parent workspace. Native task arrives as first `[Subagent Task]`. Native transcript needed: `context=\"fork\"`; else omit/isolated. Use fresh child for sidecar/parallel batch reads, multi-step search, data collection; avoid quick lookup/single read unless policy prefers. After spawn, do non-overlap work while run result returns.", "inputSchema": { @@ -259,168 +259,6 @@ "description": "", "name": "openclaw", "tools": [ - { - "deferLoading": true, - "description": "Paired nodes: status/list with active-computer presence; pass node to describe/control. Pairing lifecycle (pending/approve/reject), notify, camera_snap/camera_list/camera_clip (with audio), photos_latest, screen_snapshot, screen_record video, location_get, notifications_list + notifications_action (open/dismiss/reply), device_status/device_info/device_permissions/device_health, executable lookup (which + bins), generic invoke. Files: file_fetch.", - "inputSchema": { - "properties": { - "action": { - "enum": [ - "status", - "describe", - "pending", - "approve", - "reject", - "notify", - "camera_snap", - "camera_list", - "camera_clip", - "photos_latest", - "screen_record", - "screen_snapshot", - "location_get", - "notifications_list", - "notifications_action", - "device_status", - "device_info", - "device_permissions", - "device_health", - "which", - "invoke" - ], - "type": "string" - }, - "bins": { - "description": "which: executable names to resolve on the selected node.", - "items": { - "minLength": 1, - "type": "string" - }, - "maxItems": 64, - "minItems": 1, - "type": "array" - }, - "body": { - "type": "string" - }, - "delayMs": { - "minimum": 0, - "type": "integer" - }, - "delivery": { - "enum": ["system", "overlay", "auto"], - "type": "string" - }, - "desiredAccuracy": { - "enum": ["coarse", "balanced", "precise"], - "type": "string" - }, - "deviceId": { - "type": "string" - }, - "duration": { - "type": "string" - }, - "durationMs": { - "maximum": 300000, - "minimum": 1, - "type": "integer" - }, - "facing": { - "description": "camera_snap: front/back/both; camera_clip: front/back only.", - "enum": ["front", "back", "both"], - "type": "string" - }, - "fps": { - "exclusiveMinimum": 0, - "type": "number" - }, - "gatewayToken": { - "type": "string" - }, - "gatewayUrl": { - "type": "string" - }, - "includeAudio": { - "type": "boolean" - }, - "invokeCommand": { - "type": "string" - }, - "invokeParamsJson": { - "type": "string" - }, - "invokeTimeoutMs": { - "minimum": 1, - "type": "integer" - }, - "limit": { - "maximum": 20, - "minimum": 1, - "type": "integer" - }, - "locationTimeoutMs": { - "minimum": 1, - "type": "integer" - }, - "maxAgeMs": { - "minimum": 0, - "type": "integer" - }, - "maxWidth": { - "minimum": 1, - "type": "integer" - }, - "node": { - "description": "Node ID, name, or IP. Required for describe and node-targeted actions; use status to discover nodes.", - "type": "string" - }, - "notificationAction": { - "enum": ["open", "dismiss", "reply"], - "type": "string" - }, - "notificationKey": { - "type": "string" - }, - "notificationReplyText": { - "type": "string" - }, - "outPath": { - "type": "string" - }, - "priority": { - "enum": ["passive", "active", "timeSensitive"], - "type": "string" - }, - "quality": { - "maximum": 1, - "minimum": 0, - "type": "number" - }, - "requestId": { - "type": "string" - }, - "screenIndex": { - "minimum": 0, - "type": "integer" - }, - "sound": { - "type": "string" - }, - "timeoutMs": { - "minimum": 1, - "type": "integer" - }, - "title": { - "type": "string" - } - }, - "required": ["action"], - "type": "object" - }, - "name": "nodes", - "type": "function" - }, { "deferLoading": true, "description": "Gateway scheduler: reminders, delayed self-wakeups, loops, recurring work, event watchers. Never exec sleep/poll as timer.\n\nACTIONS: status | list [includeDisabled,limit?,offset?] (use nextOffset for the next page) | get jobId | add job | update jobId patch | remove jobId | run jobId (runMode \"force\"=now) | runs jobId = history | next_check in:\"30m\" (own paced run only) | wake text mode?:\"now\"|\"next-heartbeat\"(default) nudges a caller-owned lane (sessionKey/agentId to pick another).\n\nADD: {name?,schedule,payload,sessionTarget?,pacing?,trigger?,delivery?,enabled?}. Required: schedule+payload.\n\nSCHEDULE:\n- {kind:\"at\",at:\"ISO-8601\"} one-shot; no tz=UTC; auto-deletes after run.\n- {kind:\"every\",everyMs}.\n- {kind:\"cron\",expr,tz?:\"IANA\"}: expr is wall time in tz; never pre-convert to UTC; no tz=gateway host local. 18:00 Shanghai => {expr:\"0 18 * * *\",tz:\"Asia/Shanghai\"}.\n- {kind:\"stream\",command:[argv],mode?:\"line\"|\"match\",match?}: fires on supervised process output; needs cron.triggers.enabled.\n\nTARGET+PAYLOAD:\n- \"current\" (agentTurn default) = this conversation: run carries this chat's context, result lands here. Self-wakeup/\"continue later\"/loop = at|every + agentTurn + current.\n- \"isolated\" = fresh detached session (shows in `openclaw tasks`); standalone background work.\n- \"main\" = heartbeat lane; payload {kind:\"systemEvent\",text} (systemEvent default target).\n- \"session:\" = named session.\n- agentTurn {kind:\"agentTurn\",message,model?,thinking?,timeoutSeconds?}; timeoutSeconds 0=none.\n- script {kind:\"script\",script,timeoutSeconds?,toolBudget?}: main|isolated only; needs cron.triggers.enabled.\n\nPACED LOOP: recurring job + pacing{min?,max?} durations (\"15m\",\"4h\"; at least one). Inside its run, job calls next_check in:\"\" to set the next delay (clamped to bounds, measured from run end; failed runs keep normal backoff). Adaptive polling: tighten when active, back off when quiet.\n\nTRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call(\"exec\",{command:\"...\"}).\n\nDELIVERY {mode:\"none\"|\"announce\"|\"webhook\",channel?,to?,threadId?,bestEffort?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:\"none\". webhook posts finished-run event to URL in `to`.\n\nJob wakeMode (main jobs): \"now\"(default)|\"next-heartbeat\". Restricted cron-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.", @@ -1261,31 +1099,6 @@ "name": "cron", "type": "function" }, - { - "deferLoading": true, - "description": "Convert text to spoken audio (TTS) with the configured voice provider. Only explicit voice/speech/TTS intent or active TTS config; never ordinary text reply. Audio auto-delivered. After success follow reply instructions; no duplicate text/audio.", - "inputSchema": { - "properties": { - "channel": { - "description": "Channel id; output-format hint.", - "type": "string" - }, - "text": { - "description": "Text to speak.", - "type": "string" - }, - "timeoutMs": { - "description": "Provider timeout ms.", - "minimum": 1, - "type": "integer" - } - }, - "required": ["text"], - "type": "object" - }, - "name": "tts", - "type": "function" - }, { "deferLoading": true, "description": "Read gateway config + schema. Writes/restart: use openclaw tool.", @@ -1315,6 +1128,223 @@ "name": "gateway", "type": "function" }, + { + "deferLoading": true, + "description": "Paired nodes: status/list with active-computer presence; pass node to describe/control. Pairing lifecycle (pending/approve/reject), notify, camera_snap/camera_list/camera_clip (with audio), photos_latest, screen_snapshot, screen_record video, location_get, notifications_list + notifications_action (open/dismiss/reply), device_status/device_info/device_permissions/device_health, executable lookup (which + bins), generic invoke. Files: file_fetch.", + "inputSchema": { + "properties": { + "action": { + "enum": [ + "status", + "describe", + "pending", + "approve", + "reject", + "notify", + "camera_snap", + "camera_list", + "camera_clip", + "photos_latest", + "screen_record", + "screen_snapshot", + "location_get", + "notifications_list", + "notifications_action", + "device_status", + "device_info", + "device_permissions", + "device_health", + "which", + "invoke" + ], + "type": "string" + }, + "bins": { + "description": "which: executable names to resolve on the selected node.", + "items": { + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "type": "array" + }, + "body": { + "type": "string" + }, + "delayMs": { + "minimum": 0, + "type": "integer" + }, + "delivery": { + "enum": ["system", "overlay", "auto"], + "type": "string" + }, + "desiredAccuracy": { + "enum": ["coarse", "balanced", "precise"], + "type": "string" + }, + "deviceId": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "durationMs": { + "maximum": 300000, + "minimum": 1, + "type": "integer" + }, + "facing": { + "description": "camera_snap: front/back/both; camera_clip: front/back only.", + "enum": ["front", "back", "both"], + "type": "string" + }, + "fps": { + "exclusiveMinimum": 0, + "type": "number" + }, + "gatewayToken": { + "type": "string" + }, + "gatewayUrl": { + "type": "string" + }, + "includeAudio": { + "type": "boolean" + }, + "invokeCommand": { + "type": "string" + }, + "invokeParamsJson": { + "type": "string" + }, + "invokeTimeoutMs": { + "minimum": 1, + "type": "integer" + }, + "limit": { + "maximum": 20, + "minimum": 1, + "type": "integer" + }, + "locationTimeoutMs": { + "minimum": 1, + "type": "integer" + }, + "maxAgeMs": { + "minimum": 0, + "type": "integer" + }, + "maxWidth": { + "minimum": 1, + "type": "integer" + }, + "node": { + "description": "Node ID, name, or IP. Required for describe and node-targeted actions; use status to discover nodes.", + "type": "string" + }, + "notificationAction": { + "enum": ["open", "dismiss", "reply"], + "type": "string" + }, + "notificationKey": { + "type": "string" + }, + "notificationReplyText": { + "type": "string" + }, + "outPath": { + "type": "string" + }, + "priority": { + "enum": ["passive", "active", "timeSensitive"], + "type": "string" + }, + "quality": { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + "requestId": { + "type": "string" + }, + "screenIndex": { + "minimum": 0, + "type": "integer" + }, + "sound": { + "type": "string" + }, + "timeoutMs": { + "minimum": 1, + "type": "integer" + }, + "title": { + "type": "string" + } + }, + "required": ["action"], + "type": "object" + }, + "name": "nodes", + "type": "function" + }, + { + "deferLoading": true, + "description": "Show visible-session model/usage/time/cost/tasks. `sessionKey=\"current\"` for current; UI labels are not keys. `model` overrides; `model=default` resets. Use for active model/session questions.", + "inputSchema": { + "properties": { + "changesSince": { + "minimum": 0, + "type": "integer" + }, + "model": { + "type": "string" + }, + "sessionKey": { + "type": "string" + } + }, + "type": "object" + }, + "name": "session_status", + "type": "function" + }, + { + "deferLoading": true, + "description": "Read sanitized visible-session history. Before reply/debug/resume. Supports limit, offset, search-result sessionId/messageId anchors, and tool messages.", + "inputSchema": { + "properties": { + "includeTools": { + "type": "boolean" + }, + "limit": { + "minimum": 1, + "type": "integer" + }, + "messageId": { + "minLength": 1, + "type": "string" + }, + "offset": { + "minimum": 0, + "type": "integer" + }, + "sessionId": { + "minLength": 1, + "type": "string" + }, + "sessionKey": { + "type": "string" + } + }, + "required": ["sessionKey"], + "type": "object" + }, + "name": "sessions_history", + "type": "function" + }, { "deferLoading": true, "description": "List visible sessions; filter kind/label/agentId/search/activity/archive. Preview recent messages inline via includeLastMessage/messageLimit; includeDerivedTitles adds derived titles. Use before history/send target selection.", @@ -1366,40 +1396,6 @@ "name": "sessions_list", "type": "function" }, - { - "deferLoading": true, - "description": "Read sanitized visible-session history. Before reply/debug/resume. Supports limit, offset, search-result sessionId/messageId anchors, and tool messages.", - "inputSchema": { - "properties": { - "includeTools": { - "type": "boolean" - }, - "limit": { - "minimum": 1, - "type": "integer" - }, - "messageId": { - "minLength": 1, - "type": "string" - }, - "offset": { - "minimum": 0, - "type": "integer" - }, - "sessionId": { - "minLength": 1, - "type": "string" - }, - "sessionKey": { - "type": "string" - } - }, - "required": ["sessionKey"], - "type": "object" - }, - "name": "sessions_history", - "type": "function" - }, { "deferLoading": true, "description": "Search your own past sessions for matching user and assistant text. Follow up with sessions_history using a returned sessionKey, sessionId, and messageId for neighboring context.", @@ -1484,23 +1480,54 @@ }, { "deferLoading": true, - "description": "Show visible-session model/usage/time/cost/tasks. `sessionKey=\"current\"` for current; UI labels are not keys. `model` overrides; `model=default` resets. Use for active model/session questions.", + "description": "Convert text to spoken audio (TTS) with the configured voice provider. Only explicit voice/speech/TTS intent or active TTS config; never ordinary text reply. Audio auto-delivered. After success follow reply instructions; no duplicate text/audio.", "inputSchema": { "properties": { - "changesSince": { - "minimum": 0, - "type": "integer" - }, - "model": { + "channel": { + "description": "Channel id; output-format hint.", "type": "string" }, - "sessionKey": { + "text": { + "description": "Text to speak.", + "type": "string" + }, + "timeoutMs": { + "description": "Provider timeout ms.", + "minimum": 1, + "type": "integer" + } + }, + "required": ["text"], + "type": "object" + }, + "name": "tts", + "type": "function" + }, + { + "deferLoading": true, + "description": "Fetch URL; extract readable markdown/text. Lightweight; no browser automation.", + "inputSchema": { + "properties": { + "extractMode": { + "default": "markdown", + "description": "Extract as markdown/text.", + "enum": ["markdown", "text"], + "type": "string" + }, + "maxChars": { + "description": "Max chars returned; truncates.", + "minimum": 100, + "type": "integer" + }, + "url": { + "description": "HTTP(S) URL.", "type": "string" } }, + "required": ["url"], "type": "object" }, - "name": "session_status", + "name": "web_fetch", "type": "function" }, { @@ -1570,33 +1597,6 @@ }, "name": "web_search", "type": "function" - }, - { - "deferLoading": true, - "description": "Fetch URL; extract readable markdown/text. Lightweight; no browser automation.", - "inputSchema": { - "properties": { - "extractMode": { - "default": "markdown", - "description": "Extract as markdown/text.", - "enum": ["markdown", "text"], - "type": "string" - }, - "maxChars": { - "description": "Max chars returned; truncates.", - "minimum": 100, - "type": "integer" - }, - "url": { - "description": "HTTP(S) URL.", - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "web_fetch", - "type": "function" } ], "type": "namespace" diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.telegram-direct.json b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.telegram-direct.json index 3acdb5e5be93..5df0d27dd263 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.telegram-direct.json +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/codex-dynamic-tools.telegram-direct.json @@ -1,4 +1,15 @@ [ + { + "description": "List configured agent ids with name/model/runtime metadata, allowed as `sessions_spawn(runtime:\"subagent\")` targets.", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "required": [], + "type": "object" + }, + "name": "agents_list", + "type": "function" + }, { "description": "Send/manage channel messages. Supports actions: send.", "inputSchema": { @@ -126,17 +137,6 @@ "name": "message", "type": "function" }, - { - "description": "List configured agent ids with name/model/runtime metadata, allowed as `sessions_spawn(runtime:\"subagent\")` targets.", - "inputSchema": { - "additionalProperties": false, - "properties": {}, - "required": [], - "type": "object" - }, - "name": "agents_list", - "type": "function" - }, { "description": "Spawn clean child; default `runtime=\"subagent\"`. `mode=\"run\"` one-shot background. `agentId` targets a configured agent (see agents_list); `model` overrides its model; `cleanup` delete|keep hidden child session; `sandbox` inherit|require. `visible=true`: persistent dashboard session; subagent only; omit `mode` (no `mode=\"run\"`), `thread`, `thinking`, `lightContext`, `attachments`, `attachAs`; inherited tool allow/denylist blocks it at spawn with no config override; may check out a git worktree via `worktree`/`worktreeName`/`worktreeBaseRef`. Session listing/addressing obeys `tools.sessions.visibility` (`tree` default: current + own spawn subtree). Inherits parent workspace. Native task arrives as first `[Subagent Task]`. Native transcript needed: `context=\"fork\"`; else omit/isolated. Use fresh child for sidecar/parallel batch reads, multi-step search, data collection; avoid quick lookup/single read unless policy prefers. After spawn, do non-overlap work while run result returns.", "inputSchema": { @@ -259,168 +259,6 @@ "description": "", "name": "openclaw", "tools": [ - { - "deferLoading": true, - "description": "Paired nodes: status/list with active-computer presence; pass node to describe/control. Pairing lifecycle (pending/approve/reject), notify, camera_snap/camera_list/camera_clip (with audio), photos_latest, screen_snapshot, screen_record video, location_get, notifications_list + notifications_action (open/dismiss/reply), device_status/device_info/device_permissions/device_health, executable lookup (which + bins), generic invoke. Files: file_fetch.", - "inputSchema": { - "properties": { - "action": { - "enum": [ - "status", - "describe", - "pending", - "approve", - "reject", - "notify", - "camera_snap", - "camera_list", - "camera_clip", - "photos_latest", - "screen_record", - "screen_snapshot", - "location_get", - "notifications_list", - "notifications_action", - "device_status", - "device_info", - "device_permissions", - "device_health", - "which", - "invoke" - ], - "type": "string" - }, - "bins": { - "description": "which: executable names to resolve on the selected node.", - "items": { - "minLength": 1, - "type": "string" - }, - "maxItems": 64, - "minItems": 1, - "type": "array" - }, - "body": { - "type": "string" - }, - "delayMs": { - "minimum": 0, - "type": "integer" - }, - "delivery": { - "enum": ["system", "overlay", "auto"], - "type": "string" - }, - "desiredAccuracy": { - "enum": ["coarse", "balanced", "precise"], - "type": "string" - }, - "deviceId": { - "type": "string" - }, - "duration": { - "type": "string" - }, - "durationMs": { - "maximum": 300000, - "minimum": 1, - "type": "integer" - }, - "facing": { - "description": "camera_snap: front/back/both; camera_clip: front/back only.", - "enum": ["front", "back", "both"], - "type": "string" - }, - "fps": { - "exclusiveMinimum": 0, - "type": "number" - }, - "gatewayToken": { - "type": "string" - }, - "gatewayUrl": { - "type": "string" - }, - "includeAudio": { - "type": "boolean" - }, - "invokeCommand": { - "type": "string" - }, - "invokeParamsJson": { - "type": "string" - }, - "invokeTimeoutMs": { - "minimum": 1, - "type": "integer" - }, - "limit": { - "maximum": 20, - "minimum": 1, - "type": "integer" - }, - "locationTimeoutMs": { - "minimum": 1, - "type": "integer" - }, - "maxAgeMs": { - "minimum": 0, - "type": "integer" - }, - "maxWidth": { - "minimum": 1, - "type": "integer" - }, - "node": { - "description": "Node ID, name, or IP. Required for describe and node-targeted actions; use status to discover nodes.", - "type": "string" - }, - "notificationAction": { - "enum": ["open", "dismiss", "reply"], - "type": "string" - }, - "notificationKey": { - "type": "string" - }, - "notificationReplyText": { - "type": "string" - }, - "outPath": { - "type": "string" - }, - "priority": { - "enum": ["passive", "active", "timeSensitive"], - "type": "string" - }, - "quality": { - "maximum": 1, - "minimum": 0, - "type": "number" - }, - "requestId": { - "type": "string" - }, - "screenIndex": { - "minimum": 0, - "type": "integer" - }, - "sound": { - "type": "string" - }, - "timeoutMs": { - "minimum": 1, - "type": "integer" - }, - "title": { - "type": "string" - } - }, - "required": ["action"], - "type": "object" - }, - "name": "nodes", - "type": "function" - }, { "deferLoading": true, "description": "Gateway scheduler: reminders, delayed self-wakeups, loops, recurring work, event watchers. Never exec sleep/poll as timer.\n\nACTIONS: status | list [includeDisabled,limit?,offset?] (use nextOffset for the next page) | get jobId | add job | update jobId patch | remove jobId | run jobId (runMode \"force\"=now) | runs jobId = history | next_check in:\"30m\" (own paced run only) | wake text mode?:\"now\"|\"next-heartbeat\"(default) nudges a caller-owned lane (sessionKey/agentId to pick another).\n\nADD: {name?,schedule,payload,sessionTarget?,pacing?,trigger?,delivery?,enabled?}. Required: schedule+payload.\n\nSCHEDULE:\n- {kind:\"at\",at:\"ISO-8601\"} one-shot; no tz=UTC; auto-deletes after run.\n- {kind:\"every\",everyMs}.\n- {kind:\"cron\",expr,tz?:\"IANA\"}: expr is wall time in tz; never pre-convert to UTC; no tz=gateway host local. 18:00 Shanghai => {expr:\"0 18 * * *\",tz:\"Asia/Shanghai\"}.\n- {kind:\"stream\",command:[argv],mode?:\"line\"|\"match\",match?}: fires on supervised process output; needs cron.triggers.enabled.\n\nTARGET+PAYLOAD:\n- \"current\" (agentTurn default) = this conversation: run carries this chat's context, result lands here. Self-wakeup/\"continue later\"/loop = at|every + agentTurn + current.\n- \"isolated\" = fresh detached session (shows in `openclaw tasks`); standalone background work.\n- \"main\" = heartbeat lane; payload {kind:\"systemEvent\",text} (systemEvent default target).\n- \"session:\" = named session.\n- agentTurn {kind:\"agentTurn\",message,model?,thinking?,timeoutSeconds?}; timeoutSeconds 0=none.\n- script {kind:\"script\",script,timeoutSeconds?,toolBudget?}: main|isolated only; needs cron.triggers.enabled.\n\nPACED LOOP: recurring job + pacing{min?,max?} durations (\"15m\",\"4h\"; at least one). Inside its run, job calls next_check in:\"\" to set the next delay (clamped to bounds, measured from run end; failed runs keep normal backoff). Adaptive polling: tighten when active, back off when quiet.\n\nTRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call(\"exec\",{command:\"...\"}).\n\nDELIVERY {mode:\"none\"|\"announce\"|\"webhook\",channel?,to?,threadId?,bestEffort?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:\"none\". webhook posts finished-run event to URL in `to`.\n\nJob wakeMode (main jobs): \"now\"(default)|\"next-heartbeat\". Restricted cron-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.", @@ -1261,31 +1099,6 @@ "name": "cron", "type": "function" }, - { - "deferLoading": true, - "description": "Convert text to spoken audio (TTS) with the configured voice provider. Only explicit voice/speech/TTS intent or active TTS config; never ordinary text reply. Audio auto-delivered. After success follow reply instructions; no duplicate text/audio.", - "inputSchema": { - "properties": { - "channel": { - "description": "Channel id; output-format hint.", - "type": "string" - }, - "text": { - "description": "Text to speak.", - "type": "string" - }, - "timeoutMs": { - "description": "Provider timeout ms.", - "minimum": 1, - "type": "integer" - } - }, - "required": ["text"], - "type": "object" - }, - "name": "tts", - "type": "function" - }, { "deferLoading": true, "description": "Read gateway config + schema. Writes/restart: use openclaw tool.", @@ -1315,6 +1128,223 @@ "name": "gateway", "type": "function" }, + { + "deferLoading": true, + "description": "Paired nodes: status/list with active-computer presence; pass node to describe/control. Pairing lifecycle (pending/approve/reject), notify, camera_snap/camera_list/camera_clip (with audio), photos_latest, screen_snapshot, screen_record video, location_get, notifications_list + notifications_action (open/dismiss/reply), device_status/device_info/device_permissions/device_health, executable lookup (which + bins), generic invoke. Files: file_fetch.", + "inputSchema": { + "properties": { + "action": { + "enum": [ + "status", + "describe", + "pending", + "approve", + "reject", + "notify", + "camera_snap", + "camera_list", + "camera_clip", + "photos_latest", + "screen_record", + "screen_snapshot", + "location_get", + "notifications_list", + "notifications_action", + "device_status", + "device_info", + "device_permissions", + "device_health", + "which", + "invoke" + ], + "type": "string" + }, + "bins": { + "description": "which: executable names to resolve on the selected node.", + "items": { + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "type": "array" + }, + "body": { + "type": "string" + }, + "delayMs": { + "minimum": 0, + "type": "integer" + }, + "delivery": { + "enum": ["system", "overlay", "auto"], + "type": "string" + }, + "desiredAccuracy": { + "enum": ["coarse", "balanced", "precise"], + "type": "string" + }, + "deviceId": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "durationMs": { + "maximum": 300000, + "minimum": 1, + "type": "integer" + }, + "facing": { + "description": "camera_snap: front/back/both; camera_clip: front/back only.", + "enum": ["front", "back", "both"], + "type": "string" + }, + "fps": { + "exclusiveMinimum": 0, + "type": "number" + }, + "gatewayToken": { + "type": "string" + }, + "gatewayUrl": { + "type": "string" + }, + "includeAudio": { + "type": "boolean" + }, + "invokeCommand": { + "type": "string" + }, + "invokeParamsJson": { + "type": "string" + }, + "invokeTimeoutMs": { + "minimum": 1, + "type": "integer" + }, + "limit": { + "maximum": 20, + "minimum": 1, + "type": "integer" + }, + "locationTimeoutMs": { + "minimum": 1, + "type": "integer" + }, + "maxAgeMs": { + "minimum": 0, + "type": "integer" + }, + "maxWidth": { + "minimum": 1, + "type": "integer" + }, + "node": { + "description": "Node ID, name, or IP. Required for describe and node-targeted actions; use status to discover nodes.", + "type": "string" + }, + "notificationAction": { + "enum": ["open", "dismiss", "reply"], + "type": "string" + }, + "notificationKey": { + "type": "string" + }, + "notificationReplyText": { + "type": "string" + }, + "outPath": { + "type": "string" + }, + "priority": { + "enum": ["passive", "active", "timeSensitive"], + "type": "string" + }, + "quality": { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + "requestId": { + "type": "string" + }, + "screenIndex": { + "minimum": 0, + "type": "integer" + }, + "sound": { + "type": "string" + }, + "timeoutMs": { + "minimum": 1, + "type": "integer" + }, + "title": { + "type": "string" + } + }, + "required": ["action"], + "type": "object" + }, + "name": "nodes", + "type": "function" + }, + { + "deferLoading": true, + "description": "Show visible-session model/usage/time/cost/tasks. `sessionKey=\"current\"` for current; UI labels are not keys. `model` overrides; `model=default` resets. Use for active model/session questions.", + "inputSchema": { + "properties": { + "changesSince": { + "minimum": 0, + "type": "integer" + }, + "model": { + "type": "string" + }, + "sessionKey": { + "type": "string" + } + }, + "type": "object" + }, + "name": "session_status", + "type": "function" + }, + { + "deferLoading": true, + "description": "Read sanitized visible-session history. Before reply/debug/resume. Supports limit, offset, search-result sessionId/messageId anchors, and tool messages.", + "inputSchema": { + "properties": { + "includeTools": { + "type": "boolean" + }, + "limit": { + "minimum": 1, + "type": "integer" + }, + "messageId": { + "minLength": 1, + "type": "string" + }, + "offset": { + "minimum": 0, + "type": "integer" + }, + "sessionId": { + "minLength": 1, + "type": "string" + }, + "sessionKey": { + "type": "string" + } + }, + "required": ["sessionKey"], + "type": "object" + }, + "name": "sessions_history", + "type": "function" + }, { "deferLoading": true, "description": "List visible sessions; filter kind/label/agentId/search/activity/archive. Preview recent messages inline via includeLastMessage/messageLimit; includeDerivedTitles adds derived titles. Use before history/send target selection.", @@ -1366,40 +1396,6 @@ "name": "sessions_list", "type": "function" }, - { - "deferLoading": true, - "description": "Read sanitized visible-session history. Before reply/debug/resume. Supports limit, offset, search-result sessionId/messageId anchors, and tool messages.", - "inputSchema": { - "properties": { - "includeTools": { - "type": "boolean" - }, - "limit": { - "minimum": 1, - "type": "integer" - }, - "messageId": { - "minLength": 1, - "type": "string" - }, - "offset": { - "minimum": 0, - "type": "integer" - }, - "sessionId": { - "minLength": 1, - "type": "string" - }, - "sessionKey": { - "type": "string" - } - }, - "required": ["sessionKey"], - "type": "object" - }, - "name": "sessions_history", - "type": "function" - }, { "deferLoading": true, "description": "Search your own past sessions for matching user and assistant text. Follow up with sessions_history using a returned sessionKey, sessionId, and messageId for neighboring context.", @@ -1484,23 +1480,54 @@ }, { "deferLoading": true, - "description": "Show visible-session model/usage/time/cost/tasks. `sessionKey=\"current\"` for current; UI labels are not keys. `model` overrides; `model=default` resets. Use for active model/session questions.", + "description": "Convert text to spoken audio (TTS) with the configured voice provider. Only explicit voice/speech/TTS intent or active TTS config; never ordinary text reply. Audio auto-delivered. After success follow reply instructions; no duplicate text/audio.", "inputSchema": { "properties": { - "changesSince": { - "minimum": 0, - "type": "integer" - }, - "model": { + "channel": { + "description": "Channel id; output-format hint.", "type": "string" }, - "sessionKey": { + "text": { + "description": "Text to speak.", + "type": "string" + }, + "timeoutMs": { + "description": "Provider timeout ms.", + "minimum": 1, + "type": "integer" + } + }, + "required": ["text"], + "type": "object" + }, + "name": "tts", + "type": "function" + }, + { + "deferLoading": true, + "description": "Fetch URL; extract readable markdown/text. Lightweight; no browser automation.", + "inputSchema": { + "properties": { + "extractMode": { + "default": "markdown", + "description": "Extract as markdown/text.", + "enum": ["markdown", "text"], + "type": "string" + }, + "maxChars": { + "description": "Max chars returned; truncates.", + "minimum": 100, + "type": "integer" + }, + "url": { + "description": "HTTP(S) URL.", "type": "string" } }, + "required": ["url"], "type": "object" }, - "name": "session_status", + "name": "web_fetch", "type": "function" }, { @@ -1570,33 +1597,6 @@ }, "name": "web_search", "type": "function" - }, - { - "deferLoading": true, - "description": "Fetch URL; extract readable markdown/text. Lightweight; no browser automation.", - "inputSchema": { - "properties": { - "extractMode": { - "default": "markdown", - "description": "Extract as markdown/text.", - "enum": ["markdown", "text"], - "type": "string" - }, - "maxChars": { - "description": "Max chars returned; truncates.", - "minimum": 100, - "type": "integer" - }, - "url": { - "description": "HTTP(S) URL.", - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "web_fetch", - "type": "function" } ], "type": "namespace" diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/discord-group-codex-message-tool.md b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/discord-group-codex-message-tool.md index 5c1af6bb4ea0..3516b5c54c13 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/discord-group-codex-message-tool.md +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/discord-group-codex-message-tool.md @@ -75,21 +75,21 @@ "cwd": "/tmp/openclaw-happy-path/workspace", "developerInstructions": "", "dynamicTools": [ - "message", "agents_list", + "message", "sessions_spawn", - "nodes", "cron", - "tts", "gateway", - "sessions_list", + "nodes", + "session_status", "sessions_history", + "sessions_list", "sessions_search", "sessions_send", "subagents", - "session_status", - "web_search", + "tts", "web_fetch", + "web_search", "sessions_yield" ], "experimentalRawEvents": true, @@ -521,21 +521,21 @@ Full JSON: `codex-dynamic-tools.discord-group.json` ```json [ - "message", "agents_list", + "message", "sessions_spawn", - "nodes", "cron", - "tts", "gateway", - "sessions_list", + "nodes", + "session_status", "sessions_history", + "sessions_list", "sessions_search", "sessions_send", "subagents", - "session_status", - "web_search", + "tts", "web_fetch", + "web_search", "sessions_yield" ] ``` diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md index acf69cb6dce6..e27d1558689a 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md @@ -75,21 +75,21 @@ "cwd": "/tmp/openclaw-happy-path/workspace", "developerInstructions": "", "dynamicTools": [ - "message", "agents_list", + "message", "sessions_spawn", - "nodes", "cron", - "tts", "gateway", - "sessions_list", + "nodes", + "session_status", "sessions_history", + "sessions_list", "sessions_search", "sessions_send", "subagents", - "session_status", - "web_search", + "tts", "web_fetch", + "web_search", "sessions_yield" ], "experimentalRawEvents": true, @@ -515,21 +515,21 @@ Full JSON: `codex-dynamic-tools.telegram-direct.json` ```json [ - "message", "agents_list", + "message", "sessions_spawn", - "nodes", "cron", - "tts", "gateway", - "sessions_list", + "nodes", + "session_status", "sessions_history", + "sessions_list", "sessions_search", "sessions_send", "subagents", - "session_status", - "web_search", + "tts", "web_fetch", + "web_search", "sessions_yield" ] ``` diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-heartbeat-codex-tool.md b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-heartbeat-codex-tool.md index afee37cd57cb..8d99e51ecce8 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-heartbeat-codex-tool.md +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-heartbeat-codex-tool.md @@ -75,21 +75,21 @@ "cwd": "/tmp/openclaw-happy-path/workspace", "developerInstructions": "", "dynamicTools": [ - "message", "agents_list", + "message", "sessions_spawn", - "nodes", "cron", - "tts", "gateway", - "sessions_list", + "nodes", + "session_status", "sessions_history", + "sessions_list", "sessions_search", "sessions_send", "subagents", - "session_status", - "web_search", + "tts", "web_fetch", + "web_search", "heartbeat_respond", "sessions_yield" ], @@ -516,21 +516,21 @@ Full JSON: `codex-dynamic-tools.heartbeat-turn.json` ```json [ - "message", "agents_list", + "message", "sessions_spawn", - "nodes", "cron", - "tts", "gateway", - "sessions_list", + "nodes", + "session_status", "sessions_history", + "sessions_list", "sessions_search", "sessions_send", "subagents", - "session_status", - "web_search", + "tts", "web_fetch", + "web_search", "heartbeat_respond", "sessions_yield" ]