From f53346944db01a57e0522d494c4e15f281201c7d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 6 Jul 2026 12:30:12 +0100 Subject: [PATCH] feat: correlate native search outcomes in audit history (#98704) * feat: correlate native search outcomes in audit history Metadata-only audit ledger for agent runs and tool actions in the shared state DB: stable event identity, closed action/status/error vocabularies, one-way-hashed tool-call ids, never-inferred terminal outcomes for native web-search (explicit completed/failed only; otherwise unknown), bounded retention, audit.list gateway RPC and openclaw audit CLI. Squashed from the 82-commit audit stack for replay onto current main. * feat(audit): add audit.enabled config gate (default on) The metadata-only audit ledger records by default: an audit trail enabled only after an incident cannot explain the incident, and the rows are strictly less sensitive than the transcripts every install already stores. audit.enabled=false stops new writes at the gateway subscription seam; audit.list and openclaw audit keep serving existing records until they expire. Documented in the configuration reference, protocol page, and CLI reference. * fix: repair full-matrix CI findings after rebase - break the dynamic-tools/dynamic-tool-execution import cycle by extracting resolveCodexToolAbortTerminalReason into a leaf module - restore main's session-worktree protocol exports lost in the index.ts auto-merge - register the audit event writer worker as a knip entry point - docs table formatting; subagent wait-cancellation test scoped to its audit intent (outcome + timing) and advanced past main's new lifecycle-timeout retry grace --- .../OpenClawProtocol/GatewayModels.swift | 138 ++++ config/knip.config.ts | 1 + docs/.generated/config-baseline.sha256 | 4 +- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/.i18n/glossary.zh-CN.json | 4 + docs/cli/audit.md | 93 +++ docs/cli/index.md | 3 +- docs/concepts/agent-loop.md | 5 + docs/docs.json | 1 + docs/docs_map.md | 12 + docs/gateway/configuration-reference.md | 25 + docs/gateway/protocol.md | 30 + extensions/acpx/src/runtime-turn.test.ts | 42 + extensions/acpx/src/runtime-turn.ts | 10 +- .../src/app-server/approval-bridge.test.ts | 147 ++++ .../codex/src/app-server/approval-bridge.ts | 111 ++- .../app-server/dynamic-tool-diagnostics.ts | 12 +- .../app-server/dynamic-tool-execution.test.ts | 182 ++++- .../src/app-server/dynamic-tool-execution.ts | 59 +- .../src/app-server/dynamic-tools.test.ts | 225 +++++- .../codex/src/app-server/dynamic-tools.ts | 82 +- .../src/app-server/event-projector.test.ts | 717 +++++++++++++++++ .../codex/src/app-server/event-projector.ts | 488 ++++++++++-- .../src/app-server/native-hook-relay.test.ts | 64 +- .../codex/src/app-server/native-hook-relay.ts | 44 + extensions/codex/src/app-server/protocol.ts | 2 + .../run-attempt.dynamic-tools.test.ts | 48 +- .../run-attempt.native-hook-relay.test.ts | 50 +- .../codex/src/app-server/run-attempt.test.ts | 103 +++ .../codex/src/app-server/run-attempt.ts | 90 ++- .../run-attempt.turn-watches.test.ts | 81 +- .../src/app-server/side-question.test.ts | 726 ++++++++++++++++- .../codex/src/app-server/side-question.ts | 199 ++++- .../app-server/tool-abort-terminal-reason.ts | 34 + packages/acp-core/src/runtime/types.ts | 13 + packages/gateway-protocol/src/index.ts | 13 + packages/gateway-protocol/src/schema.ts | 1 + .../gateway-protocol/src/schema/audit.test.ts | 43 + packages/gateway-protocol/src/schema/audit.ts | 91 +++ .../src/schema/protocol-schemas.ts | 6 +- packages/gateway-protocol/src/schema/types.ts | 5 + scripts/plugin-sdk-surface-report.mjs | 6 +- scripts/release-check.ts | 1 + .../manager.turn-results.test.ts | 8 +- src/acp/control-plane/manager.turn-stream.ts | 10 +- src/acp/control-plane/manager.turn-timeout.ts | 2 + src/acp/tool-status.ts | 20 + .../agent-command.live-model-switch.test.ts | 67 ++ src/agents/agent-command.ts | 53 +- src/agents/agent-run-terminal-outcome.test.ts | 44 +- src/agents/agent-run-terminal-outcome.ts | 33 +- .../agent-tools.before-tool-call.e2e.test.ts | 353 ++++++++ ...ols.before-tool-call.embedded-mode.test.ts | 2 + src/agents/agent-tools.before-tool-call.ts | 483 +++++++---- src/agents/cli-output.test.ts | 28 +- src/agents/cli-output.ts | 20 +- src/agents/cli-runner.reliability.test.ts | 14 +- src/agents/cli-runner.spawn.test.ts | 583 +++++++++++++- src/agents/cli-runner/claude-live-session.ts | 253 +++--- .../execute.supervisor-capture.test.ts | 752 +++++++++++++++++- src/agents/cli-runner/execute.ts | 433 +++++++++- ...ttempt-execution.error-propagation.test.ts | 518 +++++++++++- .../command/attempt-execution.runtime.ts | 2 + src/agents/command/attempt-execution.ts | 325 +++++++- src/agents/embedded-agent-runner/compact.ts | 3 + .../embedded-agent-runner/run/attempt.ts | 4 + src/agents/harness/native-hook-relay.test.ts | 208 +++++ src/agents/harness/native-hook-relay.ts | 100 ++- src/agents/model-fallback.test.ts | 35 + src/agents/run-termination.test.ts | 93 +++ src/agents/run-termination.ts | 69 +- src/agents/subagent-announce-output.test.ts | 25 +- src/agents/subagent-announce-output.ts | 6 +- src/agents/subagent-registry.test.ts | 163 ++-- src/agents/subagent-registry.ts | 20 +- src/agents/tool-result-error.test.ts | 56 ++ src/agents/tool-result-error.ts | 133 +++- src/agents/tool-schema-quarantine.test.ts | 44 +- src/agents/tool-schema-quarantine.ts | 2 + src/audit/agent-event-audit.ts | 439 ++++++++++ src/audit/audit-config.test.ts | 18 + src/audit/audit-config.ts | 11 + src/audit/audit-event-store.ts | 205 +++++ src/audit/audit-event-types.ts | 71 ++ src/audit/audit-event-writer.test.ts | 58 ++ src/audit/audit-event-writer.ts | 192 +++++ src/audit/audit-event-writer.worker.ts | 52 ++ src/audit/audit-events.test.ts | 648 +++++++++++++++ src/auto-reply/reply/acp-projector.ts | 6 +- .../reply/agent-lifecycle-terminal.ts | 17 +- .../reply/agent-runner-cli-dispatch.test.ts | 42 + .../reply/agent-runner-cli-dispatch.ts | 7 +- .../reply/agent-runner-execution.test.ts | 41 + .../reply/agent-runner-execution.ts | 13 +- src/auto-reply/reply/dispatch-acp.test.ts | 168 ++++ src/auto-reply/reply/dispatch-acp.ts | 123 ++- .../dispatch-from-config.acp-abort.test.ts | 1 + ...ispatch-from-config.reply-dispatch.test.ts | 1 + ...ispatch-from-config.shared.test-harness.ts | 2 + .../reply/dispatch-from-config.test.ts | 7 +- src/auto-reply/reply/followup-runner.ts | 12 +- src/cli/program/command-registry-core.ts | 5 + src/cli/program/config-guard.test.ts | 5 +- src/cli/program/config-guard.ts | 11 +- src/cli/program/core-command-descriptors.ts | 5 + src/cli/program/register.audit.ts | 51 ++ src/commands/agent.acp.test.ts | 5 + src/commands/agent.test.ts | 5 + src/commands/audit.test.ts | 43 + src/commands/audit.ts | 120 +++ src/config/types.base.ts | 9 + src/config/types.openclaw.ts | 10 +- src/config/zod-schema.ts | 6 + src/gateway/mcp-http.handlers.ts | 51 +- src/gateway/mcp-http.loopback-runtime.ts | 48 +- src/gateway/mcp-http.test.ts | 237 +++++- src/gateway/mcp-http.ts | 7 +- src/gateway/method-scopes.test.ts | 1 + src/gateway/methods/core-descriptors.ts | 1 + src/gateway/server-close.test.ts | 4 +- src/gateway/server-close.ts | 2 +- src/gateway/server-methods.ts | 8 + src/gateway/server-methods/agent-job.ts | 27 +- .../server-methods/agent-wait-dedupe.test.ts | 12 +- src/gateway/server-methods/audit.test.ts | 74 ++ src/gateway/server-methods/audit.ts | 93 +++ src/gateway/server-runtime-handles.ts | 4 +- .../server-runtime-subscriptions.test.ts | 48 +- src/gateway/server-runtime-subscriptions.ts | 26 +- .../server.chat.gateway-server-chat.test.ts | 2 +- src/gateway/session-lifecycle-state.test.ts | 10 + src/gateway/session-lifecycle-state.ts | 1 + src/gateway/tools-invoke-http.test.ts | 3 + src/infra/agent-events.test.ts | 62 ++ src/infra/agent-events.ts | 49 +- src/infra/diagnostic-events.ts | 85 ++ src/logging/diagnostic-stability.ts | 3 + src/plugin-sdk/agent-harness-runtime.ts | 9 + src/shared/agent-liveness.ts | 11 + src/state/openclaw-state-db.generated.d.ts | 21 + src/state/openclaw-state-schema.generated.ts | 38 + src/state/openclaw-state-schema.sql | 38 + src/tasks/task-registry.ts | 1 + test/release-check.test.ts | 2 + test/scripts/lint-suppressions.test.ts | 1 + tsdown.config.ts | 1 + 146 files changed, 11598 insertions(+), 770 deletions(-) create mode 100644 docs/cli/audit.md create mode 100644 extensions/acpx/src/runtime-turn.test.ts create mode 100644 extensions/codex/src/app-server/tool-abort-terminal-reason.ts create mode 100644 packages/gateway-protocol/src/schema/audit.test.ts create mode 100644 packages/gateway-protocol/src/schema/audit.ts create mode 100644 src/acp/tool-status.ts create mode 100644 src/agents/tool-result-error.test.ts create mode 100644 src/audit/agent-event-audit.ts create mode 100644 src/audit/audit-config.test.ts create mode 100644 src/audit/audit-config.ts create mode 100644 src/audit/audit-event-store.ts create mode 100644 src/audit/audit-event-types.ts create mode 100644 src/audit/audit-event-writer.test.ts create mode 100644 src/audit/audit-event-writer.ts create mode 100644 src/audit/audit-event-writer.worker.ts create mode 100644 src/audit/audit-events.test.ts create mode 100644 src/cli/program/register.audit.ts create mode 100644 src/commands/audit.test.ts create mode 100644 src/commands/audit.ts create mode 100644 src/gateway/server-methods/audit.test.ts create mode 100644 src/gateway/server-methods/audit.ts diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 74f1657249dd..29174e92cd59 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -3077,6 +3077,144 @@ public struct SessionsUsageParams: Codable, Sendable { } } +public struct AuditEvent: Codable, Sendable { + public let eventid: String + public let sequence: Int + public let sourcesequence: Int + public let occurredat: Int + public let kind: AnyCodable + public let action: AnyCodable + public let status: AnyCodable + public let errorcode: AnyCodable? + public let actor: [String: AnyCodable] + public let agentid: String + public let sessionkey: String? + public let sessionid: String? + public let runid: String + public let toolcallid: String? + public let toolname: String? + public let redaction: String + + public init( + eventid: String, + sequence: Int, + sourcesequence: Int, + occurredat: Int, + kind: AnyCodable, + action: AnyCodable, + status: AnyCodable, + errorcode: AnyCodable?, + actor: [String: AnyCodable], + agentid: String, + sessionkey: String?, + sessionid: String?, + runid: String, + toolcallid: String?, + toolname: String?, + redaction: String) + { + self.eventid = eventid + self.sequence = sequence + self.sourcesequence = sourcesequence + self.occurredat = occurredat + self.kind = kind + self.action = action + self.status = status + self.errorcode = errorcode + self.actor = actor + self.agentid = agentid + self.sessionkey = sessionkey + self.sessionid = sessionid + self.runid = runid + self.toolcallid = toolcallid + self.toolname = toolname + self.redaction = redaction + } + + private enum CodingKeys: String, CodingKey { + case eventid = "eventId" + case sequence + case sourcesequence = "sourceSequence" + case occurredat = "occurredAt" + case kind + case action + case status + case errorcode = "errorCode" + case actor + case agentid = "agentId" + case sessionkey = "sessionKey" + case sessionid = "sessionId" + case runid = "runId" + case toolcallid = "toolCallId" + case toolname = "toolName" + case redaction + } +} + +public struct AuditListParams: Codable, Sendable { + public let agentid: String? + public let sessionkey: String? + public let runid: String? + public let kind: AnyCodable? + public let status: AnyCodable? + public let after: Int? + public let before: Int? + public let limit: Int? + public let cursor: String? + + public init( + agentid: String? = nil, + sessionkey: String?, + runid: String?, + kind: AnyCodable?, + status: AnyCodable?, + after: Int?, + before: Int?, + limit: Int?, + cursor: String?) + { + self.agentid = agentid + self.sessionkey = sessionkey + self.runid = runid + self.kind = kind + self.status = status + self.after = after + self.before = before + self.limit = limit + self.cursor = cursor + } + + private enum CodingKeys: String, CodingKey { + case agentid = "agentId" + case sessionkey = "sessionKey" + case runid = "runId" + case kind + case status + case after + case before + case limit + case cursor + } +} + +public struct AuditListResult: Codable, Sendable { + public let events: [AuditEvent] + public let nextcursor: String? + + public init( + events: [AuditEvent], + nextcursor: String?) + { + self.events = events + self.nextcursor = nextcursor + } + + private enum CodingKeys: String, CodingKey { + case events + case nextcursor = "nextCursor" + } +} + public struct TaskSummary: Codable, Sendable { public let id: String public let kind: String? diff --git a/config/knip.config.ts b/config/knip.config.ts index 08015b1f6706..484e85beb63b 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -13,6 +13,7 @@ const rootEntries = [ "src/entry.ts!", "src/cli/daemon-cli.ts!", "src/agents/code-mode.worker.ts!", + "src/audit/audit-event-writer.worker.ts!", "src/agents/model-provider-auth.worker.ts!", "src/infra/kysely-node-sqlite.ts!", "src/infra/warning-filter.ts!", diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index 062d65c1489d..e0e9d790ed79 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -d144128f5cf945ddbe411ae5d87a65e3dbb29b8deb04f6bb69dba868486f5b70 config-baseline.json -5e06bdcca5f552ddbd496f6f413c090816a4bef47ea6610205d143653060c5ef config-baseline.core.json +e7110e0263c74c772c8fc9075e3dc8c6e857ca84fa237ce9e182037e86f24dfa config-baseline.json +66bb892f3350a6cbe0bdac55e36d8960dba621266c1d21903d005862ef8a06b4 config-baseline.core.json cb7503a2c1e489fa4d154dfa4bf4a9e2f1277233ab65fac71c1ba8dcb371a032 config-baseline.channel.json 77a337c9f4f2e43d18da97427905c573abd8857014033d359bfb36b2f15c2f20 config-baseline.plugin.json diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 0949bf40c755..74f2fe32dd08 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -0dcad4f6899080c7858b4bdd709e8c3443b85659bb03525b49967abb262ecd8b plugin-sdk-api-baseline.json -a67e0ed2b47d3fdb7e450837b6edb37e0796c3451bc2097be5d828a63b8e73d4 plugin-sdk-api-baseline.jsonl +57289c57bab5188013ab1895d26acabada49aff864b3207f1a3ea3e1d92c7791 plugin-sdk-api-baseline.json +445e5e72a321486831c2c936118302dbc2cde793f5e1dc5beab29100cae8150e plugin-sdk-api-baseline.jsonl diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index 69c9f84dcb01..f4e4bb8d6197 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -1239,6 +1239,10 @@ "source": "Agent config reference", "target": "Agent 配置参考" }, + { + "source": "Audit records", + "target": "审计记录" + }, { "source": "Background process", "target": "后台进程" diff --git a/docs/cli/audit.md b/docs/cli/audit.md new file mode 100644 index 000000000000..95aa10e64217 --- /dev/null +++ b/docs/cli/audit.md @@ -0,0 +1,93 @@ +--- +summary: "CLI reference for metadata-only agent run and tool action audit records" +read_when: + - You need to answer who ran an agent or tool, when it ran, and how it ended + - You need a bounded, redaction-safe activity export +title: "Audit records" +--- + +# `openclaw audit` + +Query the Gateway's metadata-only audit ledger for agent runs and tool actions. + +Recording is on by default; set [`audit.enabled: false`](/gateway/configuration-reference#audit) +to stop new writes. Existing records stay queryable until they expire (30 days). +The ledger is separate from conversation transcripts: it records identity, +ordering, provenance, action, status, and normalized error codes, but never +stores prompts, messages, tool arguments, tool results, command output, or raw +error text. + +The Gateway writes records to the shared OpenClaw state database through a +bounded background writer. Queries never return records older than 30 days, +and the ledger is capped at 100,000 rows. Expired rows are deleted during +Gateway startup, hourly maintenance, and later writes. + +```bash +openclaw audit +openclaw audit --agent main --status failed +openclaw audit --session "agent:main:main" --after 2026-07-01T00:00:00Z +openclaw audit --run 8c69f72e-8b11-4c54-98d5-1a3dd67450c3 +openclaw audit --kind tool_action --limit 50 --json +``` + +## Filters + +- `--agent `: exact agent id +- `--session `: exact session key +- `--run `: exact run id +- `--kind `: `agent_run` or `tool_action` +- `--status `: `started`, `succeeded`, `failed`, `cancelled`, + `timed_out`, `blocked`, or `unknown` +- `--after ` / `--before `: inclusive ISO timestamp or + Unix milliseconds +- `--limit `: page size from 1 to 500; default `100` +- `--cursor `: continue a previous newest-first query +- `--json`: print the bounded page as JSON + +Text output shows time, kind, status, agent, run, and action. Tool actions also +show the tool name. JSON output is a safe bounded export of the same metadata +and includes `nextCursor` when another page exists. Pass that value to +`--cursor` to continue without reordering records that arrive during paging. + +## Recorded events + +The Gateway projects existing agent event streams into four actions: + +- `agent.run.started` +- `agent.run.finished` +- `tool.action.started` +- `tool.action.finished` + +Every record has a stable event id, a monotonically increasing ledger sequence, +the original run event sequence, lifecycle timestamp when the runtime provides +one (otherwise observation time), agent/run provenance, actor, and a +`redaction: "metadata_only"` marker. Terminal records distinguish success, +failure, cancellation, timeout, and policy blocks with closed status and error +codes. `unknown` is an explicit non-success result when an upstream runtime +does not expose an authoritative terminal outcome. Tool call ids are exported +only as stable one-way fingerprints. Tool names must match the compact +model-facing name contract; other values become `unknown`. Session ids, session +keys, run ids, and retained tool names are operator metadata; protect exports +as operational records. + +The audit ledger does not replace transcripts, task history, cron run history, +or logs. It provides a small cross-run index for operator questions without +copying conversation content into another store. + +## Gateway RPC + +`audit.list` requires `operator.read` and accepts the same filters. Example: + +```bash +openclaw gateway call audit.list --params '{"agentId":"main","status":"failed","limit":50}' +``` + +The result is `{ "events": AuditEvent[], "nextCursor"?: string }`. Results are +newest first and limited to 500 records per request. + +## Related + +- [Gateway protocol](/gateway/protocol#audit-ledger-rpc) +- [Sessions](/cli/sessions) +- [Tasks](/cli/tasks) +- [Cron jobs](/automation/cron-jobs) diff --git a/docs/cli/index.md b/docs/cli/index.md index cc10538a804e..e0e428ef31fc 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -24,7 +24,7 @@ Setup commands by intent: | Setup and onboarding | [`crestodian`](/cli/crestodian) · [`setup`](/cli/setup) · [`onboard`](/cli/onboard) · [`configure`](/cli/configure) · [`config`](/cli/config) · [`completion`](/cli/completion) · [`doctor`](/cli/doctor) · [`dashboard`](/cli/dashboard) | | Reset, backup, and migration | [`backup`](/cli/backup) · [`migrate`](/cli/migrate) · [`reset`](/cli/reset) · [`uninstall`](/cli/uninstall) · [`update`](/cli/update) | | Messaging and agents | [`message`](/cli/message) · [`agent`](/cli/agent) · [`agents`](/cli/agents) · [`attach`](/cli/attach) · [`acp`](/cli/acp) · [`mcp`](/cli/mcp) | -| Health and sessions | [`status`](/cli/status) · [`health`](/cli/health) · [`sessions`](/cli/sessions) | +| Health and sessions | [`status`](/cli/status) · [`health`](/cli/health) · [`sessions`](/cli/sessions) · [`audit`](/cli/audit) | | Gateway and logs | [`gateway`](/cli/gateway) · [`logs`](/cli/logs) · [`system`](/cli/system) | | Models and inference | [`models`](/cli/models) · [`infer`](/cli/infer) · `capability` (alias for [`infer`](/cli/infer)) · [`memory`](/cli/memory) · [`commitments`](/cli/commitments) · [`wiki`](/cli/wiki) | | Network and nodes | [`directory`](/cli/directory) · [`nodes`](/cli/nodes) · [`devices`](/cli/devices) · [`node`](/cli/node) | @@ -238,6 +238,7 @@ openclaw [--dev] [--profile ] health sessions cleanup + audit tasks list audit diff --git a/docs/concepts/agent-loop.md b/docs/concepts/agent-loop.md index ec424de807b3..870b2906bfb4 100644 --- a/docs/concepts/agent-loop.md +++ b/docs/concepts/agent-loop.md @@ -116,6 +116,11 @@ Auto-compaction emits `compaction` stream events and can trigger a retry. On ret - `assistant`: streamed deltas from the agent runtime. - `tool`: streamed tool events from the agent runtime. +The Gateway projects lifecycle and tool start/terminal events into the bounded, +metadata-only [audit ledger](/cli/audit). This projection records provenance and +result codes without copying prompts, messages, tool arguments, tool results, +or raw errors out of the transcript/runtime path. + ## Chat channel handling Assistant deltas buffer into chat `delta` messages. A chat `final` is emitted on **lifecycle end/error**. diff --git a/docs/docs.json b/docs/docs.json index 94ea79b5fdc5..b46e7efe4563 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1745,6 +1745,7 @@ "pages": [ "cli/agent", "cli/agents", + "cli/audit", "cli/hooks", "cli/infer", "cli/memory", diff --git a/docs/docs_map.md b/docs/docs_map.md index 077ed83f1ec1..4748c8e185ce 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -1241,6 +1241,16 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - Route: /cli/attach - Headings: none +## cli/audit.md + +- Route: /cli/audit +- Headings: + - H1: openclaw audit + - H2: Filters + - H2: Recorded events + - H2: Gateway RPC + - H2: Related + ## cli/backup.md - Route: /cli/backup @@ -3223,6 +3233,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H3: Secret providers config - H2: Auth storage - H3: auth.cooldowns + - H2: Audit - H2: Logging - H2: Diagnostics - H2: Update @@ -3587,6 +3598,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: RPC method families - H3: Common event families - H3: Node helper methods + - H2: Audit ledger RPC - H2: Task ledger RPCs - H2: Operator helper methods - H3: models.list views diff --git a/docs/gateway/configuration-reference.md b/docs/gateway/configuration-reference.md index dc04c62de702..ec6ff8e700d3 100644 --- a/docs/gateway/configuration-reference.md +++ b/docs/gateway/configuration-reference.md @@ -1062,6 +1062,31 @@ Notes: --- +## Audit + +```json5 +{ + audit: { + enabled: true, + }, +} +``` + +The Gateway records **metadata-only** audit events for agent runs and tool +actions into the shared state database: identity, timing, tool names, and +terminal outcomes — never prompts, messages, tool arguments, results, or raw +error text. Records expire after 30 days and the ledger is capped at 100,000 +rows. Query them with [`openclaw audit`](/cli/audit) or the +[`audit.list`](/gateway/protocol#audit-ledger-rpc) Gateway RPC. + +- `enabled`: record new audit events (default: `true`). The ledger is on by + default because an audit trail enabled only after an incident cannot explain + the incident. Setting `false` stops new writes immediately; existing records + stay readable until they expire. Turning it back on resumes recording from + that point — the gap is not backfilled. + +--- + ## Logging ```json5 diff --git a/docs/gateway/protocol.md b/docs/gateway/protocol.md index d159cdef71f5..70d970cf8182 100644 --- a/docs/gateway/protocol.md +++ b/docs/gateway/protocol.md @@ -426,6 +426,7 @@ methods. Treat this as feature discovery, not a full enumeration of - `agents.list` returns configured agent entries, including effective model and runtime metadata. - `agents.create`, `agents.update`, and `agents.delete` manage agent records and workspace wiring. - `agents.files.list`, `agents.files.get`, and `agents.files.set` manage the bootstrap workspace files exposed for an agent. + - `audit.list` returns a bounded metadata-only ledger of agent run and tool action events. - `tasks.list`, `tasks.get`, and `tasks.cancel` expose the gateway task ledger to SDK and operator clients. See [Task ledger RPCs](#task-ledger-rpcs) below. - `artifacts.list`, `artifacts.get`, and `artifacts.download` expose transcript-derived artifact summaries and downloads for an explicit `sessionKey`, `runId`, or `taskId` scope. Run and task queries resolve the owning session server-side and only return transcript media with matching provenance; unsafe or local URL sources return unsupported downloads instead of fetching server-side. - `environments.list` and `environments.status` expose read-only gateway-local and node environment discovery for SDK clients. @@ -525,6 +526,35 @@ methods. Treat this as feature discovery, not a full enumeration of Nodes may call `skills.bins` to fetch the current list of skill executables for auto-allow checks. +## Audit ledger RPC + +`audit.list` gives operator clients a stable newest-first view of agent run and +tool action metadata. It requires `operator.read`. Queries exclude records +older than 30 days, and the shared SQLite ledger is capped at 100,000 records. +Expired rows are deleted during Gateway startup, hourly maintenance, and later +writes. + +- Params: optional exact `agentId`, `sessionKey`, or `runId`; optional `kind` + (`"agent_run"` or `"tool_action"`); optional `status` (`"started"`, + `"succeeded"`, `"failed"`, `"cancelled"`, `"timed_out"`, `"blocked"`, or + `"unknown"`); optional inclusive `after` / `before` Unix-millisecond bounds; + optional `limit` from `1` to `500`; and optional string `cursor` from the + preceding page. +- Result: `{ "events": AuditEvent[], "nextCursor"?: string }`. + +Each event includes a stable event id, monotonic ledger sequence, source event +sequence, timestamp, actor, agent/session/run provenance, action, status, and a +normalized error code when applicable. Tool events may include tool call id and +tool name. The `redaction` field is always `"metadata_only"`: the ledger does +not store prompts, messages, tool arguments, tool results, command output, or +raw error text. + +Recording is on by default and controlled by +[`audit.enabled`](/gateway/configuration-reference#audit); when disabled, +`audit.list` keeps serving records written earlier until they expire. + +Use [`openclaw audit`](/cli/audit) for text queries and bounded JSON exports. + ## Task ledger RPCs Operator clients inspect and cancel gateway background task records through diff --git a/extensions/acpx/src/runtime-turn.test.ts b/extensions/acpx/src/runtime-turn.test.ts new file mode 100644 index 000000000000..43f295848cc7 --- /dev/null +++ b/extensions/acpx/src/runtime-turn.test.ts @@ -0,0 +1,42 @@ +// ACPX tests cover legacy runTurn adaptation into the terminal result contract. +import { describe, expect, it, vi } from "vitest"; +import type { AcpRuntime, AcpRuntimeEvent, AcpRuntimeTurnInput } from "../runtime-api.js"; +import { startRuntimeTurn } from "./runtime-turn.js"; + +function createLegacyRuntime(events: AcpRuntimeEvent[]): AcpRuntime { + return { + ensureSession: vi.fn(), + async *runTurn() { + yield* events; + }, + cancel: vi.fn(async () => undefined), + close: vi.fn(async () => undefined), + }; +} + +const turnInput: AcpRuntimeTurnInput = { + handle: { + sessionKey: "agent:main:acp:test", + backend: "test", + runtimeSessionName: "test", + }, + text: "hello", + mode: "prompt", + requestId: "request-1", +}; + +describe("startRuntimeTurn", () => { + it.each(["cancel", "cancelled", "manual-cancel"])( + "preserves %s cancellation from a legacy done event", + async (stopReason) => { + const turn = startRuntimeTurn(createLegacyRuntime([{ type: "done", stopReason }]), turnInput); + + expect(await turn.result).toEqual({ status: "cancelled", stopReason }); + const events: AcpRuntimeEvent[] = []; + for await (const event of turn.events) { + events.push(event); + } + expect(events).toEqual([]); + }, + ); +}); diff --git a/extensions/acpx/src/runtime-turn.ts b/extensions/acpx/src/runtime-turn.ts index d6fd4d0ab74e..157c28bb96a2 100644 --- a/extensions/acpx/src/runtime-turn.ts +++ b/extensions/acpx/src/runtime-turn.ts @@ -11,6 +11,10 @@ import type { AcpRuntimeTurnResult, } from "../runtime-api.js"; +function isCancellationStopReason(stopReason: string | undefined): boolean { + return stopReason === "cancel" || stopReason === "cancelled" || stopReason === "manual-cancel"; +} + class LegacyRunTurnEventQueue { private readonly items: AcpRuntimeEvent[] = []; private readonly waits: Array<{ @@ -100,8 +104,12 @@ function legacyRunTurnAsStartTurn(runtime: AcpRuntime, input: AcpRuntimeTurnInpu try { for await (const event of runtime.runTurn(input)) { if (event.type === "done") { + // Legacy runTurn events omit result.status but preserve stopReason, so infer + // cancellation here instead of silently converting it to success. settleResult({ - status: "completed", + status: + event.status ?? + (isCancellationStopReason(event.stopReason) ? "cancelled" : "completed"), ...(event.stopReason ? { stopReason: event.stopReason } : {}), }); continue; diff --git a/extensions/codex/src/app-server/approval-bridge.test.ts b/extensions/codex/src/app-server/approval-bridge.test.ts index 357fdb2d49f7..5f8efeb2d9dc 100644 --- a/extensions/codex/src/app-server/approval-bridge.test.ts +++ b/extensions/codex/src/app-server/approval-bridge.test.ts @@ -1424,6 +1424,43 @@ describe("Codex app-server approval bridge", () => { }); }); + it("preserves a deferred native approval failure for lifecycle projection", async () => { + const params = createParams(); + const onNativeToolFailureDisposition = vi.fn(); + mockHasNativeHookRelayInvocation.mockReturnValueOnce(true); + mockResolveNativeHookRelayDeferredToolApproval.mockResolvedValueOnce({ + handled: true, + outcome: "denied", + reason: "Approval cancelled because the run stopped", + failureDisposition: "cancelled", + }); + + const result = await handleCodexAppServerApprovalRequest({ + method: "item/commandExecution/requestApproval", + requestParams: { + threadId: "thread-1", + turnId: "turn-1", + itemId: "cmd-native-relay-deferred-failure", + command: "pnpm test extensions/codex/src/app-server", + cwd: "/workspace", + }, + paramsForRun: params, + threadId: "thread-1", + turnId: "turn-1", + nativeHookRelay: { + relayId: "relay-1", + allowedEvents: ["pre_tool_use"], + }, + onNativeToolFailureDisposition, + }); + + expect(result).toEqual({ decision: "decline" }); + expect(onNativeToolFailureDisposition).toHaveBeenCalledWith( + "cmd-native-relay-deferred-failure", + "cancelled", + ); + }); + it("fails closed when the native hook relay returns unreadable approval output", async () => { const params = createParams(); mockInvokeNativeHookRelay.mockResolvedValueOnce({ @@ -1704,6 +1741,7 @@ describe("Codex app-server approval bridge", () => { mockRunBeforeToolCallHook.mockResolvedValueOnce({ blocked: true, kind: "failure", + disposition: "blocked", deniedReason: "plugin-approval", reason: "Plugin approval required", }); @@ -1728,6 +1766,83 @@ describe("Codex app-server approval bridge", () => { }); }); + it.each(["failed", "cancelled", "timed_out"] as const)( + "preserves a %s pre-execution failure for native lifecycle projection", + async (disposition) => { + const params = createParams(); + const onNativeToolFailureDisposition = vi.fn(); + mockRunBeforeToolCallHook.mockResolvedValueOnce({ + blocked: true, + kind: "failure", + disposition, + deniedReason: "plugin-before-tool-call", + reason: "Tool call blocked because before_tool_call hook failed", + }); + + const result = await handleCodexAppServerApprovalRequest({ + method: "item/commandExecution/requestApproval", + requestParams: { + threadId: "thread-1", + turnId: "turn-1", + itemId: "cmd-policy-failure", + command: "pnpm test", + }, + paramsForRun: params, + threadId: "thread-1", + turnId: "turn-1", + onNativeToolFailureDisposition, + }); + + expect(result).toEqual({ decision: "decline" }); + expect(onNativeToolFailureDisposition).toHaveBeenCalledWith( + "cmd-policy-failure", + disposition, + ); + }, + ); + + it.each([ + { reason: "turn_progress_idle_timeout", disposition: "timed_out" }, + { reason: "turn_completion_idle_timeout", disposition: "timed_out" }, + { reason: "turn_terminal_idle_timeout", disposition: "timed_out" }, + { reason: "client_closed", disposition: "failed" }, + ] as const)( + "normalizes aborted approval reason $reason as $disposition", + async ({ reason, disposition }) => { + const params = createParams(); + const controller = new AbortController(); + controller.abort(reason); + const onNativeToolFailureDisposition = vi.fn(); + mockRunBeforeToolCallHook.mockResolvedValueOnce({ + blocked: true, + kind: "failure", + disposition: "cancelled", + deniedReason: "plugin-before-tool-call", + reason: "Approval cancelled because the run stopped", + }); + + await handleCodexAppServerApprovalRequest({ + method: "item/commandExecution/requestApproval", + requestParams: { + threadId: "thread-1", + turnId: "turn-1", + itemId: "cmd-aborted-policy", + command: "pnpm test", + }, + paramsForRun: params, + threadId: "thread-1", + turnId: "turn-1", + signal: controller.signal, + onNativeToolFailureDisposition, + }); + + expect(onNativeToolFailureDisposition).toHaveBeenCalledWith( + "cmd-aborted-policy", + disposition, + ); + }, + ); + it("describes command approvals from parsed command actions when available", async () => { const params = createParams(); mockCallGatewayTool @@ -2126,6 +2241,7 @@ describe("Codex app-server approval bridge", () => { it("fails closed when no approval route is available", async () => { const params = createParams(); + const onNativeToolFailureDisposition = vi.fn(); mockCallGatewayTool.mockResolvedValueOnce({ id: "plugin:approval-2", decision: null, @@ -2142,13 +2258,44 @@ describe("Codex app-server approval bridge", () => { paramsForRun: params, threadId: "thread-1", turnId: "turn-1", + onNativeToolFailureDisposition, }); expect(result).toEqual({ decision: "decline" }); expect(mockCallGatewayTool).toHaveBeenCalledTimes(1); + expect(onNativeToolFailureDisposition).toHaveBeenCalledWith("patch-1", "failed"); findApprovalEvent(params, { status: "unavailable", reason: "needs write access" }); }); + it("preserves an accepted approval expiry as timed out", async () => { + const params = createParams(); + const onNativeToolFailureDisposition = vi.fn(); + mockCallGatewayTool + .mockResolvedValueOnce({ id: "plugin:approval-expired", status: "accepted" }) + .mockResolvedValueOnce({ id: "plugin:approval-expired", decision: null }); + + const result = await handleCodexAppServerApprovalRequest({ + method: "item/commandExecution/requestApproval", + requestParams: { + threadId: "thread-1", + turnId: "turn-1", + itemId: "cmd-expired", + command: "pnpm test", + }, + paramsForRun: params, + threadId: "thread-1", + turnId: "turn-1", + onNativeToolFailureDisposition, + }); + + expect(result).toEqual({ decision: "decline" }); + expect(onNativeToolFailureDisposition).toHaveBeenCalledWith("cmd-expired", "timed_out"); + findApprovalEvent(params, { + status: "unavailable", + approvalId: "plugin:approval-expired", + }); + }); + it("sanitizes reason previews before forwarding approval text and events", async () => { const params = createParams(); mockCallGatewayTool.mockResolvedValueOnce({ diff --git a/extensions/codex/src/app-server/approval-bridge.ts b/extensions/codex/src/app-server/approval-bridge.ts index 22fdb1c90732..681fd0b4c940 100644 --- a/extensions/codex/src/app-server/approval-bridge.ts +++ b/extensions/codex/src/app-server/approval-bridge.ts @@ -9,6 +9,7 @@ import { import { type AgentApprovalEventData, buildAgentHookContextChannelFields, + type BeforeToolCallFailureDisposition, formatApprovalDisplayPath, hasNativeHookRelayInvocation, invokeNativeHookRelay, @@ -25,6 +26,7 @@ import { isTrustedCodexModelBackedOpenAIProvider, type OpenClawExecPolicyForCodexAppServer, } from "./config.js"; +import { resolveCodexToolAbortTerminalReason } from "./dynamic-tool-execution.js"; import { approvalRequestExplicitlyUnavailable, mapExecDecisionToOutcome, @@ -86,6 +88,10 @@ export async function handleCodexAppServerApprovalRequest(params: { internalExecAutoReview?: boolean; autoApprove?: boolean; signal?: AbortSignal; + onNativeToolFailureDisposition?: ( + itemId: string, + disposition: Exclude, + ) => void; }): Promise { const requestParams = isJsonObject(params.requestParams) ? params.requestParams : undefined; if (!matchesCurrentTurn(requestParams, params.threadId, params.turnId)) { @@ -111,6 +117,7 @@ export async function handleCodexAppServerApprovalRequest(params: { signal: params.signal, }); if (policyOutcome?.outcome === "denied") { + recordNativeToolFailureDisposition(params, context, policyOutcome.failureDisposition); emitApprovalEvent(params.paramsForRun, { phase: "resolved", kind: context.kind, @@ -183,6 +190,7 @@ export async function handleCodexAppServerApprovalRequest(params: { const approvalId = requestResult?.id; if (!approvalId) { + recordNativeToolFailureDisposition(params, context, "failed"); emitApprovalEvent(params.paramsForRun, { phase: "resolved", kind: context.kind, @@ -206,10 +214,21 @@ export async function handleCodexAppServerApprovalRequest(params: { message: "Codex app-server approval requested.", }); - const decision = approvalRequestExplicitlyUnavailable(requestResult) + const requestUnavailable = approvalRequestExplicitlyUnavailable(requestResult); + const decision = requestUnavailable ? null : await waitForPluginApprovalDecision({ approvalId, signal: params.signal }); - const outcome = mapExecDecisionToOutcome(decision); + const approvalExpired = !requestUnavailable && decision === null; + const outcome = params.signal?.aborted ? "cancelled" : mapExecDecisionToOutcome(decision); + if (outcome === "cancelled") { + recordNativeToolFailureDisposition( + params, + context, + params.signal?.aborted ? resolveCodexToolAbortTerminalReason(params.signal) : "cancelled", + ); + } else if (outcome === "unavailable") { + recordNativeToolFailureDisposition(params, context, approvalExpired ? "timed_out" : "failed"); + } emitApprovalEvent(params.paramsForRun, { phase: "resolved", @@ -232,6 +251,11 @@ export async function handleCodexAppServerApprovalRequest(params: { return buildApprovalResponse(params.method, context.requestParams, outcome); } catch (error) { const cancelled = params.signal?.aborted === true; + recordNativeToolFailureDisposition( + params, + context, + cancelled && params.signal ? resolveCodexToolAbortTerminalReason(params.signal) : "failed", + ); emitApprovalEvent(params.paramsForRun, { phase: "resolved", kind: context.kind, @@ -253,6 +277,27 @@ export async function handleCodexAppServerApprovalRequest(params: { } } +function recordNativeToolFailureDisposition( + params: Pick< + Parameters[0], + "onNativeToolFailureDisposition" | "signal" + >, + context: Pick, + disposition: Exclude | undefined, +): void { + if (!context.itemId || !disposition) { + return; + } + try { + params.onNativeToolFailureDisposition?.( + context.itemId, + params.signal?.aborted ? resolveCodexToolAbortTerminalReason(params.signal) : disposition, + ); + } catch { + // Audit projection must not alter the approval decision sent to Codex. + } +} + /** Converts an OpenClaw approval outcome into the app-server method response. */ export function buildApprovalResponse( method: string, @@ -373,7 +418,11 @@ function buildApprovalContext(params: { type ApprovalContext = ReturnType; type ApprovalPolicyOutcome = - | { outcome: "denied"; reason: string } + | { + outcome: "denied"; + reason: string; + failureDisposition?: Exclude; + } | { outcome: "approved-once" | "approved-session" } | { outcome: "no-decision" }; @@ -641,7 +690,13 @@ async function runOpenClawToolPolicyForApprovalRequest(params: { signal: params.signal, }); if (nativeRelayOutcome?.blocked) { - return { outcome: "denied", reason: nativeRelayOutcome.reason }; + return { + outcome: "denied", + reason: nativeRelayOutcome.reason, + ...(nativeRelayOutcome.failureDisposition + ? { failureDisposition: nativeRelayOutcome.failureDisposition } + : {}), + }; } if ( nativeRelayOutcome?.outcome === "approved-once" || @@ -677,7 +732,13 @@ async function runOpenClawToolPolicyForApprovalRequest(params: { }, }); if (outcome.blocked) { - return { outcome: "denied", reason: outcome.reason }; + return { + outcome: "denied", + reason: outcome.reason, + ...(outcome.kind === "failure" && outcome.disposition !== "blocked" + ? { failureDisposition: outcome.disposition } + : {}), + }; } if ("params" in outcome && toolPolicyParamsWereRewritten(policyRequest.params, outcome.params)) { return { @@ -712,6 +773,7 @@ async function runNativeRelayToolPolicyForApprovalRequest(params: { handled: true; blocked: true; reason: string; + failureDisposition?: Exclude; } | { handled: true; @@ -750,7 +812,14 @@ async function runNativeRelayToolPolicyForApprovalRequest(params: { signal: params.signal, }); if (approvalOutcome?.outcome === "denied") { - return { handled: true, blocked: true, reason: approvalOutcome.reason }; + return { + handled: true, + blocked: true, + reason: approvalOutcome.reason, + ...(approvalOutcome.failureDisposition + ? { failureDisposition: approvalOutcome.failureDisposition } + : {}), + }; } if (approvalOutcome?.outcome === "approved-once") { return { handled: true, outcome: approvalOutcome.outcome }; @@ -768,7 +837,12 @@ async function runNativeRelayToolPolicyForApprovalRequest(params: { }); const decision = readNativeRelayPreToolUseDecision(response); if (decision.blocked) { - return { handled: true, blocked: true, reason: decision.reason }; + return { + handled: true, + blocked: true, + reason: decision.reason, + ...(decision.failureDisposition ? { failureDisposition: decision.failureDisposition } : {}), + }; } const approvalOutcome = await resolveNativeHookRelayDeferredToolApproval({ relayId: params.nativeHookRelay.relayId, @@ -776,7 +850,14 @@ async function runNativeRelayToolPolicyForApprovalRequest(params: { signal: params.signal, }); if (approvalOutcome?.outcome === "denied") { - return { handled: true, blocked: true, reason: approvalOutcome.reason }; + return { + handled: true, + blocked: true, + reason: approvalOutcome.reason, + ...(approvalOutcome.failureDisposition + ? { failureDisposition: approvalOutcome.failureDisposition } + : {}), + }; } if (approvalOutcome?.outcome === "approved-once") { return { handled: true, outcome: approvalOutcome.outcome }; @@ -789,6 +870,7 @@ async function runNativeRelayToolPolicyForApprovalRequest(params: { reason: `OpenClaw native hook relay unavailable for Codex app-server approval: ${formatCodexDisplayText( formatErrorMessage(error), )}`, + failureDisposition: "failed", }; } } @@ -819,9 +901,13 @@ function buildNativeRelayPreToolUsePayload(params: { }; } -function readNativeRelayPreToolUseDecision( - response: NativeHookRelayProcessResponse | undefined, -): { blocked: true; reason: string } | { blocked: false } { +function readNativeRelayPreToolUseDecision(response: NativeHookRelayProcessResponse | undefined): + | { + blocked: true; + reason: string; + failureDisposition?: Exclude; + } + | { blocked: false } { if (!response || response.exitCode !== 0) { return { blocked: true, @@ -829,6 +915,7 @@ function readNativeRelayPreToolUseDecision( sanitizeRelayDecisionReason(response?.stderr) || sanitizeRelayDecisionReason(response?.stdout) || "OpenClaw native hook relay failed for Codex app-server approval.", + failureDisposition: response?.failureDisposition ?? "failed", }; } const stdout = response.stdout?.trim(); @@ -843,6 +930,7 @@ function readNativeRelayPreToolUseDecision( reason: readString(output, "permissionDecisionReason") || "OpenClaw native hook policy denied Codex app-server approval.", + ...(response.failureDisposition ? { failureDisposition: response.failureDisposition } : {}), }; } // The app-server bridge invokes the relay in report mode, where the relay @@ -852,6 +940,7 @@ function readNativeRelayPreToolUseDecision( reason: output ? "OpenClaw native hook relay returned a non-deny Codex app-server approval decision." : "OpenClaw native hook relay returned an unreadable Codex app-server approval result.", + failureDisposition: "failed", }; } diff --git a/extensions/codex/src/app-server/dynamic-tool-diagnostics.ts b/extensions/codex/src/app-server/dynamic-tool-diagnostics.ts index 5b96cc8fc238..923fdcfcba93 100644 --- a/extensions/codex/src/app-server/dynamic-tool-diagnostics.ts +++ b/extensions/codex/src/app-server/dynamic-tool-diagnostics.ts @@ -6,6 +6,7 @@ import type { CodexDynamicToolCallParams, CodexDynamicToolCallResponse } from ". type DynamicToolDiagnosticContext = { call: CodexDynamicToolCallParams; + agentId?: string | undefined; runId?: string | undefined; sessionId?: string | undefined; sessionKey?: string | undefined; @@ -15,6 +16,7 @@ type DynamicToolDiagnosticContext = { export function emitDynamicToolStartedDiagnostic(params: DynamicToolDiagnosticContext): void { emitTrustedDiagnosticEvent({ type: "tool.execution.started", + agentId: params.agentId, runId: params.runId, sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -27,10 +29,12 @@ export function emitDynamicToolStartedDiagnostic(params: DynamicToolDiagnosticCo export function emitDynamicToolErrorDiagnostic( params: DynamicToolDiagnosticContext & { durationMs: number; + terminalReason?: "failed" | "cancelled" | "timed_out"; }, ): void { emitTrustedDiagnosticEvent({ type: "tool.execution.error", + agentId: params.agentId, runId: params.runId, sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -38,6 +42,7 @@ export function emitDynamicToolErrorDiagnostic( toolCallId: params.call.callId, durationMs: params.durationMs, errorCategory: "codex_dynamic_tool_error", + terminalReason: params.terminalReason ?? "failed", }); } @@ -53,6 +58,7 @@ export function emitDynamicToolTerminalDiagnostic( if (terminalType === "completed") { emitTrustedDiagnosticEvent({ type: "tool.execution.completed", + agentId: params.agentId, runId: params.runId, sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -65,6 +71,7 @@ export function emitDynamicToolTerminalDiagnostic( if (terminalType === "blocked") { emitTrustedDiagnosticEvent({ type: "tool.execution.blocked", + agentId: params.agentId, runId: params.runId, sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -75,5 +82,8 @@ export function emitDynamicToolTerminalDiagnostic( }); return; } - emitDynamicToolErrorDiagnostic(params); + emitDynamicToolErrorDiagnostic({ + ...params, + terminalReason: params.response.diagnosticTerminalReason ?? "failed", + }); } diff --git a/extensions/codex/src/app-server/dynamic-tool-execution.test.ts b/extensions/codex/src/app-server/dynamic-tool-execution.test.ts index c3a9f7b448b1..2c9661e30bb0 100644 --- a/extensions/codex/src/app-server/dynamic-tool-execution.test.ts +++ b/extensions/codex/src/app-server/dynamic-tool-execution.test.ts @@ -218,6 +218,7 @@ describe("dynamic tool execution helpers", () => { }, ], }); + expect((await response).diagnosticTerminalReason).toBe("timed_out"); expect(capturedSignal?.aborted).toBe(true); expect(onFallbackSelected).toHaveBeenCalledOnce(); expect(onTimeout).toHaveBeenCalledTimes(1); @@ -231,7 +232,7 @@ describe("dynamic tool execution helpers", () => { }, ], details: { - status: "failed", + status: "timed_out", error: "OpenClaw dynamic tool call timed out after 1ms while running tool message.", }, }, @@ -239,7 +240,7 @@ describe("dynamic tool execution helpers", () => { }); }); - it("reports pre-execution aborts to the private result observer", async () => { + it("reports pre-execution cancellations to the private result observer", async () => { const controller = new AbortController(); controller.abort(new Error("run cancelled")); const onAgentToolResult = vi.fn(); @@ -266,6 +267,7 @@ describe("dynamic tool execution helpers", () => { { type: "inputText", text: "OpenClaw dynamic tool call aborted before execution." }, ], }); + expect(result.diagnosticTerminalReason).toBe("cancelled"); expect(handleToolCall).not.toHaveBeenCalled(); expect(onAgentToolResult).toHaveBeenCalledOnce(); expect(onAgentToolResult).toHaveBeenCalledWith({ @@ -273,7 +275,7 @@ describe("dynamic tool execution helpers", () => { result: { content: [{ type: "text", text: "OpenClaw dynamic tool call aborted before execution." }], details: { - status: "failed", + status: "cancelled", error: "OpenClaw dynamic tool call aborted before execution.", }, }, @@ -281,6 +283,180 @@ describe("dynamic tool execution helpers", () => { }); }); + it.each([ + Object.assign(new Error("gateway timeout"), { name: "TimeoutError" }), + "turn_completion_idle_timeout", + ])("preserves enclosing timeout provenance for pre-execution aborts", async (reason) => { + const controller = new AbortController(); + controller.abort(reason); + + const result = await handleDynamicToolCallWithTimeout({ + call: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-timeout-abort", + namespace: null, + tool: "memory_search", + arguments: {}, + }, + toolBridge: { handleToolCall: vi.fn() }, + signal: controller.signal, + timeoutMs: 1_000, + }); + + expect(result.diagnosticTerminalReason).toBe("timed_out"); + }); + + it("classifies app-server client closure as a failed tool outcome", async () => { + const controller = new AbortController(); + controller.abort("client_closed"); + + const result = await handleDynamicToolCallWithTimeout({ + call: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-client-closed", + namespace: null, + tool: "memory_search", + arguments: {}, + }, + toolBridge: { handleToolCall: vi.fn() }, + signal: controller.signal, + timeoutMs: 1_000, + }); + + expect(result.diagnosticTerminalReason).toBe("failed"); + }); + + it("preserves enclosing timeout provenance for active tool aborts", async () => { + const controller = new AbortController(); + const resultPromise = handleDynamicToolCallWithTimeout({ + call: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-active-timeout-abort", + namespace: null, + tool: "memory_search", + arguments: {}, + }, + toolBridge: { handleToolCall: vi.fn(() => new Promise(() => {})) }, + signal: controller.signal, + timeoutMs: 1_000, + }); + controller.abort(Object.assign(new Error("gateway timeout"), { name: "TimeoutError" })); + + await expect(resultPromise).resolves.toMatchObject({ + success: false, + diagnosticTerminalReason: "timed_out", + }); + }); + + it("preserves timeout provenance when the dynamic tool bridge rejects", async () => { + const timeoutError = Object.assign(new Error("tool deadline elapsed"), { + name: "TimeoutError", + }); + const onAgentToolResult = vi.fn(); + + const result = await handleDynamicToolCallWithTimeout({ + call: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-rejected-timeout", + namespace: null, + tool: "memory_search", + arguments: {}, + }, + toolBridge: { + handleToolCall: vi.fn(async () => { + throw timeoutError; + }), + }, + signal: new AbortController().signal, + timeoutMs: 1_000, + onAgentToolResult, + }); + + expect(result).toMatchObject({ + success: false, + diagnosticTerminalReason: "timed_out", + }); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "memory_search", + result: { + content: [{ type: "text", text: "tool deadline elapsed" }], + details: { status: "timed_out", error: "tool deadline elapsed" }, + }, + isError: true, + }); + }); + + it("contains hostile rejected values while notifying the private observer", async () => { + const hostileError = Object.defineProperty(new Error(), "message", { + get() { + throw new Error("message getter escaped"); + }, + }); + const onAgentToolResult = vi.fn(); + + const result = await handleDynamicToolCallWithTimeout({ + call: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-hostile-error", + namespace: null, + tool: "memory_search", + arguments: {}, + }, + toolBridge: { + handleToolCall: vi.fn(async () => { + throw hostileError; + }), + }, + signal: new AbortController().signal, + timeoutMs: 1_000, + onAgentToolResult, + }); + + expect(result).toMatchObject({ + success: false, + diagnosticTerminalReason: "failed", + contentItems: [{ type: "inputText", text: "OpenClaw dynamic tool call failed." }], + }); + expect(onAgentToolResult).toHaveBeenCalledOnce(); + }); + + it("contains hostile abort reasons while notifying the private observer", async () => { + const hostileReason = Object.defineProperty({}, "name", { + get() { + throw new Error("name getter escaped"); + }, + }); + const controller = new AbortController(); + controller.abort(hostileReason); + const onAgentToolResult = vi.fn(); + + const result = await handleDynamicToolCallWithTimeout({ + call: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-hostile-abort", + namespace: null, + tool: "memory_search", + arguments: {}, + }, + toolBridge: { handleToolCall: vi.fn() }, + signal: controller.signal, + timeoutMs: 1_000, + onAgentToolResult, + }); + + expect(result).toMatchObject({ + success: false, + diagnosticTerminalReason: "cancelled", + }); + expect(onAgentToolResult).toHaveBeenCalledOnce(); + }); + it("logs process poll timeout context separately from session idle", async () => { vi.useFakeTimers(); const warn = vi.spyOn(embeddedAgentLog, "warn").mockImplementation(() => undefined); diff --git a/extensions/codex/src/app-server/dynamic-tool-execution.ts b/extensions/codex/src/app-server/dynamic-tool-execution.ts index 9a1e9c39a2d1..d5484dc1eae5 100644 --- a/extensions/codex/src/app-server/dynamic-tool-execution.ts +++ b/extensions/codex/src/app-server/dynamic-tool-execution.ts @@ -4,6 +4,8 @@ */ import { embeddedAgentLog, + formatToolExecutionErrorMessage, + resolveToolExecutionErrorKind, type EmbeddedRunAttemptParams, } from "openclaw/plugin-sdk/agent-harness-runtime"; import { @@ -12,6 +14,9 @@ import { } from "openclaw/plugin-sdk/diagnostic-runtime"; import { parseStrictNonNegativeInteger } from "openclaw/plugin-sdk/number-runtime"; import type { CodexDynamicToolBridge } from "./dynamic-tools.js"; +import { resolveCodexToolAbortTerminalReason } from "./tool-abort-terminal-reason.js"; + +export { resolveCodexToolAbortTerminalReason } from "./tool-abort-terminal-reason.js"; import { isJsonObject, type CodexDynamicToolCallParams, @@ -149,21 +154,27 @@ export async function handleDynamicToolCallWithTimeout(params: { ); } }; - const notifyFailedToolResult = (message: string) => { + const notifyFailedToolResult = ( + message: string, + terminalReason: "failed" | "cancelled" | "timed_out" = "failed", + ) => { notifyAgentToolResult({ toolName: params.call.tool, result: { content: [{ type: "text", text: message }], - details: { status: "failed", error: message }, + details: { status: terminalReason, error: message }, }, isError: true, }); }; if (params.signal.aborted) { const message = "OpenClaw dynamic tool call aborted before execution."; + const terminalReason = resolveCodexToolAbortTerminalReason(params.signal); params.onFallbackSelected?.(); - notifyFailedToolResult(message); - return failedDynamicToolResponse(message); + notifyFailedToolResult(message, terminalReason); + return failedDynamicToolResponse(message, { + terminalReason, + }); } const controller = new AbortController(); @@ -172,10 +183,16 @@ export async function handleDynamicToolCallWithTimeout(params: { let resolveAbort: ((response: CodexDynamicToolCallResponse) => void) | undefined; const abortFromRun = () => { const message = "OpenClaw dynamic tool call aborted."; + const terminalReason = resolveCodexToolAbortTerminalReason(params.signal); params.onFallbackSelected?.(); controller.abort(params.signal.reason ?? new Error(message)); - notifyFailedToolResult(message); - resolveAbort?.(failedDynamicToolResponse(message, { sideEffectEvidence: true })); + notifyFailedToolResult(message, terminalReason); + resolveAbort?.( + failedDynamicToolResponse(message, { + sideEffectEvidence: true, + terminalReason, + }), + ); }; const abortPromise = new Promise((resolve) => { resolveAbort = resolve; @@ -192,9 +209,12 @@ export async function handleDynamicToolCallWithTimeout(params: { ...timeoutDetails.meta, consoleMessage: timeoutDetails.consoleMessage, }); - notifyFailedToolResult(timeoutDetails.responseMessage); + notifyFailedToolResult(timeoutDetails.responseMessage, "timed_out"); resolve( - failedDynamicToolResponse(timeoutDetails.responseMessage, { sideEffectEvidence: true }), + failedDynamicToolResponse(timeoutDetails.responseMessage, { + sideEffectEvidence: true, + terminalReason: "timed_out", + }), ); }, timeoutMs); timeout.unref?.(); @@ -215,14 +235,21 @@ export async function handleDynamicToolCallWithTimeout(params: { timeoutPromise, ]); if (!response.success && !didNotifyAgentToolResult) { - notifyFailedToolResult(readDynamicToolResponseText(response)); + notifyFailedToolResult( + readDynamicToolResponseText(response), + response.diagnosticTerminalReason ?? "failed", + ); } return response; } catch (error) { - const message = error instanceof Error ? error.message : String(error); - notifyFailedToolResult(message); + const terminalReason = params.signal.aborted + ? resolveCodexToolAbortTerminalReason(params.signal) + : resolveToolExecutionErrorKind(error); + const message = formatToolExecutionErrorMessage(error, "OpenClaw dynamic tool call failed."); + notifyFailedToolResult(message, terminalReason); return failedDynamicToolResponse(message, { sideEffectEvidence: true, + terminalReason, }); } finally { if (timeout) { @@ -248,7 +275,10 @@ function readDynamicToolResponseText(response: CodexDynamicToolCallResponse): st function failedDynamicToolResponse( message: string, - options?: { sideEffectEvidence?: boolean }, + options?: { + sideEffectEvidence?: boolean; + terminalReason?: "failed" | "cancelled" | "timed_out"; + }, ): CodexDynamicToolCallResponse { const response: CodexDynamicToolCallResponse = { contentItems: [{ type: "inputText", text: message }], @@ -259,6 +289,11 @@ function failedDynamicToolResponse( enumerable: false, value: "error", }); + Object.defineProperty(response, "diagnosticTerminalReason", { + configurable: true, + enumerable: false, + value: options?.terminalReason ?? "failed", + }); if (options?.sideEffectEvidence === true) { Object.defineProperty(response, "sideEffectEvidence", { configurable: true, diff --git a/extensions/codex/src/app-server/dynamic-tools.test.ts b/extensions/codex/src/app-server/dynamic-tools.test.ts index 2bd7ce28ffb9..262b29817634 100644 --- a/extensions/codex/src/app-server/dynamic-tools.test.ts +++ b/extensions/codex/src/app-server/dynamic-tools.test.ts @@ -501,9 +501,10 @@ describe("createCodexDynamicToolBridge", () => { ], signal: new AbortController().signal, hookContext: { + agentId: "agent-quarantine", runId: "run-1", sessionId: "session-1", - sessionKey: "agent:main:session-1", + sessionKey: "global", }, }); await waitForDiagnosticEventsDrained(); @@ -537,9 +538,10 @@ describe("createCodexDynamicToolBridge", () => { expect(blockedEvents).toContainEqual( expect.objectContaining({ type: "tool.execution.blocked", + agentId: "agent-quarantine", runId: "run-1", sessionId: "session-1", - sessionKey: "agent:main:session-1", + sessionKey: "global", toolName: "fuzzplugin_move_angles", deniedReason: "unsupported_tool_schema", reason: 'fuzzplugin_move_angles.inputSchema.type must be "object"', @@ -2779,6 +2781,176 @@ describe("createCodexDynamicToolBridge", () => { }); }); + it("preserves hook timeout classification for the outer lifecycle owner", async () => { + const beforeToolCall = vi.fn(async () => { + throw Object.assign(new Error("timed out after 5ms"), { name: "TimeoutError" }); + }); + initializeGlobalHookRunner( + createMockPluginRegistry([{ hookName: "before_tool_call", handler: beforeToolCall }]), + ); + const execute = vi.fn(async () => textToolResult("should not run")); + const bridge = createCodexDynamicToolBridge({ + tools: [createTool({ name: "exec", execute })], + signal: new AbortController().signal, + hookContext: { runId: "run-hook-timeout" }, + }); + + const result = await bridge.handleToolCall({ + threadId: "thread-1", + turnId: "turn-1", + callId: "call-hook-timeout", + namespace: null, + tool: "exec", + arguments: { command: "pwd" }, + }); + + expect(result.success).toBe(false); + expect(result.diagnosticTerminalType).toBe("error"); + expect(result.diagnosticTerminalReason).toBe("timed_out"); + expect(result.sideEffectEvidence).toBeUndefined(); + expect(execute).not.toHaveBeenCalled(); + }); + + it.each(["timed_out", "cancelled"] as const)( + "preserves structured %s results for the outer lifecycle owner", + async (status) => { + const bridge = createBridgeWithToolResult("exec", textToolResult("tool stopped", { status })); + + const result = await bridge.handleToolCall({ + threadId: "thread-1", + turnId: "turn-1", + callId: `call-${status}`, + namespace: null, + tool: "exec", + arguments: { command: "pwd" }, + }); + + expect(result.success).toBe(false); + expect(result.diagnosticTerminalType).toBe("error"); + expect(result.diagnosticTerminalReason).toBe(status); + }, + ); + + it("preserves thrown timeout classification for the outer lifecycle owner", async () => { + const timeoutError = Object.assign(new Error("tool deadline elapsed"), { + name: "TimeoutError", + }); + const onAgentToolResult = vi.fn(); + const bridge = createCodexDynamicToolBridge({ + tools: [ + createTool({ + name: "exec", + execute: vi.fn(async () => { + throw timeoutError; + }), + }), + ], + signal: new AbortController().signal, + }); + + const result = await bridge.handleToolCall( + { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-timeout", + namespace: null, + tool: "exec", + arguments: { command: "pwd" }, + }, + { onAgentToolResult }, + ); + + expect(result.success).toBe(false); + expect(result.diagnosticTerminalType).toBe("error"); + expect(result.diagnosticTerminalReason).toBe("timed_out"); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "exec", + result: { + content: [{ type: "text", text: "tool deadline elapsed" }], + details: { status: "timed_out", error: "tool deadline elapsed" }, + }, + isError: true, + }); + }); + + it("contains hostile thrown values while notifying the outer lifecycle owner", async () => { + const hostileError = Object.defineProperty(new Error(), "message", { + get() { + throw new Error("message getter escaped"); + }, + }); + const onAgentToolResult = vi.fn(); + const bridge = createCodexDynamicToolBridge({ + tools: [ + createTool({ + name: "exec", + execute: vi.fn(async () => { + throw hostileError; + }), + }), + ], + signal: new AbortController().signal, + }); + + const result = await bridge.handleToolCall( + { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-hostile-error", + namespace: null, + tool: "exec", + arguments: { command: "pwd" }, + }, + { onAgentToolResult }, + ); + + expect(result).toMatchObject({ + success: false, + diagnosticTerminalReason: "failed", + contentItems: [{ type: "inputText", text: "OpenClaw dynamic tool call failed." }], + }); + expect(onAgentToolResult).toHaveBeenCalledOnce(); + }); + + it("preserves report-only approval blocks for the outer lifecycle owner", async () => { + const beforeToolCall = vi.fn(async () => ({ + requireApproval: { + pluginId: "test-plugin", + title: "Needs approval", + description: "Review before running", + }, + })); + initializeGlobalHookRunner( + createMockPluginRegistry([{ hookName: "before_tool_call", handler: beforeToolCall }]), + ); + const execute = vi.fn(async () => textToolResult("should not run")); + const tool = wrapToolWithBeforeToolCallHook( + createTool({ name: "exec", execute }), + { runId: "run-approval-report" }, + { approvalMode: "report" }, + ); + const bridge = createCodexDynamicToolBridge({ + tools: [tool], + signal: new AbortController().signal, + hookContext: { runId: "run-approval-report" }, + }); + + const result = await bridge.handleToolCall({ + threadId: "thread-1", + turnId: "turn-1", + callId: "call-approval-report", + namespace: null, + tool: "exec", + arguments: { command: "pwd" }, + }); + + expect(result.success).toBe(false); + expect(result.diagnosticTerminalType).toBe("blocked"); + expect(result.diagnosticTerminalReason).toBeUndefined(); + expect(result.sideEffectEvidence).toBeUndefined(); + expect(execute).not.toHaveBeenCalled(); + }); + it("applies dynamic tool result middleware before after_tool_call observes the result", async () => { const events: string[] = []; const beforeToolCall = vi.fn(async () => { @@ -2848,6 +3020,55 @@ describe("createCodexDynamicToolBridge", () => { }); }); + it.each(["timed_out", "cancelled", "blocked"] as const)( + "preserves raw %s disposition for private observation after middleware rewrites it", + async (status) => { + const registry = createEmptyPluginRegistry(); + const handler = vi.fn(async (event: { result: AgentToolResult }) => { + event.result.content = [{ type: "text", text: "compacted failure" }]; + const details = requireRecord(event.result.details, "middleware details"); + details.stage = "middleware"; + details.status = "failed"; + return { result: event.result }; + }); + registry.agentToolResultMiddlewares.push({ + pluginId: "result-redactor", + pluginName: "Result Redactor", + rawHandler: handler, + handler, + runtimes: ["codex"], + source: "test", + }); + setActivePluginRegistry(registry); + const onAgentToolResult = vi.fn(); + const bridge = createBridgeWithToolResult("exec", textToolResult("raw failure", { status })); + + const result = await bridge.handleToolCall( + { + threadId: "thread-1", + turnId: "turn-1", + callId: `call-raw-${status}`, + namespace: null, + tool: "exec", + arguments: { command: "status" }, + }, + { onAgentToolResult }, + ); + + expect(result.success).toBe(false); + expect(result.diagnosticTerminalType).toBe(status === "blocked" ? "blocked" : "error"); + expect(result.diagnosticTerminalReason).toBe(status === "blocked" ? undefined : status); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "exec", + result: { + content: [{ type: "text", text: "compacted failure" }], + details: { stage: "middleware", status }, + }, + isError: true, + }); + }, + ); + it("reports confirmed sends as successful when result middleware fails", async () => { const registry = createEmptyPluginRegistry(); const handler = vi.fn((event: { result: AgentToolResult }) => { diff --git a/extensions/codex/src/app-server/dynamic-tools.ts b/extensions/codex/src/app-server/dynamic-tools.ts index 43524370eb3a..5bfac4e8db73 100644 --- a/extensions/codex/src/app-server/dynamic-tools.ts +++ b/extensions/codex/src/app-server/dynamic-tools.ts @@ -13,6 +13,8 @@ import { extractToolResultMediaArtifact, filterToolResultMediaUrls, finalizeToolTerminalPresentation, + formatToolExecutionErrorMessage, + getBeforeToolCallFailureDisposition, HEARTBEAT_RESPONSE_TOOL_NAME, embeddedAgentLog, getChannelAgentToolMeta, @@ -27,6 +29,8 @@ import { isMessagingToolSendAction, normalizeHeartbeatToolResponse, projectRuntimeToolInputSchema, + resolveToolExecutionErrorKind, + resolveToolResultFailureKind, runAgentHarnessAfterToolCallHook, sanitizeToolResult, setBeforeToolCallDiagnosticsEnabled, @@ -49,11 +53,13 @@ import type { CodexDynamicToolCallOutputContentItem, CodexDynamicToolCallParams, CodexDynamicToolCallResponse, + CodexDynamicToolDiagnosticTerminalReason, CodexDynamicToolDiagnosticTerminalType, CodexDynamicToolFunctionSpec, CodexDynamicToolSpec, JsonValue, } from "./protocol.js"; +import { resolveCodexToolAbortTerminalReason } from "./tool-abort-terminal-reason.js"; type CodexDynamicToolHookContext = { agentId?: string; @@ -523,6 +529,7 @@ export function createCodexDynamicToolBridge(params: { ); const telemetryRawResult = sanitizeToolResult(rawResult); const rawIsError = isCodexToolResultError(rawResult); + const rawResultFailureKind = resolveToolResultFailureKind(rawResult); const middlewareResult = await middlewareRunner.applyToolResultMiddleware({ threadId: call.threadId, turnId: call.turnId, @@ -541,7 +548,19 @@ export function createCodexDynamicToolBridge(params: { result: middlewareResult, }); const resultIsError = rawIsError || isCodexToolResultError(result); - notifyAgentToolResult(options?.onAgentToolResult, toolName, result, resultIsError); + const finalResultFailureKind = resolveToolResultFailureKind(result); + const resultFailureKind = rawResultFailureKind ?? finalResultFailureKind; + const observerResult = + rawResultFailureKind && finalResultFailureKind !== rawResultFailureKind + ? { + ...result, + details: { + ...(isRecord(result.details) ? result.details : {}), + status: rawResultFailureKind, + }, + } + : result; + notifyAgentToolResult(options?.onAgentToolResult, toolName, observerResult, resultIsError); void runAgentHarnessAfterToolCallHook({ toolName, toolCallId: call.callId, @@ -584,7 +603,8 @@ export function createCodexDynamicToolBridge(params: { isError: resultIsError, messagingTarget: confirmedMessagingTarget, }); - const terminalType = inferToolResultDiagnosticTerminalType(result, resultIsError); + const terminalType = + resultFailureKind === "blocked" ? "blocked" : resultIsError ? "error" : "completed"; const response = withDiagnosticTerminalType( { contentItems: convertToolContents(result.content, toolResultMaxChars), @@ -592,6 +612,7 @@ export function createCodexDynamicToolBridge(params: { }, terminalType, ); + withDiagnosticFailureDisposition(response, resultFailureKind); const blocksSourceReplyTermination = hasExplicitNonSourceMessageRoute( executedArgs, params.hookContext, @@ -650,7 +671,16 @@ export function createCodexDynamicToolBridge(params: { isReplaySafeToolCall(toolName, executedArgs)); return withSideEffectEvidence(response, !replaySafe); } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); + const beforeToolCallDisposition = getBeforeToolCallFailureDisposition(error); + const executionDisposition = + beforeToolCallDisposition ?? + (signal.aborted + ? resolveCodexToolAbortTerminalReason(signal) + : resolveToolExecutionErrorKind(error)); + const errorMessage = formatToolExecutionErrorMessage( + error, + "OpenClaw dynamic tool call failed.", + ); const adjustedExecutedArgs = consumeAdjustedParamsForToolCall( call.callId, toolResultHookContext.runId, @@ -661,7 +691,7 @@ export function createCodexDynamicToolBridge(params: { executionPrevented = executionPrevented || consumePreExecutionBlockedToolCall(call.callId, toolResultHookContext.runId); - const failedResult = failedToolResult(errorMessage); + const failedResult = failedToolResult(errorMessage, executionDisposition); finalizeToolTerminalPresentation({ toolCallId: call.callId, runId: toolResultHookContext.runId, @@ -697,7 +727,7 @@ export function createCodexDynamicToolBridge(params: { (isReplaySafeToolInstance(toolEntry.tool) && isReplaySafeToolCall(toolName, executedArgs)); return withSideEffectEvidence( - withDiagnosticTerminalType( + withDiagnosticFailureDisposition( { contentItems: [ { @@ -707,7 +737,7 @@ export function createCodexDynamicToolBridge(params: { ], success: false, }, - "error", + executionDisposition, ), didStartExecution && !replaySafe, ); @@ -735,10 +765,13 @@ function notifyAgentToolResult( } } -function failedToolResult(message: string): AgentToolResult { +function failedToolResult( + message: string, + status: "blocked" | CodexDynamicToolDiagnosticTerminalReason = "failed", +): AgentToolResult { return { content: [{ type: "text", text: message }], - details: { status: "failed", error: message }, + details: { status, error: message }, }; } @@ -949,6 +982,7 @@ function emitQuarantinedDynamicToolDiagnostics( for (const tool of tools) { emitTrustedDiagnosticEvent({ type: "tool.execution.blocked", + agentId: ctx?.agentId, runId: ctx?.runId, sessionId: ctx?.sessionId, sessionKey: ctx?.sessionKey, @@ -1215,20 +1249,6 @@ function isAsyncStartedToolResult(result: AgentToolResult): boolean { return isRecord(details) && details.async === true && details.status === "started"; } -function inferToolResultDiagnosticTerminalType( - result: AgentToolResult, - isError: boolean, -): CodexDynamicToolDiagnosticTerminalType { - const details = result.details; - if (isRecord(details) && typeof details.status === "string") { - const status = details.status.trim().toLowerCase(); - if (status === "blocked") { - return "blocked"; - } - } - return isError ? "error" : "completed"; -} - function withDiagnosticTerminalType( response: T, terminalType: CodexDynamicToolDiagnosticTerminalType, @@ -1241,6 +1261,24 @@ function withDiagnosticTerminalType( return response; } +function withDiagnosticFailureDisposition( + response: T, + disposition: "blocked" | CodexDynamicToolDiagnosticTerminalReason | undefined, +): T { + if (!disposition) { + return response; + } + withDiagnosticTerminalType(response, disposition === "blocked" ? "blocked" : "error"); + if (disposition !== "blocked") { + Object.defineProperty(response, "diagnosticTerminalReason", { + configurable: true, + enumerable: false, + value: disposition, + }); + } + return response; +} + function withSideEffectEvidence( response: T, sideEffectEvidence: boolean, diff --git a/extensions/codex/src/app-server/event-projector.test.ts b/extensions/codex/src/app-server/event-projector.test.ts index d0e6630e87f6..c007b9cba504 100644 --- a/extensions/codex/src/app-server/event-projector.test.ts +++ b/extensions/codex/src/app-server/event-projector.test.ts @@ -1827,6 +1827,7 @@ describe("CodexAppServerEventProjector", () => { try { await projector.handleNotification( forCurrentTurn("item/started", { + startedAtMs: 1_750_000_000_000, item: { type: "commandExecution", id: "cmd-1", @@ -1844,6 +1845,7 @@ describe("CodexAppServerEventProjector", () => { ); await projector.handleNotification( forCurrentTurn("item/completed", { + completedAtMs: 1_750_000_000_042, item: { type: "commandExecution", id: "cmd-1", @@ -1912,6 +1914,7 @@ describe("CodexAppServerEventProjector", () => { toolName: event.toolName, toolCallId: event.toolCallId, durationMs: "durationMs" in event ? event.durationMs : undefined, + sourceTimestampMs: event.sourceTimestampMs, })), ).toEqual([ { @@ -1919,12 +1922,14 @@ describe("CodexAppServerEventProjector", () => { toolName: "bash", toolCallId: "cmd-1", durationMs: undefined, + sourceTimestampMs: 1_750_000_000_000, }, { type: "tool.execution.completed", toolName: "bash", toolCallId: "cmd-1", durationMs: 42, + sourceTimestampMs: 1_750_000_000_042, }, ]); const result = projector.buildResult(buildEmptyToolTelemetry()); @@ -1958,6 +1963,521 @@ describe("CodexAppServerEventProjector", () => { expect(toolResultContentItem.content).toBe("ok"); }); + it.each([ + ["cancelled", "cancelled"], + [Object.assign(new Error("turn timed out"), { name: "TimeoutError" }), "timed_out"], + ] as const)( + "preserves enclosing %s provenance for failed native tools", + async (abortReason, terminalReason) => { + const abortController = new AbortController(); + abortController.abort(abortReason); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(undefined, { + runAbortSignal: abortController.signal, + }); + const commandItem = { + type: "commandExecution", + id: "cmd-aborted", + command: "pnpm test extensions/codex", + cwd: "/workspace", + processId: null, + source: "agent", + status: "inProgress", + commandActions: [], + aggregatedOutput: null, + exitCode: null, + durationMs: null, + }; + + try { + await projector.handleNotification(forCurrentTurn("item/started", { item: commandItem })); + await projector.handleNotification( + forCurrentTurn("item/completed", { + item: { ...commandItem, status: "failed", durationMs: 4 }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "cmd-aborted", + terminalReason, + }), + ); + }, + ); + + it.each([ + ["cancelled", "cancelled"], + [Object.assign(new Error("turn timed out"), { name: "TimeoutError" }), "timed_out"], + ] as const)( + "finalizes an active native tool as %s when building an interrupted result", + async (abortReason, terminalReason) => { + const abortController = new AbortController(); + abortController.abort(abortReason); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(undefined, { + runAbortSignal: abortController.signal, + }); + + try { + await projector.handleNotification( + forCurrentTurn("item/started", { + item: { + type: "commandExecution", + id: "cmd-active-abort", + command: "pnpm test extensions/codex", + cwd: "/workspace", + processId: null, + source: "agent", + status: "inProgress", + commandActions: [], + aggregatedOutput: null, + exitCode: null, + durationMs: null, + }, + }), + ); + projector.buildResult(buildEmptyToolTelemetry()); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "cmd-active-abort", + terminalReason, + }), + ); + expect( + diagnosticEvents + .filter((event) => "toolCallId" in event && event.toolCallId === "cmd-active-abort") + .map((event) => event.type), + ).toEqual(["tool.execution.started", "tool.execution.error"]); + }, + ); + + it.each([ + [ + "collaboration", + { + id: "collab-audit-1", + type: "collabAgentToolCall", + tool: "spawnAgent", + status: "completed", + senderThreadId: THREAD_ID, + receiverThreadIds: ["child-thread-1"], + prompt: "sensitive prompt text", + model: null, + reasoningEffort: null, + agentsStates: {}, + }, + "collab.spawnAgent", + ], + [ + "image generation", + { + id: "image-generation-audit-1", + type: "imageGeneration", + status: "completed", + revisedPrompt: "sensitive revised prompt", + result: "sensitive image payload", + }, + "image_generation", + ], + [ + "image view", + { + id: "image-view-audit-1", + type: "imageView", + path: "/workspace/sensitive-filename.png", + }, + "image_view", + ], + [ + "sleep", + { + id: "sleep-audit-1", + type: "sleep", + durationMs: 250, + }, + "sleep", + ], + ] as const)( + "emits metadata-only lifecycle diagnostics for native %s items", + async (_, item, toolName) => { + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(); + + try { + await projector.handleNotification( + forCurrentTurn("item/started", { item, startedAtMs: 1_750_000_000_000 }), + ); + await projector.handleNotification( + forCurrentTurn("item/completed", { item, completedAtMs: 1_750_000_000_042 }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter((event) => "toolCallId" in event && event.toolCallId === item.id) + .map((event) => ({ + type: event.type, + toolName: "toolName" in event ? event.toolName : null, + })), + ).toEqual([ + { type: "tool.execution.started", toolName }, + { type: "tool.execution.completed", toolName }, + ]); + expect(JSON.stringify(diagnosticEvents)).not.toContain("sensitive"); + }, + ); + + it.each([ + ["completed", "tool.execution.completed", undefined, undefined], + ["failed", "tool.execution.error", "failed", undefined], + ["cancelled", "tool.execution.error", "cancelled", undefined], + [undefined, "tool.execution.error", "failed", "tool_outcome_unknown"], + ["future_status", "tool.execution.error", "failed", "tool_outcome_unknown"], + ] as const)( + "uses raw %s status for redacted native web-search audit actions", + async (status, terminalType, terminalReason, errorCode) => { + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(); + const item = { + id: "web-search-audit-1", + type: "webSearch", + query: "sensitive query", + action: { type: "search", query: "sensitive query", queries: null }, + }; + + try { + await projector.handleNotification( + forCurrentTurn("item/started", { item, startedAtMs: 1_750_000_000_000 }), + ); + await projector.handleNotification( + forCurrentTurn("item/completed", { item, completedAtMs: 1_750_000_000_042 }), + ); + await projector.handleNotification( + forCurrentTurn("rawResponseItem/completed", { + item: { + id: item.id, + type: "web_search_call", + status, + action: item.action, + }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter((event) => "toolCallId" in event && event.toolCallId === item.id) + .map((event) => ({ + type: event.type, + toolName: "toolName" in event ? event.toolName : null, + terminalReason: "terminalReason" in event ? event.terminalReason : undefined, + errorCode: "errorCode" in event ? event.errorCode : undefined, + sourceTimestampMs: "sourceTimestampMs" in event ? event.sourceTimestampMs : undefined, + })), + ).toEqual([ + { + type: "tool.execution.started", + toolName: "web_search", + terminalReason: undefined, + errorCode: undefined, + sourceTimestampMs: 1_750_000_000_000, + }, + { + type: terminalType, + toolName: "web_search", + terminalReason, + errorCode, + sourceTimestampMs: 1_750_000_000_042, + }, + ]); + expect(JSON.stringify(diagnosticEvents)).not.toContain("sensitive"); + }, + ); + + it("keeps raw open-page status unknown until explicit completion", async () => { + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(); + const item = { + id: "web-search-open-page-1", + type: "webSearch", + query: "", + action: { type: "openPage", url: "https://example.com/sensitive" }, + }; + + try { + await projector.handleNotification(forCurrentTurn("item/started", { item })); + await projector.handleNotification(forCurrentTurn("item/completed", { item })); + await projector.handleNotification( + forCurrentTurn("rawResponseItem/completed", { + item: { + id: item.id, + type: "web_search_call", + status: "open", + action: { type: "open_page", url: "https://example.com/sensitive" }, + }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter((event) => "toolCallId" in event && event.toolCallId === item.id) + .map((event) => ({ + type: event.type, + terminalReason: "terminalReason" in event ? event.terminalReason : undefined, + errorCode: "errorCode" in event ? event.errorCode : undefined, + })), + ).toEqual([ + { + type: "tool.execution.started", + terminalReason: undefined, + errorCode: undefined, + }, + { + type: "tool.execution.error", + terminalReason: "failed", + errorCode: "tool_outcome_unknown", + }, + ]); + expect(JSON.stringify(diagnosticEvents)).not.toContain("sensitive"); + }); + + it("keeps native web-search outcomes unknown at finalization when no raw terminal arrives", async () => { + const abortController = new AbortController(); + abortController.abort("cancelled"); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(undefined, { + runAbortSignal: abortController.signal, + }); + const item = { + id: "web-search-without-raw-terminal", + type: "webSearch", + query: "sensitive extension query", + action: { type: "search", query: "sensitive extension query", queries: null }, + }; + + try { + await projector.handleNotification(forCurrentTurn("item/started", { item })); + await projector.handleNotification(forCurrentTurn("item/completed", { item })); + projector.buildResult(buildEmptyToolTelemetry()); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter((event) => "toolCallId" in event && event.toolCallId === item.id) + .map((event) => ({ + type: event.type, + terminalReason: "terminalReason" in event ? event.terminalReason : undefined, + errorCode: "errorCode" in event ? event.errorCode : undefined, + })), + ).toEqual([ + { + type: "tool.execution.started", + terminalReason: undefined, + errorCode: undefined, + }, + { + type: "tool.execution.error", + terminalReason: "failed", + errorCode: "tool_outcome_unknown", + }, + ]); + expect(JSON.stringify(diagnosticEvents)).not.toContain("sensitive extension query"); + }); + + it.each([ + [ + "web search", + "cancelled", + { + id: "web-search-started-only", + type: "webSearch", + query: "sensitive query", + action: { type: "search", query: "sensitive query", queries: null }, + }, + ], + [ + "image generation", + Object.assign(new Error("turn timed out"), { name: "TimeoutError" }), + { + id: "image-generation-started-only", + type: "imageGeneration", + status: "in_progress", + revisedPrompt: "sensitive prompt", + result: null, + }, + ], + ] as const)( + "keeps started-only native %s outcomes unknown when the enclosing run stops", + async (_, abortReason, item) => { + const abortController = new AbortController(); + abortController.abort(abortReason); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(undefined, { + runAbortSignal: abortController.signal, + }); + + try { + await projector.handleNotification(forCurrentTurn("item/started", { item })); + projector.buildResult(buildEmptyToolTelemetry()); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter((event) => "toolCallId" in event && event.toolCallId === item.id) + .map((event) => ({ + type: event.type, + terminalReason: "terminalReason" in event ? event.terminalReason : undefined, + errorCode: "errorCode" in event ? event.errorCode : undefined, + })), + ).toEqual([ + { + type: "tool.execution.started", + terminalReason: undefined, + errorCode: undefined, + }, + { + type: "tool.execution.error", + terminalReason: "failed", + errorCode: "tool_outcome_unknown", + }, + ]); + expect(JSON.stringify(diagnosticEvents)).not.toContain("sensitive"); + }, + ); + + it("projects native image-generation error status as a failed audit action", async () => { + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(); + const startedItem = { + id: "image-generation-error-1", + type: "imageGeneration", + status: "in_progress", + revisedPrompt: null, + result: null, + }; + + try { + await projector.handleNotification(forCurrentTurn("item/started", { item: startedItem })); + await projector.handleNotification( + forCurrentTurn("item/completed", { + item: { ...startedItem, status: "error" }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter((event) => "toolCallId" in event && event.toolCallId === startedItem.id) + .map((event) => ({ + type: event.type, + terminalReason: "terminalReason" in event ? event.terminalReason : undefined, + })), + ).toEqual([ + { type: "tool.execution.started", terminalReason: undefined }, + { type: "tool.execution.error", terminalReason: "failed" }, + ]); + }); + + it.each([ + ["missing", undefined, undefined], + ["in-progress", "in_progress", undefined], + [ + "unrecognized", + "future_status", + Object.assign(new Error("turn timed out"), { name: "TimeoutError" }), + ], + ] as const)( + "keeps %s native image-generation terminal status non-successful", + async (_, status, abortReason) => { + const abortController = new AbortController(); + if (abortReason) { + abortController.abort(abortReason); + } + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const projector = await createProjector(undefined, { + runAbortSignal: abortController.signal, + }); + const startedItem = { + id: `image-generation-${status ?? "missing"}`, + type: "imageGeneration", + status: "in_progress", + revisedPrompt: null, + result: null, + }; + + try { + await projector.handleNotification(forCurrentTurn("item/started", { item: startedItem })); + await projector.handleNotification( + forCurrentTurn("item/completed", { item: { ...startedItem, status } }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter((event) => "toolCallId" in event && event.toolCallId === startedItem.id) + .map((event) => ({ + type: event.type, + terminalReason: "terminalReason" in event ? event.terminalReason : undefined, + errorCode: "errorCode" in event ? event.errorCode : undefined, + })), + ).toEqual([ + { + type: "tool.execution.started", + terminalReason: undefined, + errorCode: undefined, + }, + { + type: "tool.execution.error", + terminalReason: "failed", + errorCode: "tool_outcome_unknown", + }, + ]); + }, + ); + it("synthesizes native tool progress from turn completion snapshots", async () => { const onAgentEvent = vi.fn(); const onToolResult = vi.fn(); @@ -2349,6 +2869,13 @@ describe("CodexAppServerEventProjector", () => { exitCode: null, durationMs: null, }, + { + type: "imageGeneration", + id: "image-running-snapshot", + status: "in_progress", + revisedPrompt: null, + result: null, + }, ]), ); @@ -2496,6 +3023,196 @@ describe("CodexAppServerEventProjector", () => { }); }); + it.each(["failed", "cancelled", "timed_out"] as const)( + "projects a declined native approval with %s disposition as one terminal error", + async (disposition) => { + const projector = await createProjector(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + + try { + await projector.handleNotification( + forCurrentTurn("item/started", { + item: { + type: "commandExecution", + id: "cmd-approval-failure", + command: "pnpm test extensions/codex", + cwd: "/workspace", + processId: null, + source: "agent", + status: "inProgress", + commandActions: [], + aggregatedOutput: null, + exitCode: null, + durationMs: null, + }, + }), + ); + projector.recordNativeToolApprovalFailure("cmd-approval-failure", disposition); + await projector.handleNotification( + forCurrentTurn("item/completed", { + item: { + type: "commandExecution", + id: "cmd-approval-failure", + command: "pnpm test extensions/codex", + cwd: "/workspace", + processId: null, + source: "agent", + status: "declined", + commandActions: [], + aggregatedOutput: null, + exitCode: null, + durationMs: 1, + }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter((event) => event.type.startsWith("tool.execution.")) + .map((event) => + "terminalReason" in event + ? { type: event.type, terminalReason: event.terminalReason } + : { type: event.type }, + ), + ).toEqual([ + { type: "tool.execution.started" }, + { type: "tool.execution.error", terminalReason: disposition }, + ]); + }, + ); + + it("coalesces a native pre-tool failure with the matching item terminal", async () => { + const projector = await createProjector(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + const item = { + type: "commandExecution" as const, + id: "cmd-pre-tool-failure", + command: "pnpm test extensions/codex", + cwd: "/workspace", + processId: null, + source: "agent" as const, + commandActions: [], + aggregatedOutput: null, + exitCode: null, + }; + + try { + projector.recordNativeToolPreToolUseFailure({ + toolName: "exec", + toolCallId: item.id, + disposition: "timed_out", + durationMs: 5, + }); + await projector.handleNotification( + forCurrentTurn("item/started", { + item: { ...item, status: "inProgress", durationMs: null }, + }), + ); + await projector.handleNotification( + forCurrentTurn("item/completed", { + item: { ...item, status: "declined", durationMs: 7 }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents + .filter( + (event) => + event.type.startsWith("tool.execution.") && + "toolCallId" in event && + event.toolCallId === item.id, + ) + .map((event) => + event.type === "tool.execution.error" + ? { + type: event.type, + toolName: event.toolName, + durationMs: event.durationMs, + errorCategory: event.errorCategory, + terminalReason: event.terminalReason, + } + : { + type: event.type, + toolName: "toolName" in event ? event.toolName : undefined, + }, + ), + ).toEqual([ + { type: "tool.execution.started", toolName: "bash" }, + { + type: "tool.execution.error", + toolName: "bash", + durationMs: 7, + errorCategory: "before_tool_call", + terminalReason: "timed_out", + }, + ]); + }); + + it("finalizes a native pre-tool failure when no item arrives", async () => { + const runAbortController = new AbortController(); + const projector = await createProjector(undefined, { + runAbortSignal: runAbortController.signal, + }); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); + + try { + projector.recordNativeToolPreToolUseFailure({ + toolName: "exec", + toolCallId: "native-no-item", + disposition: "failed", + durationMs: 5, + }); + runAbortController.abort("codex_side_question_finished"); + projector.buildResult(buildEmptyToolTelemetry()); + projector.recordNativeToolPreToolUseFailure({ + toolName: "exec", + toolCallId: "native-late-no-item", + disposition: "failed", + durationMs: 6, + }); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect( + diagnosticEvents.filter( + (event) => + event.type.startsWith("tool.execution.") && + "toolCallId" in event && + (event.toolCallId === "native-no-item" || event.toolCallId === "native-late-no-item"), + ), + ).toEqual([ + expect.objectContaining({ + type: "tool.execution.error", + toolName: "exec", + toolCallId: "native-no-item", + durationMs: 5, + errorCategory: "before_tool_call", + terminalReason: "failed", + }), + expect.objectContaining({ + type: "tool.execution.error", + toolName: "exec", + toolCallId: "native-late-no-item", + durationMs: 6, + errorCategory: "before_tool_call", + terminalReason: "cancelled", + }), + ]); + }); + it("clears a recovered declined native tool error", async () => { const projector = await createProjector(); diff --git a/extensions/codex/src/app-server/event-projector.ts b/extensions/codex/src/app-server/event-projector.ts index f7d21ee6122b..1f7360150d37 100644 --- a/extensions/codex/src/app-server/event-projector.ts +++ b/extensions/codex/src/app-server/event-projector.ts @@ -13,6 +13,7 @@ import { runAgentHarnessBeforeCompactionHook, TOOL_PROGRESS_OUTPUT_MAX_CHARS, type AgentMessage, + type BeforeToolCallFailureDisposition, type EmbeddedRunAttemptParams, type EmbeddedRunAttemptResult, type HeartbeatToolResponse, @@ -25,7 +26,12 @@ import { generatedImageAssetFromBase64 } from "openclaw/plugin-sdk/image-generat import type { AssistantMessage, Usage } from "openclaw/plugin-sdk/llm"; import { saveMediaBuffer } from "openclaw/plugin-sdk/media-store"; import { asDateTimestampMs } from "openclaw/plugin-sdk/number-runtime"; +import { resolveCodexToolAbortTerminalReason } from "./dynamic-tool-execution.js"; import { resolveCodexLocalRuntimeAttribution } from "./local-runtime-attribution.js"; +import { + emitCodexNativePreToolUseFailureDiagnostic, + type CodexNativePreToolUseFailure, +} from "./native-hook-relay.js"; import { readCodexNotificationThreadId, readCodexNotificationTurnId, @@ -67,9 +73,358 @@ export type CodexAppServerToolTelemetry = { export type CodexAppServerEventProjectorOptions = { nativePostToolUseRelayEnabled?: boolean; onNativeToolResultRecorded?: () => void | Promise; + runAbortSignal?: AbortSignal; trajectoryRecorder?: CodexTrajectoryRecorder | null; }; +type CodexNativeToolLifecycleContext = Pick< + EmbeddedRunAttemptParams, + "agentId" | "runId" | "sessionId" | "sessionKey" +>; + +type CodexNativeToolLifecycleProjectorOptions = { + runAbortSignal?: AbortSignal; +}; + +type CodexNativeToolAuditStatus = ReturnType | "cancelled" | "unknown"; +type CodexNativeToolUnfinishedStatus = Extract; +type CodexNativePreToolUseFailureRecord = { + failure: CodexNativePreToolUseFailure; + terminalReason: CodexNativePreToolUseFailure["disposition"]; +}; + +/** Projects metadata-only lifecycle diagnostics for native tool items. */ +export class CodexNativeToolLifecycleProjector { + private readonly startedAtByItem = new Map(); + private readonly activeItems = new Map< + string, + { toolName: string; unfinishedStatus: CodexNativeToolUnfinishedStatus } + >(); + private readonly webSearchCompletionByItem = new Map< + string, + { runWasAborted: boolean; sourceTimestampMs?: number } + >(); + private readonly completedItemIds = new Set(); + private readonly approvalFailureDispositionByItem = new Map< + string, + Exclude + >(); + private readonly preToolUseFailureByItem = new Map(); + private finalized = false; + + constructor( + private readonly context: CodexNativeToolLifecycleContext, + private readonly threadId: string, + private readonly turnId: string, + private readonly options: CodexNativeToolLifecycleProjectorOptions = {}, + ) {} + + handleNotification(notification: CodexServerNotification): void { + const params = isJsonObject(notification.params) ? notification.params : undefined; + if ( + !params || + readCodexNotificationThreadId(params) !== this.threadId || + readCodexNotificationTurnId(params) !== this.turnId + ) { + return; + } + if (notification.method === "turn/completed") { + const turn = readCodexTurn(params.turn); + if (!turn || turn.id !== this.turnId) { + return; + } + for (const item of turn.items ?? []) { + this.recordSnapshotItem(item); + } + return; + } + if (notification.method === "rawResponseItem/completed") { + const item = isJsonObject(params.item) ? params.item : undefined; + if (item) { + this.recordRawWebSearchResult(item); + } + return; + } + if (notification.method !== "item/started" && notification.method !== "item/completed") { + return; + } + const item = readItem(params.item); + if (!item) { + return; + } + this.recordItem({ + phase: notification.method === "item/started" ? "start" : "result", + item, + sourceTimestampMs: asDateTimestampMs( + notification.method === "item/started" ? params.startedAtMs : params.completedAtMs, + ), + }); + } + + recordItem(params: { + phase: "start" | "result"; + item: CodexThreadItem; + sourceTimestampMs?: number; + }): void { + const toolName = auditNativeToolName(params.item); + if (!toolName || this.completedItemIds.has(params.item.id)) { + return; + } + if (params.phase === "start") { + this.recordStarted( + params.item.id, + toolName, + auditNativeToolUnfinishedStatus(params.item), + params.sourceTimestampMs, + ); + return; + } + if (params.item.type === "webSearch") { + // Warm resumes retain raw-event delivery, while cold resumes expose no + // capability bit. Wait through the drain; finalization closes misses unknown. + this.webSearchCompletionByItem.set(params.item.id, { + runWasAborted: this.options.runAbortSignal?.aborted === true, + sourceTimestampMs: params.sourceTimestampMs, + }); + return; + } + + const itemDurationMs = + typeof params.item.durationMs === "number" ? params.item.durationMs : undefined; + this.recordTerminal(params.item.id, toolName, auditNativeToolTerminalStatus(params.item), { + itemDurationMs, + sourceTimestampMs: params.sourceTimestampMs, + }); + } + + recordApprovalFailureDisposition( + toolCallId: string, + disposition: Exclude, + ): void { + if (!this.completedItemIds.has(toolCallId)) { + this.approvalFailureDispositionByItem.set(toolCallId, disposition); + } + } + + recordPreToolUseFailure( + failure: CodexNativePreToolUseFailure, + runWasAborted = this.options.runAbortSignal?.aborted === true, + ): void { + if (this.completedItemIds.has(failure.toolCallId)) { + return; + } + const record: CodexNativePreToolUseFailureRecord = { + failure, + terminalReason: + runWasAborted && this.options.runAbortSignal + ? resolveCodexToolAbortTerminalReason(this.options.runAbortSignal) + : failure.disposition, + }; + if (this.finalized) { + // Relay subprocesses can settle after result construction. Emit the + // item-less fallback here because no later notification drain remains. + this.completedItemIds.add(failure.toolCallId); + this.emitPreToolUseFailure(record, failure.toolName, failure.durationMs); + return; + } + this.preToolUseFailureByItem.set(failure.toolCallId, record); + } + + private recordRawWebSearchResult(item: JsonObject): void { + if (readString(item, "type") !== "web_search_call") { + return; + } + const toolCallId = readString(item, "id"); + if (!toolCallId || this.completedItemIds.has(toolCallId)) { + return; + } + const toolName = "web_search"; + this.recordStarted(toolCallId, toolName, "unknown"); + const rawStatus = readString(item, "status"); + if (rawStatus === "in_progress" || rawStatus === "running") { + return; + } + const status: CodexNativeToolAuditStatus = + rawStatus === "completed" + ? "completed" + : rawStatus === "cancelled" + ? "cancelled" + : rawStatus === "failed" || rawStatus === "error" || rawStatus === "incomplete" + ? "failed" + : "unknown"; + this.recordTerminal(toolCallId, toolName, status, { + sourceTimestampMs: this.webSearchCompletionByItem.get(toolCallId)?.sourceTimestampMs, + }); + } + + private recordTerminal( + toolCallId: string, + toolName: string, + status: CodexNativeToolAuditStatus, + options: { + itemDurationMs?: number; + sourceTimestampMs?: number; + runWasAborted?: boolean; + } = {}, + ): void { + const runWasAborted = options.runWasAborted ?? this.options.runAbortSignal?.aborted === true; + const preToolUseFailure = this.preToolUseFailureByItem.get(toolCallId); + this.preToolUseFailureByItem.delete(toolCallId); + const approvalFailureDisposition = this.approvalFailureDispositionByItem.get(toolCallId); + this.approvalFailureDispositionByItem.delete(toolCallId); + this.completedItemIds.add(toolCallId); + this.activeItems.delete(toolCallId); + this.webSearchCompletionByItem.delete(toolCallId); + const startedAt = this.startedAtByItem.get(toolCallId); + this.startedAtByItem.delete(toolCallId); + const endedAt = options.sourceTimestampMs ?? Date.now(); + const durationMs = + options.itemDurationMs ?? (startedAt === undefined ? 0 : Math.max(0, endedAt - startedAt)); + if (preToolUseFailure) { + this.emitPreToolUseFailure( + preToolUseFailure, + toolName, + durationMs, + options.sourceTimestampMs, + ); + return; + } + const terminalEvent = approvalFailureDisposition + ? { + type: "tool.execution.error" as const, + durationMs, + errorCategory: "codex_native_tool_approval", + terminalReason: approvalFailureDisposition, + } + : status === "blocked" + ? { + type: "tool.execution.blocked" as const, + reason: "codex_native_tool_blocked", + deniedReason: "codex_native_tool_blocked", + } + : status === "failed" || status === "cancelled" || status === "unknown" + ? { + type: "tool.execution.error" as const, + durationMs, + errorCategory: + status === "unknown" + ? "codex_native_tool_outcome_unknown" + : status === "cancelled" + ? "aborted" + : "codex_native_tool_error", + ...(status === "unknown" ? { errorCode: "tool_outcome_unknown" } : {}), + terminalReason: + // An enclosing abort explains unfinished work, but cannot classify + // a native terminal whose status is absent or unrecognized. + status === "unknown" + ? ("failed" as const) + : runWasAborted && this.options.runAbortSignal + ? resolveCodexToolAbortTerminalReason(this.options.runAbortSignal) + : status === "cancelled" + ? ("cancelled" as const) + : ("failed" as const), + } + : { + type: "tool.execution.completed" as const, + durationMs, + }; + emitTrustedDiagnosticEvent({ + ...this.buildBase(toolCallId, toolName), + ...terminalEvent, + ...(options.sourceTimestampMs !== undefined + ? { sourceTimestampMs: options.sourceTimestampMs } + : {}), + }); + } + + finalizeActive(runWasAborted = this.options.runAbortSignal?.aborted === true): void { + this.finalized = true; + for (const [toolCallId, { toolName, unfinishedStatus }] of this.activeItems) { + const webSearchCompletion = this.webSearchCompletionByItem.get(toolCallId); + const itemRunWasAborted = webSearchCompletion + ? webSearchCompletion.runWasAborted + : runWasAborted; + this.recordTerminal(toolCallId, toolName, unfinishedStatus, { + runWasAborted: itemRunWasAborted, + sourceTimestampMs: webSearchCompletion?.sourceTimestampMs, + }); + } + for (const [toolCallId, record] of this.preToolUseFailureByItem) { + if (!this.completedItemIds.has(toolCallId)) { + this.recordTerminal(toolCallId, record.failure.toolName, "failed", { + itemDurationMs: record.failure.durationMs, + }); + } + } + this.activeItems.clear(); + this.webSearchCompletionByItem.clear(); + this.approvalFailureDispositionByItem.clear(); + this.preToolUseFailureByItem.clear(); + } + + private emitPreToolUseFailure( + record: CodexNativePreToolUseFailureRecord, + toolName: string, + durationMs: number, + sourceTimestampMs?: number, + ): void { + emitCodexNativePreToolUseFailureDiagnostic({ + agentId: this.context.agentId, + sessionId: this.context.sessionId, + sessionKey: this.context.sessionKey, + runId: this.context.runId, + failure: { ...record.failure, toolName, durationMs }, + terminalReason: record.terminalReason, + sourceTimestampMs, + }); + } + + private recordSnapshotItem(item: CodexThreadItem): void { + if ( + !auditNativeToolName(item) || + this.completedItemIds.has(item.id) || + itemStatus(item) === "running" + ) { + return; + } + const toolName = auditNativeToolName(item); + if (!toolName) { + return; + } + this.recordStarted(item.id, toolName, auditNativeToolUnfinishedStatus(item)); + this.recordItem({ phase: "result", item }); + } + + private recordStarted( + toolCallId: string, + toolName: string, + unfinishedStatus: CodexNativeToolUnfinishedStatus, + sourceTimestampMs?: number, + ): void { + if (this.activeItems.has(toolCallId)) { + return; + } + this.startedAtByItem.set(toolCallId, sourceTimestampMs ?? Date.now()); + this.activeItems.set(toolCallId, { toolName, unfinishedStatus }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + ...this.buildBase(toolCallId, toolName), + ...(sourceTimestampMs !== undefined ? { sourceTimestampMs } : {}), + }); + } + + private buildBase(toolCallId: string, toolName: string) { + return { + agentId: this.context.agentId, + runId: this.context.runId, + sessionId: this.context.sessionId, + sessionKey: this.context.sessionKey, + toolName, + toolCallId, + }; + } +} + type ReasoningDeltaMethod = "item/reasoning/summaryTextDelta" | "item/reasoning/textDelta"; type ReasoningTextGroup = { @@ -198,7 +553,7 @@ export class CodexAppServerEventProjector { private lastNativeToolError: EmbeddedRunAttemptResult["lastToolError"]; private readonly nativeGeneratedMediaItemIds = new Set(); private readonly nativeGeneratedMediaUrlsByItemId = new Map(); - private readonly diagnosticToolStartedAtByItem = new Map(); + private readonly nativeToolLifecycleProjector: CodexNativeToolLifecycleProjector; private readonly afterToolCallObservedItemIds = new Set(); private assistantStarted = false; private reasoningStarted = false; @@ -220,7 +575,16 @@ export class CodexAppServerEventProjector { private readonly threadId: string, private readonly turnId: string, private readonly options: CodexAppServerEventProjectorOptions = {}, - ) {} + ) { + this.nativeToolLifecycleProjector = new CodexNativeToolLifecycleProjector( + params, + threadId, + turnId, + { + runAbortSignal: options.runAbortSignal, + }, + ); + } getCompletedTurnStatus(): CodexTurn["status"] | undefined { return this.completedTurn?.status; @@ -295,6 +659,17 @@ export class CodexAppServerEventProjector { } } + recordNativeToolApprovalFailure( + toolCallId: string, + disposition: Exclude, + ): void { + this.nativeToolLifecycleProjector.recordApprovalFailureDisposition(toolCallId, disposition); + } + + recordNativeToolPreToolUseFailure(failure: CodexNativePreToolUseFailure): void { + this.nativeToolLifecycleProjector.recordPreToolUseFailure(failure); + } + async handleNotification(notification: CodexServerNotification): Promise { const params = isJsonObject(notification.params) ? notification.params : undefined; if (!params) { @@ -312,6 +687,7 @@ export class CodexAppServerEventProjector { } else if (!this.isNotificationForTurn(params)) { return; } + this.nativeToolLifecycleProjector.handleNotification(notification); switch (notification.method) { case "item/agentMessage/delta": @@ -372,6 +748,9 @@ export class CodexAppServerEventProjector { toolTelemetry: CodexAppServerToolTelemetry, options?: { yieldDetected?: boolean }, ): EmbeddedRunAttemptResult { + // Result construction runs after the notification queue drains. Close any + // tool lacking a terminal item so audit consumers never retain an open action. + this.nativeToolLifecycleProjector.finalizeActive(); const assistantTexts = this.collectAssistantTexts(); const reasoningText = collectReasoningTextValues( this.reasoningTextByGroup, @@ -1353,7 +1732,6 @@ export class CodexAppServerEventProjector { const args = itemToolArgs(item); const meta = itemMeta(item, this.toolProgressDetailMode()); this.recordToolTrajectoryEvent({ phase: params.phase, item, name, args, status }); - this.emitDiagnosticToolExecutionEvent({ phase: params.phase, item, name, status }); if (params.phase === "result") { this.recordNativeToolError({ item, name, meta, status }); } @@ -1480,54 +1858,6 @@ export class CodexAppServerEventProjector { }); } - private emitDiagnosticToolExecutionEvent(params: { - phase: "start" | "result"; - item: CodexThreadItem; - name: string; - status: ReturnType; - }): void { - const base = { - runId: this.params.runId, - sessionId: this.params.sessionId, - sessionKey: this.params.sessionKey, - toolName: params.name, - toolCallId: params.item.id, - }; - if (params.phase === "start") { - this.diagnosticToolStartedAtByItem.set(params.item.id, Date.now()); - emitTrustedDiagnosticEvent({ - type: "tool.execution.started", - ...base, - }); - return; - } - - const startedAt = this.diagnosticToolStartedAtByItem.get(params.item.id); - this.diagnosticToolStartedAtByItem.delete(params.item.id); - const itemDurationMs = - typeof params.item.durationMs === "number" ? params.item.durationMs : undefined; - const durationMs = - itemDurationMs ?? (startedAt === undefined ? 0 : Math.max(0, Date.now() - startedAt)); - const terminalEvent = - params.status === "blocked" - ? { - type: "tool.execution.blocked" as const, - reason: "codex_native_tool_blocked", - deniedReason: "codex_native_tool_blocked", - } - : params.status === "failed" - ? { - type: "tool.execution.error" as const, - durationMs, - errorCategory: "codex_native_tool_error", - } - : { - type: "tool.execution.completed" as const, - durationMs, - }; - emitTrustedDiagnosticEvent({ ...base, ...terminalEvent }); - } - private emitAfterToolCallObservation(item: CodexThreadItem): void { if (!this.shouldEmitAfterToolCallObservation(item)) { return; @@ -2410,18 +2740,45 @@ function itemTitle(item: CodexThreadItem): string { function itemStatus(item: CodexThreadItem): "completed" | "failed" | "running" | "blocked" { const status = readItemString(item, "status"); - if (status === "failed") { + if (status === "failed" || status === "error") { return "failed"; } if (status === "declined") { return "blocked"; } - if (status === "inProgress" || status === "running") { + if (status === "inProgress" || status === "in_progress" || status === "running") { return "running"; } return "completed"; } +function auditNativeToolTerminalStatus(item: CodexThreadItem): CodexNativeToolAuditStatus { + if (item.type === "imageView" || item.type === "sleep") { + return "completed"; + } + const status = readItemString(item, "status"); + if (status === "completed") { + return "completed"; + } + if (status === "failed" || status === "error") { + return "failed"; + } + if (status === "declined") { + return "blocked"; + } + // A completed notification with a missing, active, or new status does not + // prove success. Preserve that ambiguity at the durable audit boundary. + return "unknown"; +} + +function auditNativeToolUnfinishedStatus( + item: CodexThreadItem, +): CodexNativeToolUnfinishedStatus { + // Search and image generation publish explicit terminal states. An enclosing + // run outcome cannot substitute when that dependency-owned state is absent. + return item.type === "webSearch" || item.type === "imageGeneration" ? "unknown" : "failed"; +} + function formatMissingToolResultError(params: { id: string; name: string }): string { return `${MISSING_TOOL_RESULT_ERROR} toolCallId=${params.id}; toolName=${params.name}`; } @@ -2450,6 +2807,31 @@ function itemName(item: CodexThreadItem): string | undefined { return undefined; } +function auditNativeToolName(item: CodexThreadItem): string | undefined { + if (item.type === "dynamicToolCall") { + return undefined; + } + const progressName = itemName(item); + if (progressName) { + return progressName; + } + if (item.type === "collabAgentToolCall") { + return typeof item.tool === "string" && item.tool.trim() + ? `collab.${item.tool.trim()}` + : "collab_agent"; + } + if (item.type === "imageGeneration") { + return "image_generation"; + } + if (item.type === "imageView") { + return "image_view"; + } + if (item.type === "sleep") { + return "sleep"; + } + return undefined; +} + function isSideEffectingNativeToolItem(item: CodexThreadItem): boolean { return ( itemStatus(item) !== "blocked" && diff --git a/extensions/codex/src/app-server/native-hook-relay.test.ts b/extensions/codex/src/app-server/native-hook-relay.test.ts index 1bcc2e2253a7..c29e4a696142 100644 --- a/extensions/codex/src/app-server/native-hook-relay.test.ts +++ b/extensions/codex/src/app-server/native-hook-relay.test.ts @@ -1,14 +1,28 @@ // Codex tests cover native hook relay plugin behavior. import type { NativeHookRelayRegistrationHandle } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + onInternalDiagnosticEvent, + resetDiagnosticEventsForTest, + type DiagnosticEventPayload, +} from "openclaw/plugin-sdk/diagnostic-runtime"; import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import { buildCodexNativeHookRelayConfig, buildCodexNativeHookRelayDisabledConfig, + emitCodexNativePreToolUseFailureDiagnostic, resolveCodexNativeHookRelayCommandTimeoutMs, resolveCodexNativeHookRelayUnregisterGraceMs, } from "./native-hook-relay.js"; +afterEach(() => resetDiagnosticEventsForTest()); + +function flushDiagnosticEvents(): Promise { + return new Promise((resolve) => { + setImmediate(resolve); + }); +} + describe("Codex native hook relay config", () => { it("builds deterministic Codex config overrides with command hooks", () => { const config = buildCodexNativeHookRelayConfig({ @@ -302,6 +316,54 @@ describe("Codex native hook relay config", () => { MAX_TIMER_TIMEOUT_MS, ); }); + + it.each([ + { reason: "turn_progress_idle_timeout", terminalReason: "timed_out" }, + { reason: "turn_completion_idle_timeout", terminalReason: "timed_out" }, + { reason: "turn_terminal_idle_timeout", terminalReason: "timed_out" }, + { reason: "client_closed", terminalReason: "failed" }, + ] as const)( + "projects native pre-tool failure reason $reason without a Codex item", + async ({ reason, terminalReason }) => { + const controller = new AbortController(); + controller.abort(reason); + const events: DiagnosticEventPayload[] = []; + const unsubscribe = onInternalDiagnosticEvent((event) => events.push(event)); + try { + emitCodexNativePreToolUseFailureDiagnostic({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:session-1", + runId: "run-1", + signal: controller.signal, + failure: { + toolName: "exec", + toolCallId: "native-no-item", + disposition: "cancelled", + durationMs: 5, + }, + }); + await flushDiagnosticEvents(); + } finally { + unsubscribe(); + } + + expect(events).toContainEqual( + expect.objectContaining({ + type: "tool.execution.error", + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:session-1", + runId: "run-1", + toolName: "exec", + toolCallId: "native-no-item", + durationMs: 5, + errorCategory: "before_tool_call", + terminalReason, + }), + ); + }, + ); }); function createRelay(options?: { diff --git a/extensions/codex/src/app-server/native-hook-relay.ts b/extensions/codex/src/app-server/native-hook-relay.ts index d219e7f657f3..74971d0f9849 100644 --- a/extensions/codex/src/app-server/native-hook-relay.ts +++ b/extensions/codex/src/app-server/native-hook-relay.ts @@ -5,15 +5,18 @@ import { createHash } from "node:crypto"; import { registerNativeHookRelay, + type BeforeToolCallFailureDisposition, type EmbeddedRunAttemptParams, type NativeHookRelayEvent, type NativeHookRelayRegistrationHandle, } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { emitTrustedDiagnosticEvent } from "openclaw/plugin-sdk/diagnostic-runtime"; import { addTimerTimeoutGraceMs, finiteSecondsToTimerSafeMilliseconds, } from "openclaw/plugin-sdk/number-runtime"; import type { CodexAppServerRuntimeOptions } from "./config.js"; +import { resolveCodexToolAbortTerminalReason } from "./dynamic-tool-execution.js"; import type { JsonObject, JsonValue } from "./protocol.js"; /** Codex hook events that can be registered through OpenClaw's native relay. */ @@ -41,6 +44,13 @@ type PendingCodexNativeHookRelayUnregister = { unregister: () => void; }; +export type CodexNativePreToolUseFailure = { + toolName: string; + toolCallId: string; + disposition: Exclude; + durationMs: number; +}; + const pendingCodexNativeHookRelayUnregisters = new Set(); /** Defers relay unregister so late native hook subprocesses can still resolve. */ @@ -103,6 +113,38 @@ export function clearPendingCodexNativeHookRelayUnregistersForTests(): void { pendingCodexNativeHookRelayUnregisters.clear(); } +/** Records a native pre-tool failure that Codex does not project as a tool item. */ +export function emitCodexNativePreToolUseFailureDiagnostic(params: { + agentId: string | undefined; + sessionId: string; + sessionKey: string | undefined; + runId: string; + signal?: AbortSignal; + failure: CodexNativePreToolUseFailure; + terminalReason?: CodexNativePreToolUseFailure["disposition"]; + sourceTimestampMs?: number; +}): void { + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionId: params.sessionId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + runId: params.runId, + toolName: params.failure.toolName, + toolCallId: params.failure.toolCallId, + durationMs: params.failure.durationMs, + errorCategory: "before_tool_call", + terminalReason: + params.terminalReason ?? + (params.signal?.aborted + ? resolveCodexToolAbortTerminalReason(params.signal) + : params.failure.disposition), + ...(params.sourceTimestampMs !== undefined + ? { sourceTimestampMs: params.sourceTimestampMs } + : {}), + }); +} + /** Registers an OpenClaw native hook relay for a Codex app-server turn. */ export function createCodexNativeHookRelay(params: { options: @@ -125,6 +167,7 @@ export function createCodexNativeHookRelay(params: { startupTimeoutMs: number; turnStartTimeoutMs: number; signal: AbortSignal; + onPreToolUseFailure: (failure: CodexNativePreToolUseFailure) => void | Promise; }): NativeHookRelayRegistrationHandle | undefined { if (params.options?.enabled === false) { return undefined; @@ -154,6 +197,7 @@ export function createCodexNativeHookRelay(params: { turnStartTimeoutMs: params.turnStartTimeoutMs, }), signal: params.signal, + onPreToolUseFailure: params.onPreToolUseFailure, command: { // Hook relay subprocesses are observational for most tool events; keep // them lower priority so they do not compete with the active reply turn. diff --git a/extensions/codex/src/app-server/protocol.ts b/extensions/codex/src/app-server/protocol.ts index 8ef472d625ba..cdc09bcc2417 100644 --- a/extensions/codex/src/app-server/protocol.ts +++ b/extensions/codex/src/app-server/protocol.ts @@ -363,6 +363,7 @@ export type CodexDynamicToolCallParams = { export type CodexDynamicToolCallResponse = { asyncStarted?: boolean; contentItems: CodexDynamicToolCallOutputContentItem[]; + diagnosticTerminalReason?: CodexDynamicToolDiagnosticTerminalReason; diagnosticTerminalType?: CodexDynamicToolDiagnosticTerminalType; sideEffectEvidence?: boolean; success: boolean; @@ -370,6 +371,7 @@ export type CodexDynamicToolCallResponse = { }; export type CodexDynamicToolDiagnosticTerminalType = "blocked" | "completed" | "error"; +export type CodexDynamicToolDiagnosticTerminalReason = "failed" | "cancelled" | "timed_out"; export type CodexDynamicToolCallOutputContentItem = | { diff --git a/extensions/codex/src/app-server/run-attempt.dynamic-tools.test.ts b/extensions/codex/src/app-server/run-attempt.dynamic-tools.test.ts index ddfbe97705d1..9284f9b03562 100644 --- a/extensions/codex/src/app-server/run-attempt.dynamic-tools.test.ts +++ b/extensions/codex/src/app-server/run-attempt.dynamic-tools.test.ts @@ -1,9 +1,6 @@ // Codex tests cover run attemptynamic tools plugin behavior. import path from "node:path"; -import { - onAgentEvent, - type AgentEventPayload, -} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { onAgentEvent, type AgentEventPayload } from "openclaw/plugin-sdk/agent-harness-runtime"; import { emitTrustedDiagnosticEvent, onInternalDiagnosticEvent, @@ -56,6 +53,49 @@ function activeDiagnosticToolKeys(events: DiagnosticEventPayload[]): Set setupRunAttemptTestHooks(); describe("runCodexAppServerAttempt dynamic tools", () => { + it.each(["cancelled", "timed_out"] as const)( + "preserves the %s terminal reason in trusted tool diagnostics", + async (terminalReason) => { + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + const call = { + threadId: "thread-1", + turnId: "turn-1", + callId: `call-${terminalReason}`, + namespace: null, + tool: "lookup", + arguments: {}, + } satisfies CodexDynamicToolCallParams; + try { + emitDynamicToolStartedDiagnostic({ + call, + agentId: "agent-terminal-reason", + runId: "run-terminal-reason", + }); + emitDynamicToolTerminalDiagnostic({ + call, + agentId: "agent-terminal-reason", + runId: "run-terminal-reason", + durationMs: 1, + response: { + success: false, + diagnosticTerminalReason: terminalReason, + contentItems: [{ type: "inputText", text: "not persisted by audit" }], + }, + }); + await flushDiagnosticEvents(); + } finally { + unsubscribeDiagnostics(); + } + + expect(diagnosticEvents.find((event) => event.type === "tool.execution.error")).toMatchObject( + { agentId: "agent-terminal-reason", terminalReason }, + ); + }, + ); + it("passes the live run session key to Codex dynamic tools when sandbox policy uses another key", () => { const workspaceDir = path.join(tempDir, "workspace"); const params = createParams(path.join(tempDir, "session.jsonl"), workspaceDir); diff --git a/extensions/codex/src/app-server/run-attempt.native-hook-relay.test.ts b/extensions/codex/src/app-server/run-attempt.native-hook-relay.test.ts index 8e1e1739fd0f..130266791188 100644 --- a/extensions/codex/src/app-server/run-attempt.native-hook-relay.test.ts +++ b/extensions/codex/src/app-server/run-attempt.native-hook-relay.test.ts @@ -4,7 +4,12 @@ import { abortAgentHarnessRun, invokeNativeHookRelay, nativeHookRelayTesting, + type NativeHookRelayRegistrationHandle, } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + onInternalDiagnosticEvent, + type DiagnosticEventPayload, +} from "openclaw/plugin-sdk/diagnostic-runtime"; import { describe, expect, it, vi } from "vitest"; import * as approvalBridge from "./approval-bridge.js"; import { @@ -30,9 +35,7 @@ const DISABLED_CODEX_WEB_SEARCH_THREAD_CONFIG_FINGERPRINT = JSON.stringify({ web_search: "disabled", }); -function writeCodexAppServerBinding( - ...args: Parameters -) { +function writeCodexAppServerBinding(...args: Parameters) { const [sessionFile, binding, lookup] = args; return writeRawCodexAppServerBinding( sessionFile, @@ -721,22 +724,53 @@ describe("runCodexAppServerAttempt native hook relay", () => { it("cleans up native hook relay state when turn/start fails", async () => { const sessionFile = path.join(tempDir, "session.jsonl"); const workspaceDir = path.join(tempDir, "workspace"); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + let reportPreToolUseFailure: + | NonNullable + | undefined; const harness = createStartedThreadHarness(async (method) => { if (method === "turn/start") { + const startRequest = harness.requests.find((request) => request.method === "thread/start"); + const relayId = extractRelayIdFromThreadRequest(startRequest?.params); + const registration = nativeHookRelayTesting.getNativeHookRelayRegistrationForTests(relayId); + reportPreToolUseFailure = registration?.onPreToolUseFailure; throw new Error("turn start exploded"); } return undefined; }); - await expect( - runCodexAppServerAttempt(createParams(sessionFile, workspaceDir), { - nativeHookRelay: { enabled: true }, - }), - ).rejects.toThrow("turn start exploded"); + try { + await expect( + runCodexAppServerAttempt(createParams(sessionFile, workspaceDir), { + nativeHookRelay: { enabled: true }, + }), + ).rejects.toThrow("turn start exploded"); + await reportPreToolUseFailure?.({ + toolName: "exec", + toolCallId: "turn-start-failure-tool", + disposition: "failed", + durationMs: 5, + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + } finally { + unsubscribeDiagnostics(); + } const startRequest = harness.requests.find((request) => request.method === "thread/start"); const relayId = extractRelayIdFromThreadRequest(startRequest?.params); expect(nativeHookRelayTesting.getNativeHookRelayRegistrationForTests(relayId)).toBeUndefined(); + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "turn-start-failure-tool", + terminalReason: "failed", + }), + ); }); it("cleans up native hook relay state when the Codex turn aborts", async () => { diff --git a/extensions/codex/src/app-server/run-attempt.test.ts b/extensions/codex/src/app-server/run-attempt.test.ts index 899a77676f3e..c328c1d3bbb9 100644 --- a/extensions/codex/src/app-server/run-attempt.test.ts +++ b/extensions/codex/src/app-server/run-attempt.test.ts @@ -2382,6 +2382,109 @@ describe("runCodexAppServerAttempt", () => { expect(inputText).toContain("is the previous message trustworthy?"); }); + it("keeps resumed native web-search outcomes unknown without raw events", async () => { + const sessionFile = path.join(tempDir, "session.jsonl"); + const workspaceDir = path.join(tempDir, "workspace"); + await writeExistingBinding(sessionFile, workspaceDir, { dynamicToolsFingerprint: "[]" }); + const harness = createResumeHarness(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const stopDiagnostics = onInternalDiagnosticEvent((event) => { + if ("toolCallId" in event && event.toolCallId === "resumed-search") { + diagnosticEvents.push(event); + } + }); + + try { + const run = runCodexAppServerAttempt(createParams(sessionFile, workspaceDir)); + await harness.waitForMethod("turn/start"); + const item = { + id: "resumed-search", + type: "webSearch", + query: "sensitive resumed query", + action: { type: "search", query: "sensitive resumed query", queries: null }, + }; + await harness.notify({ + method: "item/started", + params: { threadId: "thread-existing", turnId: "turn-1", item }, + }); + await harness.notify({ + method: "item/completed", + params: { threadId: "thread-existing", turnId: "turn-1", item }, + }); + await harness.completeTurn({ threadId: "thread-existing", turnId: "turn-1" }); + await run; + await flushDiagnosticEvents(); + } finally { + stopDiagnostics(); + } + + expect(diagnosticEvents.map((event) => event.type)).toEqual([ + "tool.execution.started", + "tool.execution.error", + ]); + expect(diagnosticEvents.at(-1)).toMatchObject({ + terminalReason: "failed", + errorCode: "tool_outcome_unknown", + }); + expect(JSON.stringify(diagnosticEvents)).not.toContain("sensitive resumed query"); + }); + + it("uses retained raw web-search status on resumed threads", async () => { + const sessionFile = path.join(tempDir, "session.jsonl"); + const workspaceDir = path.join(tempDir, "workspace"); + await writeExistingBinding(sessionFile, workspaceDir, { dynamicToolsFingerprint: "[]" }); + const harness = createResumeHarness(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const stopDiagnostics = onInternalDiagnosticEvent((event) => { + if ("toolCallId" in event && event.toolCallId === "resumed-search-with-raw") { + diagnosticEvents.push(event); + } + }); + + try { + const run = runCodexAppServerAttempt(createParams(sessionFile, workspaceDir)); + await harness.waitForMethod("turn/start"); + const item = { + id: "resumed-search-with-raw", + type: "webSearch", + query: "sensitive warm-resume query", + action: { type: "search", query: "sensitive warm-resume query", queries: null }, + }; + await harness.notify({ + method: "item/started", + params: { threadId: "thread-existing", turnId: "turn-1", item }, + }); + await harness.notify({ + method: "item/completed", + params: { threadId: "thread-existing", turnId: "turn-1", item }, + }); + await harness.notify({ + method: "rawResponseItem/completed", + params: { + threadId: "thread-existing", + turnId: "turn-1", + item: { + id: item.id, + type: "web_search_call", + status: "completed", + action: item.action, + }, + }, + }); + await harness.completeTurn({ threadId: "thread-existing", turnId: "turn-1" }); + await run; + await flushDiagnosticEvents(); + } finally { + stopDiagnostics(); + } + + expect(diagnosticEvents.map((event) => event.type)).toEqual([ + "tool.execution.started", + "tool.execution.completed", + ]); + expect(JSON.stringify(diagnosticEvents)).not.toContain("sensitive warm-resume query"); + }); + it("projects only newer visible history when a resumed Codex binding is stale", async () => { const sessionFile = path.join(tempDir, "session.jsonl"); const workspaceDir = path.join(tempDir, "workspace"); diff --git a/extensions/codex/src/app-server/run-attempt.ts b/extensions/codex/src/app-server/run-attempt.ts index 5ef53af488d0..14b2f074fabf 100644 --- a/extensions/codex/src/app-server/run-attempt.ts +++ b/extensions/codex/src/app-server/run-attempt.ts @@ -20,6 +20,7 @@ import { isActiveHarnessContextEngine, loadCodexBundleMcpThreadConfig, resolveAgentHarnessBeforePromptBuildResult, + resolveAgentRunAbortLifecycleFields, resolveContextEngineOwnerPluginId, resolveSandboxContext, resolveSessionAgentIds, @@ -178,6 +179,7 @@ import { hasPendingDynamicToolTerminalDiagnostic, isDynamicToolTerminalDiagnosticEvent, isMatchingDynamicToolTerminalDiagnostic, + resolveCodexToolAbortTerminalReason, resolveDynamicToolCallTimeoutMs, resolveTerminalDynamicToolBatchAction, shouldBlockTerminalReleaseForNonTerminalDynamicToolResult, @@ -203,11 +205,13 @@ import { clearPendingCodexNativeHookRelayUnregistersForTests, CODEX_NATIVE_HOOK_RELAY_TTL_GRACE_MS, createCodexNativeHookRelay, + emitCodexNativePreToolUseFailureDiagnostic, flushPendingCodexNativeHookRelayUnregistersForTests, resolveCodexNativeHookRelayEvents, resolveCodexNativeHookRelayTtlMs, resolveCodexNativeHookRelayUnregisterGraceMs, scheduleCodexNativeHookRelayUnregister, + type CodexNativePreToolUseFailure, } from "./native-hook-relay.js"; import { registerCodexNativeSubagentMonitor } from "./native-subagent-monitor.js"; import { describeCodexNotificationCorrelation } from "./notification-correlation.js"; @@ -1428,6 +1432,41 @@ export async function runCodexAppServerAttempt( trajectoryEndRecorded = true; }; let nativeHookRelay: NativeHookRelayRegistrationHandle | undefined; + const pendingNativePreToolUseFailures: CodexNativePreToolUseFailure[] = []; + const projectorRef: { current?: CodexAppServerEventProjector } = {}; + let nativePreToolUseFailureFallbackActive = false; + let nativePreToolUseFailureFallbackTerminalReason: + | CodexNativePreToolUseFailure["disposition"] + | undefined; + const emitNativePreToolUseFailure = (failure: CodexNativePreToolUseFailure) => { + emitCodexNativePreToolUseFailureDiagnostic({ + agentId: sessionAgentId, + sessionId: params.sessionId, + sessionKey: sandboxSessionKey, + runId: params.runId, + signal: runAbortController.signal, + failure, + ...(nativePreToolUseFailureFallbackActive + ? { + terminalReason: nativePreToolUseFailureFallbackTerminalReason ?? failure.disposition, + } + : {}), + }); + }; + const flushPendingNativePreToolUseFailures = () => { + for (const failure of pendingNativePreToolUseFailures.splice(0)) { + emitNativePreToolUseFailure(failure); + } + }; + const activateNativePreToolUseFailureFallback = () => { + if (!nativePreToolUseFailureFallbackActive) { + nativePreToolUseFailureFallbackTerminalReason = runAbortController.signal.aborted + ? resolveCodexToolAbortTerminalReason(runAbortController.signal) + : undefined; + nativePreToolUseFailureFallbackActive = true; + } + flushPendingNativePreToolUseFailures(); + }; let releaseSharedClientLease: (() => void) | undefined; let sharedCodexClientRetiredForOneShotCleanup = false; const releaseSharedClientLeaseOnce = () => { @@ -1503,6 +1542,16 @@ export async function runCodexAppServerAttempt( startupTimeoutMs, turnStartTimeoutMs: params.timeoutMs, signal: runAbortController.signal, + onPreToolUseFailure: (failure) => { + const projector = projectorRef.current; + if (projector) { + projector.recordNativeToolPreToolUseFailure(failure); + } else if (nativePreToolUseFailureFallbackActive) { + emitNativePreToolUseFailure(failure); + } else { + pendingNativePreToolUseFailures.push(failure); + } + }, }); return { configPatch: nativeHookRelay @@ -1616,6 +1665,7 @@ export async function runCodexAppServerAttempt( data: { phase: "thread_ready", threadId: thread.threadId }, }); } catch (error) { + activateNativePreToolUseFailureFallback(); nativeHookRelay?.unregister(); await releaseSandboxExecEnvironment(); params.abortSignal?.removeEventListener("abort", abortFromUpstream); @@ -1696,7 +1746,6 @@ export async function runCodexAppServerAttempt( let terminalDynamicToolReleaseCheckScheduled = false; let currentTurnHadNonTerminalDynamicToolResult = false; const turnIdRef: { current?: string } = {}; - const projectorRef: { current?: CodexAppServerEventProjector } = {}; const userInputBridgeRef: { current?: ReturnType; } = {}; @@ -1889,6 +1938,23 @@ export async function runCodexAppServerAttempt( }); lifecycleTerminalEmitted = true; }; + const buildLifecycleTerminalMeta = (input: { aborted: boolean; timedOut: boolean }) => { + const abortFields = input.aborted + ? resolveAgentRunAbortLifecycleFields(runAbortController.signal) + : undefined; + if (input.timedOut || abortFields?.stopReason === "timeout") { + return { + aborted: true, + status: "timed_out", + stopReason: "timeout", + timeoutPhase: "provider", + providerStarted: true, + } as const; + } + return input.aborted + ? ({ aborted: true, status: "cancelled", stopReason: "stop" } as const) + : undefined; + }; const executionPhaseKeys = new Set(); const emitExecutionPhaseOnce = ( @@ -2374,6 +2440,8 @@ export async function runCodexAppServerAttempt( internalExecAutoReview: appServer.approvalsReviewer === "user", autoApprove: shouldAutoApproveCodexAppServerApprovals(appServer), signal: runAbortController.signal, + onNativeToolFailureDisposition: (itemId, disposition) => + projector?.recordNativeToolApprovalFailure(itemId, disposition), }); } return undefined; @@ -2406,6 +2474,7 @@ export async function runCodexAppServerAttempt( }); emitDynamicToolStartedDiagnostic({ call, + agentId: sessionAgentId, runId: params.runId, sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -2529,6 +2598,7 @@ export async function runCodexAppServerAttempt( emitDynamicToolTerminalDiagnostic({ response, call, + agentId: sessionAgentId, runId: params.runId, sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -2562,6 +2632,7 @@ export async function runCodexAppServerAttempt( ) { emitDynamicToolErrorDiagnostic({ call, + agentId: sessionAgentId, runId: params.runId, sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -2908,6 +2979,7 @@ export async function runCodexAppServerAttempt( } notificationCleanup(); requestCleanup(); + activateNativePreToolUseFailureFallback(); nativeHookRelay?.unregister(); await releaseSandboxExecEnvironment(); await runAgentCleanupStep({ @@ -2940,6 +3012,7 @@ export async function runCodexAppServerAttempt( } } if (!turn) { + activateNativePreToolUseFailureFallback(); await releaseSharedClientLeaseAndRetireOneShotClient(); throw new Error("codex app-server turn/start failed without an error"); } @@ -2977,6 +3050,7 @@ export async function runCodexAppServerAttempt( nativePostToolUseRelayEnabled: nativeHookRelay?.allowedEvents.includes("post_tool_use") === true && nativeHookRelay.shouldRelayEvent("post_tool_use"), + runAbortSignal: runAbortController.signal, trajectoryRecorder, onNativeToolResultRecorded: maybeAnnounceFastModeAutoOff, }, @@ -3021,6 +3095,9 @@ export async function runCodexAppServerAttempt( turnWatches.touchActivity("turn:start", { arm: true }); turnWatches.armAttemptIdleWatch(); turnWatches.touchActivity("turn:start", { attemptProgress: true }); + for (const failure of pendingNativePreToolUseFailures.splice(0)) { + activeProjector.recordNativeToolPreToolUseFailure(failure); + } for (const notification of pendingNotifications.splice(0)) { await enqueueNotification(notification); } @@ -3461,11 +3538,12 @@ export async function runCodexAppServerAttempt( emitLifecycleTerminal({ phase: "error", error: formatErrorMessage(finalPromptError), + ...buildLifecycleTerminalMeta({ aborted: finalAborted, timedOut: effectiveTimedOut }), }); } else { emitLifecycleTerminal({ phase: "end", - ...(finalAborted ? { aborted: true } : {}), + ...buildLifecycleTerminalMeta({ aborted: finalAborted, timedOut: effectiveTimedOut }), }); } return { @@ -3489,6 +3567,10 @@ export async function runCodexAppServerAttempt( emitLifecycleTerminal({ phase: "error", error: "codex app-server run completed without lifecycle terminal event", + ...buildLifecycleTerminalMeta({ + aborted: runAbortController.signal.aborted && !clientClosedAbort, + timedOut, + }), }); if (trajectoryRecorder && !trajectoryEndRecorded) { trajectoryRecorder.recordEvent("session.ended", { @@ -3793,6 +3875,9 @@ function handleApprovalRequest(params: { internalExecAutoReview?: boolean; autoApprove?: boolean; signal?: AbortSignal; + onNativeToolFailureDisposition?: Parameters< + typeof handleCodexAppServerApprovalRequest + >[0]["onNativeToolFailureDisposition"]; }): Promise { return handleCodexAppServerApprovalRequest({ method: params.method, @@ -3806,6 +3891,7 @@ function handleApprovalRequest(params: { internalExecAutoReview: params.internalExecAutoReview, autoApprove: params.autoApprove, signal: params.signal, + onNativeToolFailureDisposition: params.onNativeToolFailureDisposition, }); } diff --git a/extensions/codex/src/app-server/run-attempt.turn-watches.test.ts b/extensions/codex/src/app-server/run-attempt.turn-watches.test.ts index 1e9c58a75fd7..789ba8b5b116 100644 --- a/extensions/codex/src/app-server/run-attempt.turn-watches.test.ts +++ b/extensions/codex/src/app-server/run-attempt.turn-watches.test.ts @@ -135,8 +135,10 @@ function completedCommand(id: string, command: string): CodexServerNotification async function runTurnWatchTimeoutScenario(notifications: CodexServerNotification[]) { const harness = createStartedThreadHarness(); + const onRunAgentEvent = vi.fn(); const params = createParams(path.join(tempDir, "session.jsonl"), path.join(tempDir, "workspace")); params.timeoutMs = 100; + params.onAgentEvent = onRunAgentEvent; const run = runCodexAppServerAttempt(params, { turnCompletionIdleTimeoutMs: 500, turnAssistantCompletionIdleTimeoutMs: 1_000, @@ -146,7 +148,7 @@ async function runTurnWatchTimeoutScenario(notifications: CodexServerNotificatio for (const notification of notifications) { await harness.notify(notification); } - return { params, result: await run }; + return { onRunAgentEvent, params, result: await run }; } async function runClientCloseScenario(notifications: CodexServerNotification[]) { @@ -572,7 +574,7 @@ describe("runCodexAppServerAttempt turn watches", () => { it("recovers completed assistant output from a non-completion timeout", async () => { const warn = vi.spyOn(embeddedAgentLog, "warn").mockImplementation(() => undefined); - const { params, result } = await runTurnWatchTimeoutScenario([ + const { onRunAgentEvent, params, result } = await runTurnWatchTimeoutScenario([ completedCommand("cmd-1", "touch done.txt"), completedAssistant("msg-1", "Finished."), ]); @@ -586,6 +588,12 @@ describe("runCodexAppServerAttempt turn watches", () => { expect(result.itemLifecycle.completedCount).toBe(2); expect(result.codexAppServerFailure).toBeUndefined(); expect(result.promptTimeoutOutcome).toBeUndefined(); + const terminalLifecycle = onRunAgentEvent.mock.calls + .map(([event]) => event) + .find((event) => event.stream === "lifecycle" && event.data.phase === "end")?.data; + expect(terminalLifecycle).toMatchObject({ phase: "end" }); + expect(terminalLifecycle?.status).toBeUndefined(); + expect(terminalLifecycle?.aborted).toBeUndefined(); expect(warn).toHaveBeenCalledWith( "codex app-server recovered completed assistant output after missing turn completion", expect.objectContaining({ @@ -3366,11 +3374,13 @@ describe("runCodexAppServerAttempt turn watches", () => { it("does not treat global rate-limit notifications as turn progress", async () => { const warn = vi.spyOn(embeddedAgentLog, "warn").mockImplementation(() => undefined); const harness = createStartedThreadHarness(); + const onRunAgentEvent = vi.fn(); const params = createParams( path.join(tempDir, "session.jsonl"), path.join(tempDir, "workspace"), ); params.timeoutMs = 200; + params.onAgentEvent = onRunAgentEvent; const run = runCodexAppServerAttempt(params, { turnCompletionIdleTimeoutMs: 15 }); await harness.waitForMethod("turn/start"); @@ -3433,6 +3443,17 @@ describe("runCodexAppServerAttempt turn watches", () => { turnId: "turn-1", }), ); + expect( + onRunAgentEvent.mock.calls + .map(([event]) => event) + .find((event) => event.stream === "lifecycle" && event.data.phase === "error")?.data, + ).toMatchObject({ + aborted: true, + status: "timed_out", + stopReason: "timeout", + timeoutPhase: "provider", + providerStarted: true, + }); }); it("clears the thread binding after a completion-idle timeout so the next turn starts fresh", async () => { @@ -5161,11 +5182,13 @@ describe("runCodexAppServerAttempt turn watches", () => { it("keeps upstream cancellation aborted when Codex completes the turn as interrupted", async () => { const harness = createStartedThreadHarness(); const abortController = new AbortController(); + const onRunAgentEvent = vi.fn(); const params = createParams( path.join(tempDir, "session.jsonl"), path.join(tempDir, "workspace"), ); params.abortSignal = abortController.signal; + params.onAgentEvent = onRunAgentEvent; const run = runCodexAppServerAttempt(params, { turnTerminalIdleTimeoutMs: 60_000 }); await harness.waitForMethod("turn/start"); @@ -5183,6 +5206,52 @@ describe("runCodexAppServerAttempt turn watches", () => { expect(result.aborted).toBe(true); expect(result.timedOut).toBe(false); expect(result.promptError).toBeNull(); + expect( + onRunAgentEvent.mock.calls + .map(([event]) => event) + .find((event) => event.stream === "lifecycle" && event.data.phase === "end")?.data, + ).toMatchObject({ aborted: true, status: "cancelled", stopReason: "stop" }); + }); + + it("classifies an upstream hard timeout as timed out lifecycle", async () => { + const harness = createStartedThreadHarness(); + const abortController = new AbortController(); + const onRunAgentEvent = vi.fn(); + const params = createParams( + path.join(tempDir, "session.jsonl"), + path.join(tempDir, "workspace"), + ); + params.abortSignal = abortController.signal; + params.onAgentEvent = onRunAgentEvent; + const run = runCodexAppServerAttempt(params, { turnTerminalIdleTimeoutMs: 60_000 }); + + await harness.waitForMethod("turn/start"); + const timeoutError = new Error("cron watchdog timeout"); + timeoutError.name = "TimeoutError"; + abortController.abort(timeoutError); + await harness.notify({ + method: "turn/completed", + params: { + threadId: "thread-1", + turnId: "turn-1", + turn: { id: "turn-1", status: "interrupted" }, + }, + }); + + const result = await run; + expect(result.aborted).toBe(true); + expect(result.promptError).toBeNull(); + expect( + onRunAgentEvent.mock.calls + .map(([event]) => event) + .find((event) => event.stream === "lifecycle" && event.data.phase === "end")?.data, + ).toMatchObject({ + aborted: true, + status: "timed_out", + stopReason: "timeout", + timeoutPhase: "provider", + providerStarted: true, + }); }); it("releases completion when the app-server client closes during an active turn", async () => { @@ -5517,11 +5586,13 @@ describe("runCodexAppServerAttempt turn watches", () => { // gateway session lane stays locked and every follow-up message queues // behind a run that will never resolve. let notify: (notification: CodexServerNotification) => Promise = async () => undefined; + let turnStarted = false; const request = vi.fn(async (method: string) => { if (method === "thread/start") { return threadStartResult("thread-1"); } if (method === "turn/start") { + turnStarted = true; return turnStartResult("turn-1", "inProgress"); } return {}; @@ -5543,6 +5614,12 @@ describe("runCodexAppServerAttempt turn watches", () => { path.join(tempDir, "workspace"), ); params.onAgentEvent = () => { + // Only explode once the turn is live: pre-turn run-lifecycle events + // would otherwise kill the attempt before the projector path under + // test (turn/completed handling) ever runs. + if (!turnStarted) { + return; + } throw new Error("downstream consumer exploded"); }; const run = runCodexAppServerAttempt(params); diff --git a/extensions/codex/src/app-server/side-question.test.ts b/extensions/codex/src/app-server/side-question.test.ts index 0a781ca9e95f..9b13b6a8a41a 100644 --- a/extensions/codex/src/app-server/side-question.test.ts +++ b/extensions/codex/src/app-server/side-question.test.ts @@ -1,5 +1,8 @@ // Codex tests cover side question plugin behavior. -import { nativeHookRelayTesting } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + nativeHookRelayTesting, + type NativeHookRelayRegistrationHandle, +} from "openclaw/plugin-sdk/agent-harness-runtime"; import { onInternalDiagnosticEvent, resetDiagnosticEventsForTest, @@ -11,7 +14,7 @@ import { } from "openclaw/plugin-sdk/hook-runtime"; import { createMockPluginRegistry } from "openclaw/plugin-sdk/plugin-test-runtime"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { CodexServerNotification, JsonObject, RpcRequest } from "./protocol.js"; +import type { CodexServerNotification, JsonObject, JsonValue, RpcRequest } from "./protocol.js"; const readCodexAppServerBindingMock = vi.fn(); const isCodexAppServerNativeAuthProfileMock = vi.fn(); @@ -294,6 +297,26 @@ function turnCompleted(threadId: string, turnId: string, text: string): CodexSer }; } +function nativeCommandItem( + id: string, + status: "inProgress" | "completed", + durationMs: number | null, +) { + return { + type: "commandExecution", + id, + command: "git status --short", + cwd: "/tmp/workspace", + processId: null, + source: "agent", + status, + commandActions: [], + aggregatedOutput: status === "completed" ? "" : null, + exitCode: status === "completed" ? 0 : null, + durationMs, + }; +} + function turnCompletedWithNestedThread( threadId: string, turnId: string, @@ -439,6 +462,7 @@ describe("runCodexAppServerSideQuestion", () => { nativeHookRelayTesting.clearNativeHookRelaysForTests(); resetDiagnosticEventsForTest(); resetGlobalHookRunner(); + vi.useRealTimers(); }); it("forks an ephemeral side thread and returns the completed assistant text", async () => { @@ -585,6 +609,22 @@ describe("runCodexAppServerSideQuestion", () => { expect(toolOptions).toHaveProperty("requireExplicitMessageTarget", true); }); + it("allocates one fallback run ID per side-question invocation", async () => { + const client = createFakeClient(); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + await runCodexAppServerSideQuestion(sideParams()); + await runCodexAppServerSideQuestion(sideParams()); + + const runIds = createOpenClawCodingToolsMock.mock.calls.map( + ([options]) => (options as { runId: string }).runId, + ); + expect(runIds).toHaveLength(2); + expect(runIds[0]).toMatch(/^[0-9a-f-]{36}$/); + expect(runIds[1]).toMatch(/^[0-9a-f-]{36}$/); + expect(new Set(runIds).size).toBe(2); + }); + it("replays app-scoped reviewer policy into side-thread forks", async () => { const client = createFakeClient(); getSharedCodexAppServerClientMock.mockResolvedValue(client); @@ -1026,6 +1066,9 @@ describe("runCodexAppServerSideQuestion", () => { const turnStartCall = client.request.mock.calls.find(([method]) => method === "turn/start"); expect(turnStartCall?.[1]).not.toHaveProperty("config"); expect(relayIdDuringFork).toBeDefined(); + expect(createOpenClawCodingToolsMock).toHaveBeenCalledWith( + expect.objectContaining({ runId: "run-side-1" }), + ); expect( nativeHookRelayTesting.getNativeHookRelayRegistrationForTests(relayIdDuringFork!), ).toBeUndefined(); @@ -1585,6 +1628,218 @@ describe("runCodexAppServerSideQuestion", () => { expect(relayId).toBe(relayIdDuringFork); }); + it("emits a buffered native pre-tool failure when side turn startup fails", async () => { + const client = createFakeClient(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + let relayId: string | undefined; + let reportPreToolUseFailure: + | NonNullable + | undefined; + client.request.mockImplementation(async (method: string, requestParams: unknown) => { + if (method === "thread/fork") { + relayId = extractRelayIdFromThreadConfig( + (requestParams as { config?: Record }).config, + ); + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + if (!relayId) { + throw new Error("Expected native hook relay id"); + } + reportPreToolUseFailure = + nativeHookRelayTesting.getNativeHookRelayRegistrationForTests( + relayId, + )?.onPreToolUseFailure; + throw new Error("side turn start exploded"); + } + if (method === "thread/unsubscribe" || method === "turn/interrupt") { + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + try { + await expect( + runCodexAppServerSideQuestion(sideParams(), { + nativeHookRelay: { enabled: true }, + }), + ).rejects.toThrow("side turn start exploded"); + await reportPreToolUseFailure?.({ + toolName: "exec", + toolCallId: "side-turn-start-failure-tool", + disposition: "failed", + durationMs: 5, + }); + await flushDiagnosticEvents(); + } finally { + unsubscribeDiagnostics(); + } + + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "side-turn-start-failure-tool", + terminalReason: "failed", + }), + ); + }); + + it("preserves a late native pre-tool failure after side turn cleanup", async () => { + const client = createFakeClient(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + let reportPreToolUseFailure: + | NonNullable + | undefined; + client.request.mockImplementation(async (method: string, requestParams: unknown) => { + if (method === "thread/fork") { + const relayId = extractRelayIdFromThreadConfig( + (requestParams as { config?: Record }).config, + ); + reportPreToolUseFailure = + nativeHookRelayTesting.getNativeHookRelayRegistrationForTests( + relayId, + )?.onPreToolUseFailure; + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + queueMicrotask(() => { + client.emit(agentDelta("side-thread", "turn-1", "Side answer.")); + client.emit(turnCompleted("side-thread", "turn-1", "Side answer.")); + }); + return turnStartResult("turn-1"); + } + if (method === "thread/unsubscribe" || method === "turn/interrupt") { + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + try { + await expect( + runCodexAppServerSideQuestion(sideParams(), { + nativeHookRelay: { enabled: true }, + }), + ).resolves.toEqual({ text: "Side answer." }); + await reportPreToolUseFailure?.({ + toolName: "exec", + toolCallId: "late-side-tool", + disposition: "failed", + durationMs: 5, + }); + await flushDiagnosticEvents(); + } finally { + unsubscribeDiagnostics(); + } + + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "late-side-tool", + terminalReason: "failed", + }), + ); + }); + + it("coalesces a native pre-tool failure that arrives during side turn cleanup", async () => { + const client = createFakeClient(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + let reportPreToolUseFailure: + | NonNullable + | undefined; + client.request.mockImplementation(async (method: string, requestParams: unknown) => { + if (method === "thread/fork") { + const relayId = extractRelayIdFromThreadConfig( + (requestParams as { config?: Record }).config, + ); + reportPreToolUseFailure = + nativeHookRelayTesting.getNativeHookRelayRegistrationForTests( + relayId, + )?.onPreToolUseFailure; + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + queueMicrotask(() => { + client.emit({ + method: "item/started", + params: { + threadId: "side-thread", + turnId: "turn-1", + item: nativeCommandItem("side-cleanup-failure-tool", "inProgress", null), + }, + }); + client.emit(turnCompleted("side-thread", "turn-1", "Side answer.")); + }); + return turnStartResult("turn-1"); + } + if (method === "thread/unsubscribe") { + await reportPreToolUseFailure?.({ + toolName: "exec", + toolCallId: "side-cleanup-failure-tool", + disposition: "failed", + durationMs: 5, + }); + return {}; + } + if (method === "turn/interrupt") { + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + try { + await expect( + runCodexAppServerSideQuestion(sideParams(), { + nativeHookRelay: { enabled: true }, + }), + ).resolves.toEqual({ text: "Side answer." }); + await flushDiagnosticEvents(); + } finally { + unsubscribeDiagnostics(); + } + + expect( + diagnosticEvents.filter( + (event) => + event.type.startsWith("tool.execution.") && + "toolCallId" in event && + event.toolCallId === "side-cleanup-failure-tool", + ), + ).toEqual([ + expect.objectContaining({ + type: "tool.execution.started", + toolCallId: "side-cleanup-failure-tool", + }), + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "side-cleanup-failure-tool", + errorCategory: "before_tool_call", + terminalReason: "failed", + }), + ]); + expect(activeDiagnosticToolKeys(diagnosticEvents)).toEqual(new Set()); + }); + it("bridges side-thread dynamic tool requests to OpenClaw tools", async () => { const client = createFakeClient(); let toolResponse: unknown; @@ -1637,6 +1892,69 @@ describe("runCodexAppServerSideQuestion", () => { }); }); + it("aborts active side tools before waiting for thread cleanup", async () => { + const client = createFakeClient(); + let releaseUnsubscribe: (() => void) | undefined; + const unsubscribePending = new Promise((resolve) => { + releaseUnsubscribe = resolve; + }); + let toolAborted = false; + toolExecuteMock.mockImplementation( + (_callId: string, _args: unknown, signal?: AbortSignal) => + new Promise((_resolve, reject) => { + signal?.addEventListener( + "abort", + () => { + toolAborted = true; + reject(new Error("side tool aborted")); + }, + { once: true }, + ); + }), + ); + client.request.mockImplementation(async (method: string) => { + if (method === "thread/fork") { + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + setTimeout(() => { + void client.handleRequest({ + id: 42, + method: "item/tool/call", + params: { + threadId: "side-thread", + turnId: "turn-1", + callId: "tool-1", + tool: "wiki_status", + arguments: {}, + }, + }); + client.emit(turnCompleted("side-thread", "turn-1", "Finished answer.")); + }, 0); + return turnStartResult("turn-1"); + } + if (method === "thread/unsubscribe") { + await unsubscribePending; + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + const run = runCodexAppServerSideQuestion(sideParams()); + await vi.waitFor(() => + expect(client.request.mock.calls.some(([method]) => method === "thread/unsubscribe")).toBe( + true, + ), + ); + expect(toolAborted).toBe(true); + releaseUnsubscribe?.(); + await expect(run).resolves.toEqual({ text: "Finished answer." }); + }); + it("clears side-thread dynamic tool diagnostics at the app-server request boundary", async () => { const client = createFakeClient(); const diagnosticEvents: DiagnosticEventPayload[] = []; @@ -1714,6 +2032,410 @@ describe("runCodexAppServerSideQuestion", () => { expect(activeDiagnosticToolKeys(diagnosticEvents)).toEqual(new Set()); }); + it("projects native side-thread tool notifications into trusted diagnostics", async () => { + const client = createFakeClient(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + client.request.mockImplementation(async (method: string) => { + if (method === "thread/fork") { + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + setTimeout(() => { + client.emit({ + method: "item/started", + params: { + threadId: "side-thread", + turnId: "turn-1", + item: nativeCommandItem("native-tool-1", "inProgress", null), + }, + }); + client.emit({ + method: "item/completed", + params: { + threadId: "side-thread", + turnId: "turn-1", + item: nativeCommandItem("native-tool-1", "completed", 12), + }, + }); + const webSearchItem = { + type: "webSearch", + id: "native-search-1", + query: "sensitive side-thread query", + action: { + type: "search", + query: "sensitive side-thread query", + queries: null, + }, + }; + client.emit({ + method: "item/started", + params: { threadId: "side-thread", turnId: "turn-1", item: webSearchItem }, + }); + client.emit({ + method: "item/completed", + params: { threadId: "side-thread", turnId: "turn-1", item: webSearchItem }, + }); + client.emit(turnCompleted("side-thread", "turn-1", "Native tool answer.")); + }, 0); + return turnStartResult("turn-1"); + } + if (method === "thread/unsubscribe" || method === "turn/interrupt") { + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + try { + await runCodexAppServerSideQuestion( + sideParams({ + agentId: "side-agent", + sessionKey: "agent:side-agent:main", + opts: { runId: "run-side-native-tool" }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribeDiagnostics(); + } + + type ToolExecutionEvent = Extract< + DiagnosticEventPayload, + { + type: + | "tool.execution.started" + | "tool.execution.completed" + | "tool.execution.error" + | "tool.execution.blocked"; + } + >; + const toolEvents = diagnosticEvents.filter((event): event is ToolExecutionEvent => + event.type.startsWith("tool.execution."), + ); + expect( + toolEvents.map((event) => ({ + type: event.type, + agentId: event.agentId, + toolName: "toolName" in event ? event.toolName : undefined, + toolCallId: "toolCallId" in event ? event.toolCallId : undefined, + durationMs: "durationMs" in event ? event.durationMs : undefined, + })), + ).toEqual([ + { + type: "tool.execution.started", + agentId: "side-agent", + toolName: "bash", + toolCallId: "native-tool-1", + durationMs: undefined, + }, + { + type: "tool.execution.completed", + agentId: "side-agent", + toolName: "bash", + toolCallId: "native-tool-1", + durationMs: 12, + }, + { + type: "tool.execution.started", + agentId: "side-agent", + toolName: "web_search", + toolCallId: "native-search-1", + durationMs: undefined, + }, + { + type: "tool.execution.error", + agentId: "side-agent", + toolName: "web_search", + toolCallId: "native-search-1", + durationMs: expect.any(Number), + }, + ]); + expect(toolEvents.at(-1)).toMatchObject({ + errorCode: "tool_outcome_unknown", + terminalReason: "failed", + }); + expect(activeDiagnosticToolKeys(diagnosticEvents)).toEqual(new Set()); + expect(JSON.stringify(toolEvents)).not.toContain("sensitive side-thread query"); + }); + + it("keeps cleanup-only aborts out of unfinished native tool outcomes", async () => { + const client = createFakeClient(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + client.request.mockImplementation(async (method: string) => { + if (method === "thread/fork") { + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + setTimeout(() => { + client.emit({ + method: "item/started", + params: { + threadId: "side-thread", + turnId: "turn-1", + item: nativeCommandItem("native-tool-unfinished", "inProgress", null), + }, + }); + client.emit(turnCompleted("side-thread", "turn-1", "Native tool answer.")); + }, 0); + return turnStartResult("turn-1"); + } + if (method === "thread/unsubscribe" || method === "turn/interrupt") { + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + try { + await runCodexAppServerSideQuestion( + sideParams({ + agentId: "side-agent", + sessionKey: "agent:side-agent:main", + opts: { runId: "run-side-native-unfinished" }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribeDiagnostics(); + } + + expect( + diagnosticEvents.filter( + (event) => + event.type.startsWith("tool.execution.") && + "toolCallId" in event && + event.toolCallId === "native-tool-unfinished", + ), + ).toEqual([ + expect.objectContaining({ + type: "tool.execution.started", + toolCallId: "native-tool-unfinished", + }), + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "native-tool-unfinished", + errorCategory: "codex_native_tool_error", + terminalReason: "failed", + }), + ]); + expect(activeDiagnosticToolKeys(diagnosticEvents)).toEqual(new Set()); + }); + + it("projects snapshot-only native side-thread tools exactly once", async () => { + const client = createFakeClient(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + client.request.mockImplementation(async (method: string) => { + if (method === "thread/fork") { + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + setTimeout(() => { + const notification = turnCompleted("side-thread", "turn-1", "Snapshot answer."); + const turn = (notification.params as JsonObject).turn as JsonObject; + turn.items = [ + nativeCommandItem("snapshot-tool-1", "completed", 19), + ...(turn.items as JsonValue[]), + ]; + client.emit(notification); + }, 0); + return turnStartResult("turn-1"); + } + if (method === "thread/unsubscribe" || method === "turn/interrupt") { + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + try { + await runCodexAppServerSideQuestion( + sideParams({ + agentId: "side-agent", + sessionKey: "agent:side-agent:main", + opts: { runId: "run-side-snapshot-tool" }, + }), + ); + await flushDiagnosticEvents(); + } finally { + unsubscribeDiagnostics(); + } + + expect( + diagnosticEvents + .filter((event) => event.type.startsWith("tool.execution.")) + .map((event) => ({ + type: event.type, + toolCallId: "toolCallId" in event ? event.toolCallId : undefined, + durationMs: "durationMs" in event ? event.durationMs : undefined, + })), + ).toEqual([ + { + type: "tool.execution.started", + toolCallId: "snapshot-tool-1", + durationMs: undefined, + }, + { + type: "tool.execution.completed", + toolCallId: "snapshot-tool-1", + durationMs: 19, + }, + ]); + expect(activeDiagnosticToolKeys(diagnosticEvents)).toEqual(new Set()); + }); + + it("finalizes an active native side-thread tool when side completion times out", async () => { + vi.useFakeTimers(); + const client = createFakeClient(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + client.request.mockImplementation(async (method: string) => { + if (method === "thread/fork") { + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + setTimeout(() => { + client.emit({ + method: "item/started", + params: { + threadId: "side-thread", + turnId: "turn-1", + item: nativeCommandItem("native-tool-timeout", "inProgress", null), + }, + }); + }, 0); + return turnStartResult("turn-1"); + } + if (method === "thread/unsubscribe" || method === "turn/interrupt") { + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + try { + const runResult = runCodexAppServerSideQuestion( + sideParams({ + agentId: "side-agent", + sessionKey: "agent:side-agent:main", + opts: { runId: "run-side-native-timeout" }, + }), + ).catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(600_000); + + await expect(runResult).resolves.toMatchObject({ name: "TimeoutError" }); + await vi.runAllTimersAsync(); + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + type: "tool.execution.error", + agentId: "side-agent", + toolCallId: "native-tool-timeout", + terminalReason: "timed_out", + }), + ); + expect(activeDiagnosticToolKeys(diagnosticEvents)).toEqual(new Set()); + } finally { + unsubscribeDiagnostics(); + } + }); + + it("classifies an active side tool as timed out when side completion expires", async () => { + vi.useFakeTimers(); + const client = createFakeClient(); + const diagnosticEvents: DiagnosticEventPayload[] = []; + const unsubscribeDiagnostics = onInternalDiagnosticEvent((event) => + diagnosticEvents.push(event), + ); + toolExecuteMock.mockImplementation( + (_callId: string, _args: unknown, signal?: AbortSignal) => + new Promise((_resolve, reject) => { + signal?.addEventListener( + "abort", + () => reject(signal.reason instanceof Error ? signal.reason : new Error("aborted")), + { once: true }, + ); + }), + ); + client.request.mockImplementation(async (method: string) => { + if (method === "thread/fork") { + return threadResult("side-thread"); + } + if (method === "thread/inject_items") { + return {}; + } + if (method === "turn/start") { + setTimeout(() => { + void client.handleRequest({ + id: 42, + method: "item/tool/call", + params: { + threadId: "side-thread", + turnId: "turn-1", + callId: "tool-timeout", + tool: "wiki_status", + arguments: {}, + }, + }); + }, 0); + return turnStartResult("turn-1"); + } + if (method === "thread/unsubscribe" || method === "turn/interrupt") { + return {}; + } + throw new Error(`unexpected request: ${method}`); + }); + getSharedCodexAppServerClientMock.mockResolvedValue(client); + + try { + const runPromise = runCodexAppServerSideQuestion( + sideParams({ + agentId: "side-agent", + sessionKey: "global", + opts: { runId: "run-side-timeout" }, + }), + ); + const runResult = runPromise.catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(600_000); + + await expect(runResult).resolves.toMatchObject({ name: "TimeoutError" }); + await vi.advanceTimersByTimeAsync(0); + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + type: "tool.execution.error", + agentId: "side-agent", + toolCallId: "tool-timeout", + terminalReason: "timed_out", + }), + ); + } finally { + unsubscribeDiagnostics(); + } + }); + it("normalizes hook channel ids for side-thread dynamic tool requests", async () => { const beforeToolCall = vi.fn((...args: unknown[]) => { const context = args[1] as { channelId?: string }; diff --git a/extensions/codex/src/app-server/side-question.ts b/extensions/codex/src/app-server/side-question.ts index bce57d15e678..0cb05a02bdde 100644 --- a/extensions/codex/src/app-server/side-question.ts +++ b/extensions/codex/src/app-server/side-question.ts @@ -1,4 +1,5 @@ // Codex plugin module implements side question behavior. +import { randomUUID } from "node:crypto"; import { buildAgentHookContextChannelFields, embeddedAgentLog, @@ -41,16 +42,20 @@ import { emitDynamicToolStartedDiagnostic, emitDynamicToolTerminalDiagnostic, } from "./dynamic-tool-diagnostics.js"; +import { resolveCodexToolAbortTerminalReason } from "./dynamic-tool-execution.js"; import { filterCodexDynamicTools, resolveCodexDynamicToolsLoading, } from "./dynamic-tool-profile.js"; import { createCodexDynamicToolBridge, type CodexDynamicToolBridge } from "./dynamic-tools.js"; import { handleCodexAppServerElicitationRequest } from "./elicitation-bridge.js"; +import { CodexNativeToolLifecycleProjector } from "./event-projector.js"; import { buildCodexNativeHookRelayConfig, buildCodexNativeHookRelayDisabledConfig, CODEX_NATIVE_HOOK_RELAY_EVENTS, + emitCodexNativePreToolUseFailureDiagnostic, + type CodexNativePreToolUseFailure, } from "./native-hook-relay.js"; import { readCodexNotificationThreadId, @@ -105,6 +110,10 @@ const CODEX_SIDE_DYNAMIC_TOOL_MAX_TIMEOUT_MS = 600_000; const CODEX_SIDE_DYNAMIC_IMAGE_GENERATION_TOOL_TIMEOUT_MS = 120_000; const CODEX_SIDE_DYNAMIC_IMAGE_TOOL_TIMEOUT_MS = 60_000; const SIDE_QUESTION_COMPLETION_TIMEOUT_MS = 600_000; + +class CodexSideQuestionTimeoutError extends Error { + override name = "TimeoutError"; +} const CODEX_SIDE_NATIVE_HOOK_RELAY_MIN_TTL_MS = 30 * 60_000; const CODEX_SIDE_NATIVE_HOOK_RELAY_TTL_GRACE_MS = 5 * 60_000; const CODEX_SIDE_NATIVE_HOOK_RELAY_STARTUP_REQUEST_COUNT = 3; @@ -209,7 +218,8 @@ export async function runCodexAppServerSideQuestion( agentDir: params.agentDir, }); const cwd = binding.cwd || params.workspaceDir || process.cwd(); - const sideRunParams = buildSideRunAttemptParams(params, { cwd, authProfileId }); + const runId = params.opts?.runId ?? randomUUID(); + const sideRunParams = buildSideRunAttemptParams(params, { cwd, authProfileId, runId }); const nativeExecutionBlock = resolveCodexNativeExecutionBlock({ config: sideRunParams.config, sessionKey: sideRunParams.sandboxSessionKey?.trim() || sideRunParams.sessionKey, @@ -233,10 +243,60 @@ export async function runCodexAppServerSideQuestion( config: params.cfg, }); const collector = new CodexSideQuestionCollector(params); - const removeNotificationHandler = client.addNotificationHandler((notification) => - collector.handleNotification(notification), - ); const runAbortController = new AbortController(); + let nativeToolLifecycleProjector: CodexNativeToolLifecycleProjector | undefined; + const pendingNativeToolNotifications: CodexServerNotification[] = []; + const pendingNativePreToolUseFailures: CodexNativePreToolUseFailure[] = []; + let nativePreToolUseFailureFallbackActive = false; + let nativeToolRunWasAbortedBeforeCleanup: boolean | undefined; + let nativePreToolUseFailureFallbackTerminalReason: + | CodexNativePreToolUseFailure["disposition"] + | undefined; + const emitNativePreToolUseFailure = (failure: CodexNativePreToolUseFailure) => { + emitCodexNativePreToolUseFailureDiagnostic({ + agentId: sessionAgentId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + runId: sideRunParams.runId, + signal: runAbortController.signal, + failure, + ...(nativePreToolUseFailureFallbackActive + ? { + terminalReason: nativePreToolUseFailureFallbackTerminalReason ?? failure.disposition, + } + : {}), + }); + }; + const flushPendingNativePreToolUseFailures = () => { + for (const failure of pendingNativePreToolUseFailures.splice(0)) { + emitNativePreToolUseFailure(failure); + } + }; + const activateNativePreToolUseFailureFallback = () => { + if (!nativePreToolUseFailureFallbackActive) { + nativePreToolUseFailureFallbackTerminalReason = nativeToolRunWasAbortedBeforeCleanup + ? resolveCodexToolAbortTerminalReason(runAbortController.signal) + : undefined; + nativePreToolUseFailureFallbackActive = true; + } + flushPendingNativePreToolUseFailures(); + }; + const removeNotificationHandler = client.addNotificationHandler((notification) => { + collector.handleNotification(notification); + if ( + notification.method !== "item/started" && + notification.method !== "item/completed" && + notification.method !== "rawResponseItem/completed" && + notification.method !== "turn/completed" + ) { + return; + } + if (!nativeToolLifecycleProjector) { + pendingNativeToolNotifications.push(notification); + return; + } + nativeToolLifecycleProjector.handleNotification(notification); + }); const abortFromUpstream = () => runAbortController.abort(params.opts?.abortSignal?.reason ?? "codex_side_question_abort"); if (params.opts?.abortSignal?.aborted) { @@ -290,6 +350,7 @@ export async function runCodexAppServerSideQuestion( sessionAgentId, nativeToolSurfaceEnabled, nativeProviderWebSearchSupport, + runId, signal: runAbortController.signal, }); removeRequestHandler = client.addRequestHandler(async (request) => { @@ -335,6 +396,8 @@ export async function runCodexAppServerSideQuestion( sandbox, }), signal: runAbortController.signal, + onNativeToolFailureDisposition: (itemId, disposition) => + nativeToolLifecycleProjector?.recordApprovalFailureDisposition(itemId, disposition), }); } if (request.method !== "item/tool/call") { @@ -351,6 +414,7 @@ export async function runCodexAppServerSideQuestion( const toolStartedAt = Date.now(); const diagnosticContext = { call, + agentId: sessionAgentId, runId: sideRunParams.runId, sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -376,6 +440,9 @@ export async function runCodexAppServerSideQuestion( emitDynamicToolErrorDiagnostic({ ...diagnosticContext, durationMs: Math.max(0, Date.now() - toolStartedAt), + terminalReason: runAbortController.signal.aborted + ? resolveCodexToolAbortTerminalReason(runAbortController.signal) + : "failed", }); throw error; } @@ -407,6 +474,18 @@ export async function runCodexAppServerSideQuestion( SIDE_QUESTION_COMPLETION_TIMEOUT_MS, ), signal: runAbortController.signal, + onPreToolUseFailure: (failure) => { + if (nativePreToolUseFailureFallbackActive) { + emitNativePreToolUseFailure(failure); + } else if (nativeToolLifecycleProjector) { + nativeToolLifecycleProjector.recordPreToolUseFailure( + failure, + nativeToolRunWasAbortedBeforeCleanup, + ); + } else { + pendingNativePreToolUseFailures.push(failure); + } + }, }) : undefined; const nativeHookRelayConfig = nativeHookRelay @@ -497,14 +576,38 @@ export async function runCodexAppServerSideQuestion( ); turnId = turnResponse.turn.id; collector.setTurn(childThreadId, turnId); + nativeToolLifecycleProjector = new CodexNativeToolLifecycleProjector( + { ...sideRunParams, agentId: sessionAgentId }, + childThreadId, + turnId, + { + runAbortSignal: runAbortController.signal, + }, + ); + for (const failure of pendingNativePreToolUseFailures) { + nativeToolLifecycleProjector.recordPreToolUseFailure(failure); + } + pendingNativePreToolUseFailures.length = 0; + for (const notification of pendingNativeToolNotifications) { + nativeToolLifecycleProjector.handleNotification(notification); + } + pendingNativeToolNotifications.length = 0; - const text = await collector.wait({ - signal: params.opts?.abortSignal, - timeoutMs: Math.max( - appServer.turnCompletionIdleTimeoutMs, - SIDE_QUESTION_COMPLETION_TIMEOUT_MS, - ), - }); + let text: string; + try { + text = await collector.wait({ + signal: params.opts?.abortSignal, + timeoutMs: Math.max( + appServer.turnCompletionIdleTimeoutMs, + SIDE_QUESTION_COMPLETION_TIMEOUT_MS, + ), + }); + } catch (error) { + if (error instanceof CodexSideQuestionTimeoutError && !runAbortController.signal.aborted) { + runAbortController.abort(error); + } + throw error; + } const trimmed = text.trim(); if (!trimmed) { throw new Error("Codex /btw completed without an answer."); @@ -512,19 +615,36 @@ export async function runCodexAppServerSideQuestion( return { text: trimmed }; } finally { try { + // Cleanup aborts are ownership teardown, not a terminal run outcome. + // Snapshot the real state while late app-server notifications can still drain. + const runWasAbortedBeforeCleanup = runAbortController.signal.aborted; + nativeToolRunWasAbortedBeforeCleanup = runWasAbortedBeforeCleanup; params.opts?.abortSignal?.removeEventListener("abort", abortFromUpstream); + removeRequestHandler?.(); + // Stop dispatched side tools before cleanup waits on the app server; + // otherwise a stuck tool can outlive the side turn that owns it. if (!runAbortController.signal.aborted) { runAbortController.abort("codex_side_question_finished"); } - removeNotificationHandler(); - removeRequestHandler?.(); - await cleanupCodexSideThread(client, { - threadId: childThreadId, - turnId, - interrupt: !collector.completed, - timeoutMs: appServer.requestTimeoutMs, - }); + try { + await cleanupCodexSideThread(client, { + threadId: childThreadId, + turnId, + interrupt: !collector.completed, + timeoutMs: appServer.requestTimeoutMs, + }); + } finally { + removeNotificationHandler(); + try { + nativeToolLifecycleProjector?.finalizeActive(runWasAbortedBeforeCleanup); + } finally { + // Keep cleanup-time relay failures with their active projected item. + // Direct emission owns only failures that arrive after projector retirement. + activateNativePreToolUseFailureFallback(); + } + } } finally { + flushPendingNativePreToolUseFailures(); releaseLeasedSharedCodexAppServerClient(client); nativeHookRelay?.unregister(); } @@ -559,6 +679,7 @@ function registerCodexSideNativeHookRelay(params: { requestTimeoutMs: number; completionTimeoutMs: number; signal: AbortSignal; + onPreToolUseFailure: (failure: CodexNativePreToolUseFailure) => void; }): NativeHookRelayRegistrationHandle | undefined { if (params.options.enabled === false) { return undefined; @@ -578,6 +699,7 @@ function registerCodexSideNativeHookRelay(params: { completionTimeoutMs: params.completionTimeoutMs, }), signal: params.signal, + onPreToolUseFailure: params.onPreToolUseFailure, command: { timeoutMs: params.options.gatewayTimeoutMs, }, @@ -601,7 +723,7 @@ function resolveCodexSideNativeHookRelayTtlMs(params: { function buildSideRunAttemptParams( params: AgentHarnessSideQuestionParams, - options: { cwd: string; authProfileId?: string }, + options: { cwd: string; authProfileId?: string; runId: string }, ): EmbeddedRunAttemptParams { const sideParams = { params, @@ -640,7 +762,7 @@ function buildSideRunAttemptParams( authStorage: undefined as never, authProfileStore: undefined as never, modelRegistry: undefined as never, - runId: params.opts?.runId ?? `codex-btw:${params.sessionId}`, + runId: options.runId, abortSignal: params.opts?.abortSignal, onAgentEvent: (event: { stream: string; data: Record }) => { if (event.stream === "approval") { @@ -660,6 +782,7 @@ async function createCodexSideToolBridge(input: { sessionAgentId: string; nativeToolSurfaceEnabled: boolean; nativeProviderWebSearchSupport: CodexNativeWebSearchSupport; + runId: string; signal: AbortSignal; }): Promise<{ toolBridge: CodexDynamicToolBridge; webSearchPlan: CodexWebSearchPlan }> { const runtimeModel = @@ -688,7 +811,7 @@ async function createCodexSideToolBridge(input: { ? input.params.sessionKey : undefined, sessionId: input.params.sessionId, - runId: input.params.opts?.runId ?? `codex-btw:${input.params.sessionId}`, + runId: input.runId, agentDir: input.params.agentDir ?? resolveAgentDir(input.params.cfg ?? {}, input.sessionAgentId), workspaceDir: input.cwd, @@ -787,7 +910,7 @@ async function createCodexSideToolBridge(input: { config: input.params.cfg, sessionId: input.params.sessionId, sessionKey: input.params.sessionKey, - runId: input.params.opts?.runId ?? `codex-btw:${input.params.sessionId}`, + runId: input.runId, currentChannelProvider: messageToolProvider, ...hookChannelFields, }, @@ -803,7 +926,10 @@ async function handleSideDynamicToolCallWithTimeout(params: { timeoutMs: number; }): Promise { if (params.signal.aborted) { - return failedSideDynamicToolResponse("OpenClaw dynamic tool call aborted before execution."); + return failedSideDynamicToolResponse( + "OpenClaw dynamic tool call aborted before execution.", + resolveCodexToolAbortTerminalReason(params.signal), + ); } const controller = new AbortController(); @@ -812,7 +938,9 @@ async function handleSideDynamicToolCallWithTimeout(params: { const abortFromRun = () => { const message = "OpenClaw dynamic tool call aborted."; controller.abort(params.signal.reason ?? new Error(message)); - resolveAbort?.(failedSideDynamicToolResponse(message)); + resolveAbort?.( + failedSideDynamicToolResponse(message, resolveCodexToolAbortTerminalReason(params.signal)), + ); }; const abortPromise = new Promise((resolve) => { resolveAbort = resolve; @@ -822,7 +950,10 @@ async function handleSideDynamicToolCallWithTimeout(params: { timeout = setTimeout(() => { controller.abort(new Error(`OpenClaw dynamic tool call timed out after ${timeoutMs}ms.`)); resolve( - failedSideDynamicToolResponse(`OpenClaw dynamic tool call timed out after ${timeoutMs}ms.`), + failedSideDynamicToolResponse( + `OpenClaw dynamic tool call timed out after ${timeoutMs}ms.`, + "timed_out", + ), ); }, timeoutMs); timeout.unref?.(); @@ -852,7 +983,10 @@ async function handleSideDynamicToolCallWithTimeout(params: { } } -function failedSideDynamicToolResponse(message: string): CodexDynamicToolCallResponse { +function failedSideDynamicToolResponse( + message: string, + terminalReason: "failed" | "cancelled" | "timed_out" = "failed", +): CodexDynamicToolCallResponse { const response: CodexDynamicToolCallResponse = { contentItems: [{ type: "inputText", text: message }], success: false, @@ -862,6 +996,11 @@ function failedSideDynamicToolResponse(message: string): CodexDynamicToolCallRes enumerable: false, value: "error", }); + Object.defineProperty(response, "diagnosticTerminalReason", { + configurable: true, + enumerable: false, + value: terminalReason, + }); return response; } @@ -1093,7 +1232,11 @@ class CodexSideQuestionCollector { () => { cleanup(); this.settle = undefined; - reject(new Error("Codex /btw timed out waiting for the side thread to finish.")); + reject( + new CodexSideQuestionTimeoutError( + "Codex /btw timed out waiting for the side thread to finish.", + ), + ); }, Math.max(100, options.timeoutMs), ); diff --git a/extensions/codex/src/app-server/tool-abort-terminal-reason.ts b/extensions/codex/src/app-server/tool-abort-terminal-reason.ts new file mode 100644 index 000000000000..796d6f612a2a --- /dev/null +++ b/extensions/codex/src/app-server/tool-abort-terminal-reason.ts @@ -0,0 +1,34 @@ +/** Leaf helper shared by native and dynamic tool diagnostics. */ + +const CODEX_TIMEOUT_ABORT_REASONS = new Set([ + "codex_startup_timeout", + "turn_completion_idle_timeout", + "turn_progress_idle_timeout", + "turn_terminal_idle_timeout", +]); + +/** Preserves timeout provenance when an enclosing run aborts an active tool. */ +export function resolveCodexToolAbortTerminalReason( + signal: AbortSignal, +): "failed" | "cancelled" | "timed_out" { + try { + const reason = signal.reason; + if (typeof reason === "string") { + if (CODEX_TIMEOUT_ABORT_REASONS.has(reason)) { + return "timed_out"; + } + // Transport loss is a run failure, not an operator cancellation. Native + // and dynamic tool diagnostics share this helper and must agree with it. + return reason === "client_closed" ? "failed" : "cancelled"; + } + if (reason && typeof reason === "object") { + const record = reason as { name?: unknown; reason?: unknown }; + if (record.name === "TimeoutError" || record.reason === "timeout") { + return "timed_out"; + } + } + } catch { + return "cancelled"; + } + return "cancelled"; +} diff --git a/packages/acp-core/src/runtime/types.ts b/packages/acp-core/src/runtime/types.ts index 3f24c2e2c343..f620fda424c4 100644 --- a/packages/acp-core/src/runtime/types.ts +++ b/packages/acp-core/src/runtime/types.ts @@ -113,9 +113,22 @@ export type AcpRuntimeEvent = toolCallId?: string; status?: string; title?: string; + kind?: + | "read" + | "edit" + | "delete" + | "move" + | "search" + | "execute" + | "fetch" + | "switch_mode" + | "think" + | "other"; } | { type: "done"; + /** Closed result status when the manager synthesizes the terminal event. */ + status?: "completed" | "cancelled"; stopReason?: string; } | { diff --git a/packages/gateway-protocol/src/index.ts b/packages/gateway-protocol/src/index.ts index 3256d883215a..bca74fa9ccf7 100644 --- a/packages/gateway-protocol/src/index.ts +++ b/packages/gateway-protocol/src/index.ts @@ -12,6 +12,12 @@ import { Compile, type Validator as TypeBoxValidator } from "typebox/compile"; import { type AgentEvent, AgentEventSchema, + type AuditEvent, + AuditEventSchema, + type AuditListParams, + AuditListParamsSchema, + type AuditListResult, + AuditListResultSchema, type AgentIdentityParams, AgentIdentityParamsSchema, type AgentIdentityResult, @@ -637,6 +643,7 @@ export const validateMessageActionParams = export const validateSendParams = lazyCompile(SendParamsSchema); export const validatePollParams = lazyCompile(PollParamsSchema); export const validateAgentParams = lazyCompile(AgentParamsSchema); +export const validateAuditListParams = lazyCompile(AuditListParamsSchema); export const validateAgentIdentityParams = lazyCompile(AgentIdentityParamsSchema); export const validateAgentWaitParams = lazyCompile(AgentWaitParamsSchema); @@ -1186,6 +1193,9 @@ export { ArtifactsListParamsSchema, ArtifactsGetParamsSchema, ArtifactsDownloadParamsSchema, + AuditEventSchema, + AuditListParamsSchema, + AuditListResultSchema, TaskSummarySchema, TasksListParamsSchema, TasksListResultSchema, @@ -1547,6 +1557,9 @@ export type { SessionsDeleteParams, SessionsCompactParams, SessionsUsageParams, + AuditEvent, + AuditListParams, + AuditListResult, TaskSummary, TasksListParams, TasksListResult, diff --git a/packages/gateway-protocol/src/schema.ts b/packages/gateway-protocol/src/schema.ts index c78731dfb292..31ca2e6d2d48 100644 --- a/packages/gateway-protocol/src/schema.ts +++ b/packages/gateway-protocol/src/schema.ts @@ -8,6 +8,7 @@ export * from "./schema/primitives.js"; export * from "./schema/agent.js"; export * from "./schema/agents-models-skills.js"; export * from "./schema/artifacts.js"; +export * from "./schema/audit.js"; export * from "./schema/channels.js"; export * from "./schema/commands.js"; export * from "./schema/config.js"; diff --git a/packages/gateway-protocol/src/schema/audit.test.ts b/packages/gateway-protocol/src/schema/audit.test.ts new file mode 100644 index 000000000000..8c553b83cfc3 --- /dev/null +++ b/packages/gateway-protocol/src/schema/audit.test.ts @@ -0,0 +1,43 @@ +import { Compile } from "typebox/compile"; +import { describe, expect, it } from "vitest"; +import { validateAuditListParams } from "../index.js"; +import { AuditEventSchema } from "./audit.js"; + +describe("audit protocol schemas", () => { + it("accepts bounded query filters and rejects oversized pages", () => { + expect( + validateAuditListParams({ + agentId: "main", + kind: "tool_action", + status: "failed", + limit: 500, + cursor: "42", + }), + ).toBe(true); + expect(validateAuditListParams({ status: "unknown" })).toBe(true); + expect(validateAuditListParams({ limit: 501 })).toBe(false); + }); + + it("rejects content fields from metadata-only records", () => { + const validate = Compile(AuditEventSchema); + const event = { + eventId: "event-1", + sequence: 1, + sourceSequence: 1, + occurredAt: 1, + kind: "tool_action", + action: "tool.action.finished", + status: "unknown", + errorCode: "tool_outcome_unknown", + actor: { type: "agent", id: "main" }, + agentId: "main", + sessionKey: "agent:main:main", + runId: "run-1", + toolCallId: "call-1", + toolName: "exec", + redaction: "metadata_only", + }; + expect(validate.Check(event)).toBe(true); + expect(validate.Check({ ...event, result: "secret" })).toBe(false); + }); +}); diff --git a/packages/gateway-protocol/src/schema/audit.ts b/packages/gateway-protocol/src/schema/audit.ts new file mode 100644 index 000000000000..67384348bb5b --- /dev/null +++ b/packages/gateway-protocol/src/schema/audit.ts @@ -0,0 +1,91 @@ +// Gateway Protocol schema module defines metadata-only audit query payloads. +import { Type } from "typebox"; +import { NonEmptyString } from "./primitives.js"; + +export const AuditEventKindSchema = Type.Union([ + Type.Literal("agent_run"), + Type.Literal("tool_action"), +]); + +export const AuditEventActionSchema = Type.Union([ + Type.Literal("agent.run.started"), + Type.Literal("agent.run.finished"), + Type.Literal("tool.action.started"), + Type.Literal("tool.action.finished"), +]); + +export const AuditEventStatusSchema = Type.Union([ + Type.Literal("started"), + Type.Literal("succeeded"), + Type.Literal("failed"), + Type.Literal("cancelled"), + Type.Literal("timed_out"), + Type.Literal("blocked"), + Type.Literal("unknown"), +]); + +export const AuditEventErrorCodeSchema = Type.Union([ + Type.Literal("run_failed"), + Type.Literal("run_cancelled"), + Type.Literal("run_timed_out"), + Type.Literal("run_blocked"), + Type.Literal("tool_failed"), + Type.Literal("tool_cancelled"), + Type.Literal("tool_timed_out"), + Type.Literal("tool_blocked"), + Type.Literal("tool_outcome_unknown"), +]); + +/** One content-free run/tool audit record. */ +export const AuditEventSchema = Type.Object( + { + eventId: NonEmptyString, + sequence: Type.Integer({ minimum: 1 }), + sourceSequence: Type.Integer({ minimum: 1 }), + occurredAt: Type.Integer({ minimum: 0 }), + kind: AuditEventKindSchema, + action: AuditEventActionSchema, + status: AuditEventStatusSchema, + errorCode: Type.Optional(AuditEventErrorCodeSchema), + actor: Type.Object( + { + type: Type.Union([Type.Literal("agent"), Type.Literal("system")]), + id: NonEmptyString, + }, + { additionalProperties: false }, + ), + agentId: NonEmptyString, + sessionKey: Type.Optional(NonEmptyString), + sessionId: Type.Optional(NonEmptyString), + runId: NonEmptyString, + toolCallId: Type.Optional(NonEmptyString), + toolName: Type.Optional(NonEmptyString), + redaction: Type.Literal("metadata_only"), + }, + { additionalProperties: false }, +); + +/** Bounded newest-first audit query filters. */ +export const AuditListParamsSchema = Type.Object( + { + agentId: Type.Optional(NonEmptyString), + sessionKey: Type.Optional(NonEmptyString), + runId: Type.Optional(NonEmptyString), + kind: Type.Optional(AuditEventKindSchema), + status: Type.Optional(AuditEventStatusSchema), + after: Type.Optional(Type.Integer({ minimum: 0 })), + before: Type.Optional(Type.Integer({ minimum: 0 })), + limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 500 })), + cursor: Type.Optional(NonEmptyString), + }, + { additionalProperties: false }, +); + +/** Stable sequence-cursor page suitable for bounded JSON export. */ +export const AuditListResultSchema = Type.Object( + { + events: Type.Array(AuditEventSchema), + nextCursor: Type.Optional(NonEmptyString), + }, + { additionalProperties: false }, +); diff --git a/packages/gateway-protocol/src/schema/protocol-schemas.ts b/packages/gateway-protocol/src/schema/protocol-schemas.ts index 056179ee0bdf..cd17fb25424e 100644 --- a/packages/gateway-protocol/src/schema/protocol-schemas.ts +++ b/packages/gateway-protocol/src/schema/protocol-schemas.ts @@ -88,6 +88,7 @@ import { ArtifactsListParamsSchema, ArtifactsListResultSchema, } from "./artifacts.js"; +import { AuditEventSchema, AuditListParamsSchema, AuditListResultSchema } from "./audit.js"; import { ChannelsStartParamsSchema, ChannelsStopParamsSchema, @@ -466,7 +467,10 @@ export const ProtocolSchemas = { SessionsCompactParams: SessionsCompactParamsSchema, SessionsUsageParams: SessionsUsageParamsSchema, - // Task ledger and config/wizard setup payloads. + // Audit/task ledgers and config/wizard setup payloads. + AuditEvent: AuditEventSchema, + AuditListParams: AuditListParamsSchema, + AuditListResult: AuditListResultSchema, TaskSummary: TaskSummarySchema, TasksListParams: TasksListParamsSchema, TasksListResult: TasksListResultSchema, diff --git a/packages/gateway-protocol/src/schema/types.ts b/packages/gateway-protocol/src/schema/types.ts index d12f3bc55fb3..30d27c95ce7c 100644 --- a/packages/gateway-protocol/src/schema/types.ts +++ b/packages/gateway-protocol/src/schema/types.ts @@ -109,6 +109,11 @@ export type SessionsDeleteParams = SchemaType<"SessionsDeleteParams">; export type SessionsCompactParams = SchemaType<"SessionsCompactParams">; export type SessionsUsageParams = SchemaType<"SessionsUsageParams">; +/** Metadata-only audit query payloads. */ +export type AuditEvent = SchemaType<"AuditEvent">; +export type AuditListParams = SchemaType<"AuditListParams">; +export type AuditListResult = SchemaType<"AuditListResult">; + /** Task ledger query and cancellation payloads. */ export type TaskSummary = SchemaType<"TaskSummary">; export type TasksListParams = SchemaType<"TasksListParams">; diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index c66646f4f8f0..067888b19e04 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -113,7 +113,7 @@ export const defaultPublicDeprecatedExportsByEntrypointBudget = Object.freeze({ "outbound-send-deps": 4, "outbound-runtime": 16, "file-access-runtime": 2, - "infra-runtime": 585, + "infra-runtime": 588, "ssrf-policy": 1, "ssrf-runtime": 1, "media-runtime": 2, @@ -192,12 +192,12 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { ), publicExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", - 10435, + 10453, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", - 5207, + 5218, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/scripts/release-check.ts b/scripts/release-check.ts index 7437bb8cc75f..671274f5ba47 100755 --- a/scripts/release-check.ts +++ b/scripts/release-check.ts @@ -95,6 +95,7 @@ const requiredPathGroups = [ "dist/plugin-sdk/root-alias.cjs", "dist/agents/compaction-planning.worker.js", "dist/agents/model-provider-auth.worker.js", + "dist/audit/audit-event-writer.worker.js", "dist/task-registry-control.runtime.js", "dist/telegram-ingress-worker.runtime.js", "dist/build-info.json", diff --git a/src/acp/control-plane/manager.turn-results.test.ts b/src/acp/control-plane/manager.turn-results.test.ts index 5eab75ddd976..d44fbec2f4c0 100644 --- a/src/acp/control-plane/manager.turn-results.test.ts +++ b/src/acp/control-plane/manager.turn-results.test.ts @@ -671,7 +671,6 @@ describe("AcpSessionManager turn results", () => { })(), result: Promise.resolve({ status: "cancelled" as const, - stopReason: "manual-cancel", }), cancel: vi.fn(async () => {}), closeStream, @@ -686,7 +685,7 @@ describe("AcpSessionManager turn results", () => { acp: readySessionMeta(), }); - const events: string[] = []; + const events: AcpRuntimeEvent[] = []; const manager = new AcpSessionManager(); await manager.runTurn({ cfg: baseCfg, @@ -695,13 +694,14 @@ describe("AcpSessionManager turn results", () => { mode: "prompt", requestId: "run-1", onEvent: (event) => { - events.push(event.type); + events.push(event); }, }); expect(runtimeState.runTurn).not.toHaveBeenCalled(); expect(closeStream).toHaveBeenCalledWith({ reason: "turn-result-cancelled" }); - expect(events).toEqual(["text_delta", "done"]); + expect(events.map((event) => event.type)).toEqual(["text_delta", "done"]); + expect(events.at(-1)).toEqual({ type: "done", status: "cancelled" }); const states = extractStatesFromUpserts(); expect(states).toContain("running"); expect(states).toContain("idle"); diff --git a/src/acp/control-plane/manager.turn-stream.ts b/src/acp/control-plane/manager.turn-stream.ts index 7452505f8283..3c30670f1d92 100644 --- a/src/acp/control-plane/manager.turn-stream.ts +++ b/src/acp/control-plane/manager.turn-stream.ts @@ -83,16 +83,10 @@ async function notifyTerminalResult(params: { if (!params.eventGate.open) { return; } - if (params.result.status === "completed") { - await params.onEvent?.({ - type: "done", - ...(params.result.stopReason ? { stopReason: params.result.stopReason } : {}), - }); - return; - } - if (params.result.status === "cancelled") { + if (params.result.status === "completed" || params.result.status === "cancelled") { await params.onEvent?.({ type: "done", + status: params.result.status, ...(params.result.stopReason ? { stopReason: params.result.stopReason } : {}), }); return; diff --git a/src/acp/control-plane/manager.turn-timeout.ts b/src/acp/control-plane/manager.turn-timeout.ts index 8b1cdc3149d3..79583fffd12e 100644 --- a/src/acp/control-plane/manager.turn-timeout.ts +++ b/src/acp/control-plane/manager.turn-timeout.ts @@ -10,6 +10,7 @@ import { resolveRuntimeOptionsFromMeta } from "./runtime-options.js"; const ACP_TURN_TIMEOUT_CLEANUP_GRACE_MS = 2_000; const ACP_TURN_TIMEOUT_REASON = "turn-timeout"; +export const ACP_TURN_TIMEOUT_DETAIL_CODE = "TURN_TIMEOUT"; /** Resolves the effective ACP turn timeout from session runtime options or agent defaults. */ export function resolveTurnTimeoutMs(params: { @@ -96,6 +97,7 @@ export async function awaitTurnWithTimeout(params: { throw new AcpRuntimeError( "ACP_TURN_FAILED", `ACP turn timed out after ${Math.max(1, Math.round(params.timeoutLabelMs / 1_000))}s.`, + { detailCode: ACP_TURN_TIMEOUT_DETAIL_CODE }, ); } if (outcome.kind === "error") { diff --git a/src/acp/tool-status.ts b/src/acp/tool-status.ts new file mode 100644 index 000000000000..a4a45b679bc1 --- /dev/null +++ b/src/acp/tool-status.ts @@ -0,0 +1,20 @@ +import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce"; + +const ACP_TOOL_TERMINAL_OUTCOMES = { + completed: "completed", + done: "completed", + failed: "failed", + error: "failed", + cancelled: "cancelled", +} as const; + +export type AcpToolTerminalOutcome = + (typeof ACP_TOOL_TERMINAL_OUTCOMES)[keyof typeof ACP_TOOL_TERMINAL_OUTCOMES]; + +export function resolveAcpToolTerminalOutcome(status: unknown): AcpToolTerminalOutcome | undefined { + const normalized = normalizeOptionalLowercaseString(status); + if (!normalized || !Object.hasOwn(ACP_TOOL_TERMINAL_OUTCOMES, normalized)) { + return undefined; + } + return ACP_TOOL_TERMINAL_OUTCOMES[normalized as keyof typeof ACP_TOOL_TERMINAL_OUTCOMES]; +} diff --git a/src/agents/agent-command.live-model-switch.test.ts b/src/agents/agent-command.live-model-switch.test.ts index d0cdca6ead0e..0e11858eab7e 100644 --- a/src/agents/agent-command.live-model-switch.test.ts +++ b/src/agents/agent-command.live-model-switch.test.ts @@ -81,6 +81,11 @@ vi.mock("./model-fallback.js", () => ({ vi.mock("./command/attempt-execution.runtime.js", () => ({ buildAcpResult: (...args: unknown[]) => state.buildAcpResultMock(...args), + createAcpToolLifecycleTracker: () => ({ + active: new Map(), + terminalToolCallIds: new Set(), + saturated: false, + }), createAcpVisibleTextAccumulator: () => state.createAcpVisibleTextAccumulatorMock(), emitAcpAssistantDelta: vi.fn(), emitAcpLifecycleEnd: (...args: unknown[]) => state.emitAcpLifecycleEndMock(...args), @@ -2625,6 +2630,14 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { expect(state.persistSessionEntryMock).not.toHaveBeenCalled(); expect(state.updateSessionStoreAfterAgentRunMock).not.toHaveBeenCalled(); expect(sessionStore["agent:main:main"]).toBe(visibleEntry); + expect(state.registerAgentRunContextMock).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + sessionKey: "agent:main:main", + sessionId: "session-1", + isControlUiVisible: false, + }), + ); }); it("does not duplicate finishing lifecycle when an attempt already emitted finishing", async () => { @@ -3578,6 +3591,31 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { expect(transcriptParams.transcriptBody).not.toContain(INTERNAL_RUNTIME_CONTEXT_END); }); + it("keeps session provenance for internal ACP turns", async () => { + state.acpResolveSessionMock.mockReturnValue({ + kind: "ready", + meta: { + agent: "claude", + cwd: "/tmp/workspace", + }, + }); + + await agentCommand({ + message: "internal ACP turn", + sessionKey: "agent:main:main", + sessionEffects: "internal", + }); + + expect(state.registerAgentRunContextMock).toHaveBeenCalledWith( + "session-1", + expect.objectContaining({ + sessionKey: "agent:main:main", + sessionId: "session-1", + isControlUiVisible: false, + }), + ); + }); + it("allows manual ACP spawn turns when ACP dispatch is disabled", async () => { state.acpResolveSessionMock.mockReturnValue({ kind: "ready", @@ -3623,6 +3661,35 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { expect(state.resolveAcpExplicitTurnPolicyErrorMock).not.toHaveBeenCalled(); expect(state.resolveAcpDispatchPolicyErrorMock).toHaveBeenCalledTimes(1); expect(state.acpRunTurnMock).not.toHaveBeenCalled(); + expect(state.emitAcpLifecycleErrorMock).toHaveBeenCalledWith( + expect.objectContaining({ terminalOutcome: "blocked" }), + ); + }); + + it("preserves ACP cancelled results without a stop reason", async () => { + state.acpResolveSessionMock.mockReturnValue({ + kind: "ready", + meta: { + agent: "claude", + cwd: "/tmp/workspace", + }, + }); + state.acpRunTurnMock.mockImplementationOnce(async (params: unknown) => { + const onEvent = (params as { onEvent?: (event: unknown) => void }).onEvent; + onEvent?.({ type: "done", status: "cancelled" }); + }); + + await agentCommand({ + message: "cancelled ACP turn", + sessionKey: "agent:main:main", + }); + + expect(state.emitAcpLifecycleEndMock).toHaveBeenCalledWith( + expect.objectContaining({ resultStatus: "cancelled", stopReason: undefined }), + ); + expect(state.buildAcpResultMock).toHaveBeenCalledWith( + expect.objectContaining({ resultStatus: "cancelled", stopReason: undefined }), + ); }); it("flips hasSessionModelOverride on provider-only switch with same model", async () => { diff --git a/src/agents/agent-command.ts b/src/agents/agent-command.ts index a13a4181b0ec..690b09f32d95 100644 --- a/src/agents/agent-command.ts +++ b/src/agents/agent-command.ts @@ -143,6 +143,7 @@ import { isAgentRunDirectAbortReason, isAgentRunRestartAbortReason, resolveAgentRunAbortLifecycleFields, + resolveAgentRunErrorLifecycleFields, } from "./run-termination.js"; import { normalizeSpawnedRunMetadata } from "./spawned-context.js"; import { resolveAgentTimeoutMs } from "./timeout.js"; @@ -999,21 +1000,26 @@ async function agentCommandInternal( if (!isRawModelRun && acpResolution?.kind === "ready" && sessionKey) { assertAgentRunLifecycleGenerationCurrent(lifecycleGeneration); const attemptExecutionRuntime = await loadAttemptExecutionRuntime(); + const acpToolTracker = attemptExecutionRuntime.createAcpToolLifecycleTracker(); const startedAt = Date.now(); - registerAgentRunContext( + registerAgentRunContext(runId, { + sessionKey, + sessionId, + agentId: sessionAgentId, + lifecycleGeneration, + ...(suppressVisibleSessionEffects ? { isControlUiVisible: false } : {}), + }); + attemptExecutionRuntime.emitAcpLifecycleStart({ runId, - suppressVisibleSessionEffects - ? { isControlUiVisible: false, lifecycleGeneration } - : { - sessionKey, - sessionId, - lifecycleGeneration, - }, - ); - attemptExecutionRuntime.emitAcpLifecycleStart({ runId, startedAt, lifecycleGeneration }); + startedAt, + agentId: sessionAgentId, + lifecycleGeneration, + }); const visibleTextAccumulator = attemptExecutionRuntime.createAcpVisibleTextAccumulator(); let stopReason: string | undefined; + let resultStatus: "completed" | "cancelled" | undefined; + let terminalOutcome: "blocked" | undefined; try { const { resolveAcpAgentPolicyError, @@ -1025,6 +1031,7 @@ async function agentCommandInternal( ? resolveAcpExplicitTurnPolicyError(cfg) : resolveAcpDispatchPolicyError(cfg); if (turnPolicyError) { + terminalOutcome = "blocked"; throw turnPolicyError; } const acpAgent = normalizeAgentId( @@ -1032,6 +1039,7 @@ async function agentCommandInternal( ); const agentPolicyError = resolveAcpAgentPolicyError(cfg, acpAgent); if (agentPolicyError) { + terminalOutcome = "blocked"; throw agentPolicyError; } @@ -1058,12 +1066,16 @@ async function agentCommandInternal( if (event.type !== "text_delta") { attemptExecutionRuntime.emitAcpRuntimeEvent({ runId, + toolTracker: acpToolTracker, sessionKey, + agentId: sessionAgentId, + abortSignal: opts.abortSignal, event, }); } if (event.type === "done") { stopReason = event.stopReason; + resultStatus = event.status; return; } if (event.type !== "text_delta") { @@ -1098,10 +1110,13 @@ async function agentCommandInternal( }); attemptExecutionRuntime.emitAcpLifecycleError({ runId, + toolTracker: acpToolTracker, error: acpError, sessionKey, + agentId: sessionAgentId, lifecycleGeneration, abortSignal: opts.abortSignal, + ...(terminalOutcome ? { terminalOutcome } : {}), }); throw acpError; } @@ -1166,8 +1181,10 @@ async function agentCommandInternal( if (isAgentRunRestartAbortReason(restartAbortReason)) { attemptExecutionRuntime.emitAcpLifecycleError({ runId, + toolTracker: acpToolTracker, error: restartAbortReason, sessionKey, + agentId: sessionAgentId, lifecycleGeneration, abortSignal: opts.abortSignal, }); @@ -1175,8 +1192,12 @@ async function agentCommandInternal( } attemptExecutionRuntime.emitAcpLifecycleEnd({ runId, + toolTracker: acpToolTracker, + agentId: sessionAgentId, lifecycleGeneration, abortSignal: opts.abortSignal, + stopReason, + resultStatus, }); const result = applyAgentRunAbortMetadata( @@ -1184,6 +1205,7 @@ async function agentCommandInternal( payloadText: finalText, startedAt, stopReason, + resultStatus, abortSignal: opts.abortSignal, }), opts.abortSignal, @@ -1214,7 +1236,8 @@ async function agentCommandInternal( assertAgentRunLifecycleGenerationCurrent(lifecycleGeneration); if (sessionKey || suppressVisibleSessionEffects) { registerAgentRunContext(runId, { - ...(sessionKey && !suppressVisibleSessionEffects ? { sessionKey, sessionId } : {}), + ...(sessionKey ? { sessionKey, sessionId } : {}), + agentId: sessionAgentId, lifecycleGeneration, verboseLevel: resolvedVerboseLevel, isControlUiVisible: !suppressVisibleSessionEffects, @@ -1872,7 +1895,7 @@ async function agentCommandInternal( startedAt, endedAt: Date.now(), error: error instanceof Error ? error.message : "Agent run failed", - ...resolveAgentRunAbortLifecycleFields(opts.abortSignal), + ...resolveAgentRunErrorLifecycleFields(error, opts.abortSignal), }, }); }; @@ -2210,9 +2233,9 @@ async function agentCommandInternal( continue; } if (!attemptLifecycleState.lifecycleEnded) { - const abortLifecycleFields = isAgentRunDirectAbortReason(err) + const errorLifecycleFields = isAgentRunDirectAbortReason(err) ? { aborted: true as const, stopReason: "aborted" as const } - : resolveAgentRunAbortLifecycleFields(opts.abortSignal); + : resolveAgentRunErrorLifecycleFields(err, opts.abortSignal); emitAgentEvent({ runId, lifecycleGeneration, @@ -2222,7 +2245,7 @@ async function agentCommandInternal( startedAt, endedAt: Date.now(), error: err instanceof Error ? err.message : "Agent run failed", - ...abortLifecycleFields, + ...errorLifecycleFields, }, }); } diff --git a/src/agents/agent-run-terminal-outcome.test.ts b/src/agents/agent-run-terminal-outcome.test.ts index a29d69b622b2..80a262b1256c 100644 --- a/src/agents/agent-run-terminal-outcome.test.ts +++ b/src/agents/agent-run-terminal-outcome.test.ts @@ -8,11 +8,12 @@ import { describe("agent run terminal outcome", () => { it("treats provider/preflight/post-turn timeout phases as hard run timeouts", () => { expect( - ["preflight", "provider", "post_turn", "queue", "gateway_draining"].map((timeoutPhase) => - buildAgentRunTerminalOutcome({ - status: "timeout", - timeoutPhase, - }).reason + ["preflight", "provider", "post_turn", "queue", "gateway_draining"].map( + (timeoutPhase) => + buildAgentRunTerminalOutcome({ + status: "timeout", + timeoutPhase, + }).reason, ), ).toEqual(["hard_timeout", "hard_timeout", "hard_timeout", "timed_out", "timed_out"]); }); @@ -51,7 +52,7 @@ describe("agent run terminal outcome", () => { }); expect(rpcCancel.reason).toBe("cancelled"); - expect(rpcCancel.status).toBe("timeout"); + expect(rpcCancel.status).toBe("error"); expect(mergeAgentRunTerminalOutcome(rpcCancel, lateCompletion)).toBe(rpcCancel); expect( buildAgentRunTerminalOutcome({ @@ -77,7 +78,7 @@ describe("agent run terminal outcome", () => { expect(restartCancel).toMatchObject({ reason: "cancelled", - status: "timeout", + status: "error", stopReason: "restart", }); expect(mergeAgentRunTerminalOutcome(restartCancel, lateCompletion)).toBe(restartCancel); @@ -205,6 +206,35 @@ describe("agent run terminal outcome", () => { }); }); + it("classifies abandoned successful waits as incomplete failures", () => { + expect( + buildAgentRunTerminalOutcome({ + status: "ok", + livenessState: "abandoned", + }), + ).toEqual({ + reason: "abandoned", + status: "error", + error: "Agent run ended before producing a complete result.", + livenessState: "abandoned", + }); + }); + + it("keeps explicit cancellation ahead of abandoned liveness", () => { + expect( + buildAgentRunTerminalOutcome({ + status: "error", + stopReason: "stop", + livenessState: "abandoned", + }), + ).toEqual({ + reason: "cancelled", + status: "error", + stopReason: "stop", + livenessState: "abandoned", + }); + }); + it("keeps a hard timeout over later aborts or failures for the same run", () => { const timeout = buildAgentRunTerminalOutcome({ status: "timeout", diff --git a/src/agents/agent-run-terminal-outcome.ts b/src/agents/agent-run-terminal-outcome.ts index 6cf7e2a8f552..15d364af4970 100644 --- a/src/agents/agent-run-terminal-outcome.ts +++ b/src/agents/agent-run-terminal-outcome.ts @@ -1,5 +1,10 @@ /** Normalizes agent run wait/liveness/timeout metadata into sticky terminal outcomes. */ -import { formatBlockedLivenessError, isBlockedLivenessState } from "../shared/agent-liveness.js"; +import { + formatAbandonedLivenessError, + formatBlockedLivenessError, + isAbandonedLivenessState, + isBlockedLivenessState, +} from "../shared/agent-liveness.js"; import { AGENT_RUN_ABORTED_ERROR, AGENT_RUN_RESTART_ABORT_STOP_REASON, @@ -22,6 +27,7 @@ type AgentRunTerminalReason = | "cancelled" | "aborted" | "blocked" + | "abandoned" | "failed"; /** Normalized terminal outcome for an agent run. */ @@ -54,6 +60,9 @@ type AgentRunTerminalWaitInput = Omit & { status?: unknown; }; +/** Shared grace window for terminal observations that may still be followed by a retry. */ +export const AGENT_RUN_TERMINAL_RETRY_GRACE_MS = 15_000; + const HARD_TIMEOUT_PHASES = new Set(["preflight", "provider", "post_turn"]); function asFiniteTimestamp(value: unknown): number | undefined { @@ -115,13 +124,18 @@ export function buildAgentRunTerminalOutcome( const cancelled = restartCancelled || (input.status !== "ok" && isCancellationStopReason(stopReason)); const blocked = isBlockedLivenessState(livenessState); + const abandoned = isAbandonedLivenessState(livenessState); const error = hardTimeout ? rawError : blocked ? formatBlockedLivenessError(rawError) : aborted && !rawError ? AGENT_RUN_ABORTED_ERROR - : rawError; + : aborted || cancelled + ? rawError + : abandoned + ? formatAbandonedLivenessError(rawError) + : rawError; const reason: AgentRunTerminalReason = hardTimeout ? "hard_timeout" : blocked @@ -130,18 +144,19 @@ export function buildAgentRunTerminalOutcome( ? "aborted" : cancelled ? "cancelled" - : input.status === "timeout" - ? "timed_out" - : input.status === "error" - ? "failed" - : "completed"; + : abandoned + ? "abandoned" + : input.status === "timeout" + ? "timed_out" + : input.status === "error" + ? "failed" + : "completed"; return { reason, status: reason === "completed" ? "ok" - : reason === "hard_timeout" || - (input.status === "timeout" && (reason === "timed_out" || reason === "cancelled")) + : reason === "hard_timeout" || reason === "timed_out" ? "timeout" : "error", ...(error ? { error } : {}), diff --git a/src/agents/agent-tools.before-tool-call.e2e.test.ts b/src/agents/agent-tools.before-tool-call.e2e.test.ts index 1ea5fa46c71c..d72bdd4816b5 100644 --- a/src/agents/agent-tools.before-tool-call.e2e.test.ts +++ b/src/agents/agent-tools.before-tool-call.e2e.test.ts @@ -26,6 +26,7 @@ import { setPluginToolMeta } from "../plugins/tools.js"; import { createCanonicalFixtureSkill } from "../skills/test-support/test-helpers.js"; import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js"; import { + getBeforeToolCallFailureDisposition, getBeforeToolCallPolicyDiagnosticState, runBeforeToolCallHook, wrapToolWithBeforeToolCallHook, @@ -573,6 +574,248 @@ describe("before_tool_call loop detection behavior", () => { }); }); + it.each([ + { label: "fails", error: new Error("hook crashed"), terminalReason: "failed" }, + { + label: "times out", + error: Object.assign(new Error("timed out after 5ms"), { name: "TimeoutError" }), + terminalReason: "timed_out", + }, + ] as const)( + "emits a terminal diagnostic when a before_tool_call hook $label", + async (testCase) => { + hookRunner.hasHooks.mockImplementation((hookName: string) => hookName === "before_tool_call"); + hookRunner.runBeforeToolCall.mockRejectedValueOnce(testCase.error); + const execute = vi.fn().mockResolvedValue({ content: [{ type: "text", text: "ok" }] }); + const tool = wrapToolWithBeforeToolCallHook( + { name: "exec", execute } as unknown as AnyAgentTool, + { + agentId: "main", + sessionKey: "session-key", + sessionId: "session-id", + runId: "run-1", + loopDetection: { enabled: false }, + }, + ); + + await withToolExecutionEvents(async (emitted, flush) => { + await expect( + tool.execute("tool-call-hook-failure", { command: "private" }, undefined, undefined), + ).rejects.toThrow("Tool call blocked because before_tool_call hook failed"); + await flush(); + + expect(execute).not.toHaveBeenCalled(); + expect(emitted.map((event) => event.type)).toEqual(["tool.execution.error"]); + const terminal = expectEventFields(emitted[0], { + type: "tool.execution.error", + runId: "run-1", + sessionKey: "session-key", + sessionId: "session-id", + agentId: "main", + toolName: "exec", + toolCallId: "tool-call-hook-failure", + paramsSummary: { kind: "object" }, + errorCategory: "before_tool_call", + terminalReason: testCase.terminalReason, + }); + expect(typeof terminal.durationMs).toBe("number"); + expect(JSON.stringify(emitted)).not.toContain("private"); + }); + }, + ); + + it("emits a terminal diagnostic when hook preflight rejects", async () => { + const execute = vi.fn().mockResolvedValue({ content: [{ type: "text", text: "ok" }] }); + const params = Object.defineProperty({}, "private", { + enumerable: true, + get() { + throw new Error("private hook preflight failure"); + }, + }); + const tool = wrapToolWithBeforeToolCallHook( + { name: "read", execute } as unknown as AnyAgentTool, + { + agentId: "main", + sessionKey: "session-key", + runId: "run-1", + }, + ); + + await withToolExecutionEvents(async (emitted, flush) => { + await expect( + tool.execute("tool-call-preflight", params, undefined, undefined), + ).rejects.toThrow("Tool call blocked because before_tool_call hook failed"); + await flush(); + + expect(execute).not.toHaveBeenCalled(); + expect(emitted.map((event) => event.type)).toEqual(["tool.execution.error"]); + expectEventFields(emitted[0], { + type: "tool.execution.error", + runId: "run-1", + sessionKey: "session-key", + agentId: "main", + toolName: "read", + toolCallId: "tool-call-preflight", + paramsSummary: { kind: "object" }, + errorCategory: "before_tool_call", + terminalReason: "failed", + }); + expect(JSON.stringify(emitted)).not.toContain("private hook preflight failure"); + }); + }); + + it("preserves preparation timeout disposition when wrapper diagnostics are delegated", async () => { + const timeout = Object.assign(new Error("private preparation timeout"), { + name: "TimeoutError", + }); + const tool = wrapToolWithBeforeToolCallHook( + { + name: "exec", + execute: vi.fn(), + prepareBeforeToolCallParams: vi.fn().mockRejectedValue(timeout), + } as unknown as AnyAgentTool, + { runId: "run-1" }, + { emitDiagnostics: false }, + ); + + const error = await tool + .execute("tool-call-preparation-timeout", { command: "private" }, undefined, undefined) + .catch((cause: unknown) => cause); + + expect(getBeforeToolCallFailureDisposition(error)).toBe("timed_out"); + expect(error).toHaveProperty("cause", timeout); + }); + + it("emits a blocked terminal diagnostic when tool approval is denied", async () => { + hookRunner.hasHooks.mockImplementation((hookName: string) => hookName === "before_tool_call"); + hookRunner.runBeforeToolCall.mockResolvedValueOnce({ + requireApproval: { title: "Approve", description: "Approve tool" }, + }); + const mockCallGateway = vi.mocked(callGatewayTool); + mockCallGateway.mockResolvedValueOnce({ id: "approval-1", decision: "deny" }); + const execute = vi.fn().mockResolvedValue({ content: [{ type: "text", text: "ok" }] }); + const tool = wrapToolWithBeforeToolCallHook( + { name: "exec", execute } as unknown as AnyAgentTool, + { agentId: "main", sessionKey: "session-key", runId: "run-1" }, + ); + + await withToolExecutionEvents(async (emitted, flush) => { + await expect( + tool.execute("tool-call-denied", { command: "private" }, undefined, undefined), + ).rejects.toThrow("Denied by user"); + await flush(); + + expect(execute).not.toHaveBeenCalled(); + expect(emitted.map((event) => event.type)).toEqual(["tool.execution.blocked"]); + expectEventFields(emitted[0], { + type: "tool.execution.blocked", + runId: "run-1", + sessionKey: "session-key", + toolName: "exec", + toolCallId: "tool-call-denied", + deniedReason: "plugin-approval", + reason: "plugin-approval", + }); + expect(JSON.stringify(emitted)).not.toContain("private"); + }); + mockCallGateway.mockReset(); + }); + + it("emits a blocked terminal diagnostic when approval is report-only", async () => { + hookRunner.hasHooks.mockImplementation((hookName: string) => hookName === "before_tool_call"); + hookRunner.runBeforeToolCall.mockResolvedValueOnce({ + requireApproval: { title: "Approve", description: "Review before running" }, + }); + const mockCallGateway = vi.mocked(callGatewayTool); + const execute = vi.fn().mockResolvedValue({ content: [{ type: "text", text: "ok" }] }); + const tool = wrapToolWithBeforeToolCallHook( + { name: "exec", execute } as unknown as AnyAgentTool, + { agentId: "main", sessionKey: "session-key", runId: "run-1" }, + { approvalMode: "report" }, + ); + + await withToolExecutionEvents(async (emitted, flush) => { + await expect( + tool.execute("tool-call-report", { command: "private" }, undefined, undefined), + ).rejects.toThrow("Review before running"); + await flush(); + + expect(execute).not.toHaveBeenCalled(); + expect(mockCallGateway).not.toHaveBeenCalled(); + expect(emitted.map((event) => event.type)).toEqual(["tool.execution.blocked"]); + expectEventFields(emitted[0], { + type: "tool.execution.blocked", + runId: "run-1", + sessionKey: "session-key", + toolName: "exec", + toolCallId: "tool-call-report", + deniedReason: "plugin-approval", + reason: "plugin-approval", + }); + expect(JSON.stringify(emitted)).not.toContain("private"); + expect(JSON.stringify(emitted)).not.toContain("Review before running"); + }); + mockCallGateway.mockReset(); + }); + + it.each([ + { + label: "failure", + details: { status: "failed", exitCode: 1 }, + terminal: { + type: "tool.execution.error", + errorCategory: "tool_result_error", + terminalReason: "failed", + }, + }, + { + label: "timeout", + details: { status: "timeout", timedOut: true }, + terminal: { + type: "tool.execution.error", + errorCategory: "tool_result_error", + terminalReason: "timed_out", + }, + }, + { + label: "cancellation", + details: { status: "cancelled" }, + terminal: { + type: "tool.execution.error", + errorCategory: "tool_result_error", + terminalReason: "cancelled", + }, + }, + { + label: "blocked action", + details: { status: "blocked" }, + terminal: { + type: "tool.execution.blocked", + deniedReason: "tool_result_blocked", + reason: "tool_result_blocked", + }, + }, + ])("classifies a resolved $label result as terminal failure", async ({ details, terminal }) => { + const execute = vi.fn().mockResolvedValue({ + content: [{ type: "text", text: "tool failed" }], + details, + }); + const tool = wrapToolWithBeforeToolCallHook({ name: "exec", execute } as any, { + agentId: "main", + sessionKey: "session-key", + runId: "run-1", + loopDetection: { enabled: false }, + }); + + await withToolExecutionEvents(async (emitted, flush) => { + await tool.execute("tool-call-1", { command: "false" }, undefined, undefined); + await flush(); + + expect(emitted.map((event) => event.type)).toEqual(["tool.execution.started", terminal.type]); + expectEventFields(emitted[1], terminal); + }); + }); + it("classifies plugin and MCP tool execution diagnostics with bounded owner labels", async () => { const execute = vi.fn().mockResolvedValue({ content: [{ type: "text", text: "ok" }] }); const rawTool = { name: "mcp_search", execute } as unknown as AnyAgentTool; @@ -818,6 +1061,91 @@ describe("before_tool_call loop detection behavior", () => { }); }); + it("classifies a tool error as cancelled only when the run signal is aborted", async () => { + const abortController = new AbortController(); + const execute = vi.fn().mockImplementation(() => { + abortController.abort(); + throw new Error("tool stopped with run"); + }); + const tool = wrapToolWithBeforeToolCallHook({ name: "read", execute } as any, { + agentId: "main", + sessionKey: "session-key", + loopDetection: { enabled: false }, + }); + + await withToolExecutionEvents(async (emitted, flush) => { + await expect( + tool.execute( + "tool-call-cancelled", + { path: "/tmp/file" }, + abortController.signal, + undefined, + ), + ).rejects.toThrow("tool stopped with run"); + await flush(); + + expectEventFields(emitted[1], { + type: "tool.execution.error", + toolCallId: "tool-call-cancelled", + errorCategory: "aborted", + terminalReason: "cancelled", + }); + }); + }); + + it("classifies a tool error as timed out when the run timeout signal is aborted", async () => { + const abortController = new AbortController(); + const execute = vi.fn().mockImplementation(() => { + abortController.abort(Object.assign(new Error("timed out"), { name: "TimeoutError" })); + throw new Error("tool stopped with timeout"); + }); + const tool = wrapToolWithBeforeToolCallHook({ name: "read", execute } as any, { + agentId: "main", + sessionKey: "session-key", + loopDetection: { enabled: false }, + }); + + await withToolExecutionEvents(async (emitted, flush) => { + await expect( + tool.execute("tool-call-timeout", { path: "/tmp/file" }, abortController.signal, undefined), + ).rejects.toThrow("tool stopped with timeout"); + await flush(); + + expectEventFields(emitted[1], { + type: "tool.execution.error", + toolCallId: "tool-call-timeout", + terminalReason: "timed_out", + }); + }); + }); + + it("classifies a tool-local timeout without an aborted run signal", async () => { + const execute = vi + .fn() + .mockRejectedValue( + Object.assign(new Error("tool deadline elapsed"), { name: "TimeoutError" }), + ); + const tool = wrapToolWithBeforeToolCallHook({ name: "read", execute } as any, { + agentId: "main", + sessionKey: "session-key", + loopDetection: { enabled: false }, + }); + const runSignal = new AbortController().signal; + + await withToolExecutionEvents(async (emitted, flush) => { + await expect( + tool.execute("tool-call-local-timeout", { path: "/tmp/file" }, runSignal, undefined), + ).rejects.toThrow("tool deadline elapsed"); + await flush(); + + expectEventFields(emitted[1], { + type: "tool.execution.error", + toolCallId: "tool-call-local-timeout", + terminalReason: "timed_out", + }); + }); + }); + it("emits blocked diagnostics without error severity for intentional hook vetoes", async () => { hookRunner.hasHooks.mockImplementation((hookName: string) => hookName === "before_tool_call"); hookRunner.runBeforeToolCall.mockResolvedValue({ @@ -1136,12 +1464,35 @@ describe("before_tool_call requireApproval handling", () => { }); expect(result.blocked).toBe(true); + expect(result).toHaveProperty("disposition", "failed"); expect(result).toHaveProperty( "reason", "Tool call blocked because before_tool_call hook failed", ); }); + it("classifies a loop preflight exception as a before-tool failure", async () => { + const ctx = { + sessionKey: "main", + get loopDetection(): never { + throw new Error("loop state unavailable"); + }, + }; + + const result = await runBeforeToolCallHook({ + toolName: "bash", + params: { command: "ls" }, + ctx, + }); + + expect(result).toMatchObject({ + blocked: true, + kind: "failure", + disposition: "failed", + reason: "Tool call blocked because before_tool_call hook failed", + }); + }); + it("passes diagnostic trace context to before_tool_call hooks", async () => { const trace = { traceId: "4bf92f3577b34da6a3ce929d0e0e4736", @@ -1511,6 +1862,7 @@ describe("before_tool_call requireApproval handling", () => { }); expect(result.blocked).toBe(true); + expect(result).toHaveProperty("disposition", "blocked"); expect(result).toHaveProperty("reason", "Denied by user"); }); @@ -1566,6 +1918,7 @@ describe("before_tool_call requireApproval handling", () => { }); expect(result.blocked).toBe(true); + expect(result).toHaveProperty("disposition", "timed_out"); expect(result).toHaveProperty( "reason", "Approval timed out\n\nConfigure Telegram native approval setup.", diff --git a/src/agents/agent-tools.before-tool-call.embedded-mode.test.ts b/src/agents/agent-tools.before-tool-call.embedded-mode.test.ts index 96c355f487ee..0db7f8c30d62 100644 --- a/src/agents/agent-tools.before-tool-call.embedded-mode.test.ts +++ b/src/agents/agent-tools.before-tool-call.embedded-mode.test.ts @@ -124,6 +124,7 @@ describe("runBeforeToolCallHook — embedded mode approvals", () => { expect(result).toEqual({ blocked: true, kind: "failure", + disposition: "failed", deniedReason: "plugin-approval", reason: "Plugin approval required (gateway unavailable)", params: { command: "ls" }, @@ -238,6 +239,7 @@ describe("runBeforeToolCallHook — embedded mode approvals", () => { expect(result).toEqual({ blocked: true, kind: "failure", + disposition: "blocked", deniedReason: "plugin-approval", reason: "Review before running", params: { command: "ls" }, diff --git a/src/agents/agent-tools.before-tool-call.ts b/src/agents/agent-tools.before-tool-call.ts index 55bb2ce00f21..e421e4d04832 100644 --- a/src/agents/agent-tools.before-tool-call.ts +++ b/src/agents/agent-tools.before-tool-call.ts @@ -20,6 +20,7 @@ import { type DiagnosticEventPrivateData, type DiagnosticToolParamsSummary, type DiagnosticToolSource, + type DiagnosticToolTerminalReason, } from "../infra/diagnostic-events.js"; import { cloneDiagnosticContentValue, @@ -76,6 +77,7 @@ import { recordStructuredReplaySafeToolCall, structuredReplaySafeToolCallIds, } from "./agent-tools.before-tool-call.state.js"; +import { resolveAgentRunAbortLifecycleFields } from "./run-termination.js"; export { consumeAdjustedParamsForToolCall, consumePreExecutionBlockedToolCall, @@ -103,6 +105,11 @@ import { } from "./code-mode-control-tools.js"; import type { SandboxFsBridge } from "./sandbox/fs-bridge.js"; import { normalizeToolName } from "./tool-policy.js"; +import { + formatToolExecutionErrorMessage, + resolveToolExecutionErrorKind, + resolveToolResultFailureKind, +} from "./tool-result-error.js"; import { copyToolTerminalPresentation } from "./tool-terminal-presentation.js"; import { getToolTerminalPresentation } from "./tool-terminal-presentation.js"; import type { AnyAgentTool } from "./tools/common.js"; @@ -156,16 +163,20 @@ export type HookContext = { }; }; -type HookBlockedKind = "veto" | "failure"; type HookBlockedReason = "plugin-before-tool-call" | "plugin-approval" | "tool-loop"; +export type BeforeToolCallFailureDisposition = "blocked" | DiagnosticToolTerminalReason; +type HookBlockedOutcome = { + blocked: true; + deniedReason?: HookBlockedReason; + reason: string; + params?: unknown; +}; type HookOutcome = - | { - blocked: true; - kind?: HookBlockedKind; - deniedReason?: HookBlockedReason; - reason: string; - params?: unknown; - } + | (HookBlockedOutcome & { kind: "veto" }) + | (HookBlockedOutcome & { + kind: "failure"; + disposition: BeforeToolCallFailureDisposition; + }) | { blocked: false; params: unknown; @@ -356,6 +367,44 @@ export class BeforeToolCallBlockedError extends Error { } } +class BeforeToolCallFailureError extends Error { + constructor( + message: string, + readonly disposition: BeforeToolCallFailureDisposition, + cause?: unknown, + ) { + super(message, cause === undefined ? undefined : { cause }); + this.name = "BeforeToolCallFailureError"; + } +} + +function tagBeforeToolCallFailure( + error: unknown, + signal?: AbortSignal, +): BeforeToolCallFailureError { + try { + if (error instanceof BeforeToolCallFailureError) { + return error; + } + } catch { + // Continue through the guarded formatter and classifier for hostile values. + } + const message = formatToolExecutionErrorMessage(error, "before_tool_call failed"); + const disposition = resolveToolErrorDiagnostic(error, signal).terminalReason; + return new BeforeToolCallFailureError(message, disposition, error); +} + +/** Return the closed terminal disposition carried by a before-tool failure. */ +export function getBeforeToolCallFailureDisposition( + error: unknown, +): BeforeToolCallFailureDisposition | undefined { + try { + return error instanceof BeforeToolCallFailureError ? error.disposition : undefined; + } catch { + return undefined; + } +} + /** Remember hook-adjusted params for later adapter-side execution. */ export function recordAdjustedParamsForToolCall( toolCallId: string | undefined, @@ -448,6 +497,75 @@ function unwrapErrorCause(err: unknown): unknown { return err; } +function resolveToolErrorDiagnostic( + err: unknown, + signal?: AbortSignal, + errorCategory?: string, +): { + errorCategory: string; + errorCode?: string; + terminalReason: DiagnosticToolTerminalReason; +} { + const cause = unwrapErrorCause(err); + const errorCode = diagnosticHttpStatusCode(cause); + const abortFields = resolveAgentRunAbortLifecycleFields(signal); + const terminalReason = !abortFields.aborted + ? resolveToolExecutionErrorKind(cause) + : abortFields.stopReason === "timeout" + ? "timed_out" + : "cancelled"; + return { + errorCategory: + terminalReason === "cancelled" + ? "aborted" + : (errorCategory ?? diagnosticErrorCategory(cause)), + terminalReason, + ...(errorCode ? { errorCode } : {}), + }; +} + +type ResolvedToolTerminalDiagnostic = + | { + type: "tool.execution.blocked"; + deniedReason: "tool_result_blocked"; + reason: "tool_result_blocked"; + } + | { + type: "tool.execution.completed"; + durationMs: number; + } + | { + type: "tool.execution.error"; + durationMs: number; + errorCategory: "tool_result_error"; + terminalReason: DiagnosticToolTerminalReason; + }; + +function resolveToolResultTerminalDiagnostic( + result: unknown, + durationMs: number, +): ResolvedToolTerminalDiagnostic { + // Tool execution may resolve with a structured failure. Classify that here + // so every diagnostic consumer sees one canonical terminal outcome. + const failureKind = resolveToolResultFailureKind(result); + if (!failureKind) { + return { type: "tool.execution.completed", durationMs }; + } + if (failureKind === "blocked") { + return { + type: "tool.execution.blocked", + deniedReason: "tool_result_blocked", + reason: "tool_result_blocked", + }; + } + return { + type: "tool.execution.error", + durationMs, + errorCategory: "tool_result_error", + terminalReason: failureKind, + }; +} + type ToolDiagnosticIdentity = { toolSource: DiagnosticToolSource; toolOwner?: string; @@ -747,6 +865,7 @@ async function requestPluginToolApproval(params: { return { blocked: true, kind: "failure", + disposition: "blocked", deniedReason: "plugin-approval", reason: "Denied by user", params: params.baseParams, @@ -759,13 +878,24 @@ async function requestPluginToolApproval(params: { approvalResolution: resolution, }; } - return { - blocked: true, - kind: approval.timeoutReason ? "veto" : "failure", - deniedReason: "plugin-approval", - reason: approval.timeoutReason ?? "Approval timed out", - params: params.baseParams, - }; + // Veto carries the plugin-supplied reason; plain timeouts record a + // timed_out failure disposition for the audit ledger. + return approval.timeoutReason + ? { + blocked: true, + kind: "veto", + deniedReason: "plugin-approval", + reason: approval.timeoutReason, + params: params.baseParams, + } + : { + blocked: true, + kind: "failure", + disposition: "timed_out", + deniedReason: "plugin-approval", + reason: "Approval timed out", + params: params.baseParams, + }; } gatewayApprovalPhase = "request"; @@ -808,6 +938,7 @@ async function requestPluginToolApproval(params: { return { blocked: true, kind: "failure", + disposition: "failed", deniedReason: "plugin-approval", reason: approval.description || "Plugin approval request failed", params: params.baseParams, @@ -822,6 +953,7 @@ async function requestPluginToolApproval(params: { return { blocked: true, kind: "failure", + disposition: "failed", deniedReason: "plugin-approval", reason: buildPluginApprovalFailureReason({ fallbackReason: "Plugin approval unavailable (no approval route)", @@ -888,6 +1020,7 @@ async function requestPluginToolApproval(params: { return { blocked: true, kind: "failure", + disposition: "blocked", deniedReason: "plugin-approval", reason: "Denied by user", params: params.baseParams, @@ -912,6 +1045,7 @@ async function requestPluginToolApproval(params: { return { blocked: true, kind: approval.timeoutReason ? "veto" : "failure", + disposition: "timed_out", deniedReason: "plugin-approval", reason: timeoutReason, params: params.baseParams, @@ -929,6 +1063,7 @@ async function requestPluginToolApproval(params: { return { blocked: true, kind: "failure", + disposition: resolveToolErrorDiagnostic(err, signal).terminalReason, deniedReason: "plugin-approval", reason: "Approval cancelled (run aborted)", params: params.baseParams, @@ -947,6 +1082,7 @@ async function requestPluginToolApproval(params: { return { blocked: true, kind: "failure", + disposition: resolveToolErrorDiagnostic(err, signal).terminalReason, deniedReason: "plugin-approval", reason, params: params.baseParams, @@ -1010,6 +1146,7 @@ async function resolveBeforeToolCallApprovalOutcome(params: { return { blocked: true, kind: "failure", + disposition: "blocked", deniedReason: "plugin-approval", reason: approval.description || approval.title || "Plugin approval required", params: params.baseParams, @@ -1193,77 +1330,77 @@ export async function runBeforeToolCallHook(args: { const toolName = normalizeToolName(args.toolName || "tool"); const params = args.params; - if (args.ctx?.sessionKey) { - const { getDiagnosticSessionState, logToolLoopAction, detectToolCallLoop, recordToolCall } = - await loadBeforeToolCallRuntime(); - const sessionState = getDiagnosticSessionState({ - sessionKey: args.ctx.sessionKey, - sessionId: args.ctx.sessionId, - }); + try { + if (args.ctx?.sessionKey) { + const { getDiagnosticSessionState, logToolLoopAction, detectToolCallLoop, recordToolCall } = + await loadBeforeToolCallRuntime(); + const sessionState = getDiagnosticSessionState({ + sessionKey: args.ctx.sessionKey, + sessionId: args.ctx.sessionId, + }); - const loopScope = args.ctx.runId ? { runId: args.ctx.runId } : undefined; - const loopResult = detectToolCallLoop( - sessionState, - toolName, - params, - args.ctx.loopDetection, - loopScope, - ); - - if (loopResult.stuck) { - if (loopResult.level === "critical") { - log.error(`Blocking ${toolName} due to critical loop: ${loopResult.message}`); - logToolLoopAction({ - sessionKey: args.ctx.sessionKey, - sessionId: args.ctx.sessionId, - toolName, - level: "critical", - action: "block", - detector: loopResult.detector, - count: loopResult.count, - message: loopResult.message, - pairedToolName: loopResult.pairedToolName, - }); - return { - blocked: true, - kind: "veto", - deniedReason: "tool-loop", - reason: loopResult.message, - params, - }; - } - const baseWarningKey = loopResult.warningKey ?? `${loopResult.detector}:${toolName}`; - const warningKey = args.ctx.runId ? `${args.ctx.runId}:${baseWarningKey}` : baseWarningKey; - if (shouldEmitLoopWarning(sessionState, warningKey, loopResult.count)) { - log.warn(`Loop warning for ${toolName}: ${loopResult.message}`); - logToolLoopAction({ - sessionKey: args.ctx.sessionKey, - sessionId: args.ctx.sessionId, - toolName, - level: "warning", - action: "warn", - detector: loopResult.detector, - count: loopResult.count, - message: loopResult.message, - pairedToolName: loopResult.pairedToolName, - }); - } - } - - if (args.ctx.loopDetection?.enabled !== false) { - recordToolCall( + const loopScope = args.ctx.runId ? { runId: args.ctx.runId } : undefined; + const loopResult = detectToolCallLoop( sessionState, toolName, params, - args.toolCallId, args.ctx.loopDetection, loopScope, ); - } - } - const hookRunner = getGlobalHookRunner(); - try { + if (loopResult.stuck) { + if (loopResult.level === "critical") { + log.error(`Blocking ${toolName} due to critical loop: ${loopResult.message}`); + logToolLoopAction({ + sessionKey: args.ctx.sessionKey, + sessionId: args.ctx.sessionId, + toolName, + level: "critical", + action: "block", + detector: loopResult.detector, + count: loopResult.count, + message: loopResult.message, + pairedToolName: loopResult.pairedToolName, + }); + return { + blocked: true, + kind: "veto", + deniedReason: "tool-loop", + reason: loopResult.message, + params, + }; + } + const baseWarningKey = loopResult.warningKey ?? `${loopResult.detector}:${toolName}`; + const warningKey = args.ctx.runId ? `${args.ctx.runId}:${baseWarningKey}` : baseWarningKey; + if (shouldEmitLoopWarning(sessionState, warningKey, loopResult.count)) { + log.warn(`Loop warning for ${toolName}: ${loopResult.message}`); + logToolLoopAction({ + sessionKey: args.ctx.sessionKey, + sessionId: args.ctx.sessionId, + toolName, + level: "warning", + action: "warn", + detector: loopResult.detector, + count: loopResult.count, + message: loopResult.message, + pairedToolName: loopResult.pairedToolName, + }); + } + } + + if (args.ctx.loopDetection?.enabled !== false) { + recordToolCall( + sessionState, + toolName, + params, + args.toolCallId, + args.ctx.loopDetection, + loopScope, + ); + } + } + + const hookRunner = getGlobalHookRunner(); const hasBeforeToolCallHooks = hookRunner?.hasHooks("before_tool_call") === true; const policyRegistry = getGlobalHookRunnerRegistry() ?? undefined; const shouldRunTrustedPolicies = hasTrustedToolPolicies(policyRegistry); @@ -1491,6 +1628,7 @@ export async function runBeforeToolCallHook(args: { blocked: true, kind: "failure", deniedReason: "plugin-before-tool-call", + disposition: resolveToolErrorDiagnostic(cause, args.signal).terminalReason, reason: BEFORE_TOOL_CALL_HOOK_FAILURE_REASON, params, }; @@ -1522,43 +1660,110 @@ export function wrapToolWithBeforeToolCallHook( // Allocate before any async preparation so parallel completions retain // the assistant message's tool-call order. const toolCallOrdinal = ctx?.allocateToolOutcomeOrdinal?.(toolCallId); - const prepare = (tool as BeforeToolCallPreparingTool).prepareBeforeToolCallParams; - const preparedParams = prepare - ? await prepare(params, { - ...(toolCallId ? { toolCallId } : {}), - ...(ctx ? { hookContext: ctx } : {}), - ...(signal ? { signal } : {}), - }) - : params; - const hookParams = normalizeCodeModeExecBeforeHookParams({ tool, params: preparedParams }); - const hookMetadata = getCodeModeExecBeforeHookMetadata({ tool, params: preparedParams }); - const outcome = await runBeforeToolCallHook({ - toolName, - params: hookParams, - ...hookMetadata, - toolCallId, - ctx, - signal, - approvalMode: hookOptions.approvalMode, - }); - if (outcome.blocked) { - if (outcome.kind !== "veto") { - throw new Error(outcome.reason); - } - const normalizedToolName = normalizeToolName(toolName || "tool"); - const trace = ctx?.trace + const preExecutionStartedAt = Date.now(); + const normalizedToolName = normalizeToolName(toolName || "tool"); + const trace = + hookOptions.emitDiagnostics && ctx?.trace ? freezeDiagnosticTraceContext(createChildDiagnosticTraceContext(ctx.trace)) : undefined; - const eventBase = { - ...(ctx?.runId && { runId: ctx.runId }), - ...(ctx?.sessionKey && { sessionKey: ctx.sessionKey }), - ...(ctx?.sessionId && { sessionId: ctx.sessionId }), - ...(trace && { trace }), - toolName: normalizedToolName, - ...diagnosticIdentity, - ...(toolCallId && { toolCallId }), - paramsSummary: summarizeToolParams(outcome.params ?? hookParams), - }; + const buildEventBase = (toolParams: unknown) => ({ + ...(ctx?.runId && { runId: ctx.runId }), + ...(ctx?.sessionKey && { sessionKey: ctx.sessionKey }), + ...(ctx?.sessionId && { sessionId: ctx.sessionId }), + ...(ctx?.agentId && { agentId: ctx.agentId }), + ...(trace && { trace }), + toolName: normalizedToolName, + ...diagnosticIdentity, + ...(toolCallId && { toolCallId }), + paramsSummary: summarizeToolParams(toolParams), + }); + const recordPreExecutionError = ( + error: unknown, + toolParams: unknown, + errorCategory?: string, + ) => { + recordPreExecutionBlockedToolCall(toolCallId, ctx?.runId); + if (!hookOptions.emitDiagnostics) { + return; + } + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + ...buildEventBase(toolParams), + durationMs: Date.now() - preExecutionStartedAt, + ...resolveToolErrorDiagnostic(error, signal, errorCategory), + }); + }; + const recordPreExecutionDisposition = ( + toolParams: unknown, + disposition: BeforeToolCallFailureDisposition, + errorCategory: string, + deniedReason?: HookBlockedReason, + ) => { + recordPreExecutionBlockedToolCall(toolCallId, ctx?.runId); + if (!hookOptions.emitDiagnostics) { + return; + } + const eventBase = buildEventBase(toolParams); + if (disposition === "blocked") { + const reason = deniedReason ?? "plugin-before-tool-call"; + emitTrustedDiagnosticEvent({ + type: "tool.execution.blocked", + ...eventBase, + deniedReason: reason, + reason, + }); + return; + } + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + ...eventBase, + durationMs: Date.now() - preExecutionStartedAt, + errorCategory: disposition === "cancelled" ? "aborted" : errorCategory, + terminalReason: disposition, + }); + }; + const prepare = (tool as BeforeToolCallPreparingTool).prepareBeforeToolCallParams; + let preparedParams: unknown; + try { + preparedParams = prepare + ? await prepare(params, { + ...(toolCallId ? { toolCallId } : {}), + ...(ctx ? { hookContext: ctx } : {}), + ...(signal ? { signal } : {}), + }) + : params; + } catch (error) { + recordPreExecutionError(error, params, "tool_preparation"); + throw tagBeforeToolCallFailure(error, signal); + } + const hookParams = normalizeCodeModeExecBeforeHookParams({ tool, params: preparedParams }); + const hookMetadata = getCodeModeExecBeforeHookMetadata({ tool, params: preparedParams }); + let outcome: HookOutcome; + try { + outcome = await runBeforeToolCallHook({ + toolName, + params: hookParams, + ...hookMetadata, + toolCallId, + ctx, + signal, + approvalMode: hookOptions.approvalMode, + }); + } catch (error) { + recordPreExecutionError(error, hookParams, "before_tool_call"); + throw tagBeforeToolCallFailure(error, signal); + } + if (outcome.blocked) { + if (outcome.kind !== "veto") { + recordPreExecutionDisposition( + outcome.params ?? hookParams, + outcome.disposition, + outcome.deniedReason === "plugin-approval" ? "plugin_approval" : "before_tool_call", + outcome.deniedReason, + ); + throw new BeforeToolCallFailureError(outcome.reason, outcome.disposition); + } + const eventBase = buildEventBase(outcome.params ?? hookParams); if (hookOptions.emitDiagnostics) { emitTrustedDiagnosticEvent({ type: "tool.execution.blocked", @@ -1591,32 +1796,25 @@ export function wrapToolWithBeforeToolCallHook( }); return blockedResult; } - let executeParams = reconcileCodeModeExecBeforeHookParams({ - tool, - originalParams: preparedParams, - hookParams, - adjustedParams: outcome.params, - }); - executeParams = - (tool as BeforeToolCallPreparingTool).finalizeBeforeToolCallParams?.( - executeParams, - preparedParams, - ) ?? executeParams; + let executeParams: unknown; + try { + executeParams = reconcileCodeModeExecBeforeHookParams({ + tool, + originalParams: preparedParams, + hookParams, + adjustedParams: outcome.params, + }); + executeParams = + (tool as BeforeToolCallPreparingTool).finalizeBeforeToolCallParams?.( + executeParams, + preparedParams, + ) ?? executeParams; + } catch (error) { + recordPreExecutionError(error, outcome.params ?? hookParams, "tool_preparation"); + throw tagBeforeToolCallFailure(error, signal); + } recordAdjustedParamsForToolCall(toolCallId, executeParams, ctx?.runId); - const normalizedToolName = normalizeToolName(toolName || "tool"); - const trace = ctx?.trace - ? freezeDiagnosticTraceContext(createChildDiagnosticTraceContext(ctx.trace)) - : undefined; - const eventBase = { - ...(ctx?.runId && { runId: ctx.runId }), - ...(ctx?.sessionKey && { sessionKey: ctx.sessionKey }), - ...(ctx?.sessionId && { sessionId: ctx.sessionId }), - ...(trace && { trace }), - toolName: normalizedToolName, - ...diagnosticIdentity, - ...(toolCallId && { toolCallId }), - paramsSummary: summarizeToolParams(executeParams), - }; + const eventBase = buildEventBase(executeParams); if (hookOptions.emitDiagnostics) { emitTrustedDiagnosticEvent({ type: "tool.execution.started", @@ -1662,11 +1860,11 @@ export function wrapToolWithBeforeToolCallHook( toolCallId, }); } + const terminalEvent = resolveToolResultTerminalDiagnostic(result, durationMs); emitTrustedDiagnosticEventWithPrivateData( { - type: "tool.execution.completed", ...eventBase, - durationMs, + ...terminalEvent, }, buildToolContentPrivateData(toolContentPolicy, { input: executeParams, @@ -1677,16 +1875,13 @@ export function wrapToolWithBeforeToolCallHook( } return result; } catch (err) { - const cause = unwrapErrorCause(err); - const errorCode = diagnosticHttpStatusCode(cause); if (hookOptions.emitDiagnostics) { emitTrustedDiagnosticEventWithPrivateData( { type: "tool.execution.error", ...eventBase, durationMs: Date.now() - startedAt, - errorCategory: diagnosticErrorCategory(cause), - ...(errorCode ? { errorCode } : {}), + ...resolveToolErrorDiagnostic(err, signal), }, buildToolContentPrivateData(toolContentPolicy, { input: executeParams, diff --git a/src/agents/cli-output.test.ts b/src/agents/cli-output.test.ts index 33454b411fa2..6e1abe6102d7 100644 --- a/src/agents/cli-output.test.ts +++ b/src/agents/cli-output.test.ts @@ -1740,6 +1740,7 @@ describe("createCliJsonlStreamingParser", () => { { toolCallId: "tool-1", name: "mcp_openclaw_create_goal", + kind: "tool_use", args: { objective: "Update files" }, }, ]); @@ -1943,7 +1944,9 @@ describe("createCliJsonlStreamingParser", () => { ); parser.finish(); - expect(starts).toEqual([{ toolCallId: "toolu_1", name: "Bash", args: { command: "ls -la" } }]); + expect(starts).toEqual([ + { toolCallId: "toolu_1", name: "Bash", kind: "tool_use", args: { command: "ls -la" } }, + ]); expect(results).toEqual([ { toolCallId: "toolu_1", name: "Bash", isError: false, result: "total 0\n" }, ]); @@ -1998,7 +2001,12 @@ describe("createCliJsonlStreamingParser", () => { parser.finish(); expect(starts).toEqual([ - { toolCallId: "toolu_chunked", name: "Bash", args: { command: "echo hi" } }, + { + toolCallId: "toolu_chunked", + name: "Bash", + kind: "tool_use", + args: { command: "echo hi" }, + }, ]); }); @@ -2042,7 +2050,7 @@ describe("createCliJsonlStreamingParser", () => { ); parser.finish(); - expect(starts).toEqual([{ toolCallId: "toolu_bad", name: "Bash", args: {} }]); + expect(starts).toEqual([{ toolCallId: "toolu_bad", name: "Bash", kind: "tool_use", args: {} }]); }); it.each(["server_tool_use", "mcp_tool_use"])("recognizes %s blocks", (type) => { @@ -2086,7 +2094,12 @@ describe("createCliJsonlStreamingParser", () => { parser.finish(); expect(starts).toEqual([ - { toolCallId: "toolu_hosted", name: "web_search", args: { query: "openclaw" } }, + { + toolCallId: "toolu_hosted", + name: "web_search", + kind: type, + args: { query: "openclaw" }, + }, ]); }); @@ -2153,7 +2166,12 @@ describe("createCliJsonlStreamingParser", () => { parser.finish(); expect(starts).toEqual([ - { toolCallId: fixture.toolCallId, name: fixture.name, args: fixture.input }, + { + toolCallId: fixture.toolCallId, + name: fixture.name, + kind: fixture.useType, + args: fixture.input, + }, ]); expect(results).toEqual([ { diff --git a/src/agents/cli-output.ts b/src/agents/cli-output.ts index 640696188c1f..46571fe9c133 100644 --- a/src/agents/cli-output.ts +++ b/src/agents/cli-output.ts @@ -91,6 +91,8 @@ export type CliThinkingProgress = { export type CliToolUseStartDelta = { toolCallId: string; name: string; + // Preserve the producer kind: a server-native start without its result is not a failed local call. + kind: "tool_use" | "server_tool_use" | "mcp_tool_use"; args: Record; }; @@ -588,6 +590,7 @@ function parseClaudeCliStreamingDelta(params: { type PendingToolUse = { toolCallId: string; name: string; + kind: CliToolUseStartDelta["kind"]; inputJsonParts: string[]; }; @@ -611,6 +614,7 @@ function emitToolStartOnce( tracker: ToolUseTracker, toolCallId: string, name: string, + kind: CliToolUseStartDelta["kind"], args: Record, onToolUseStart?: (delta: CliToolUseStartDelta) => void, ): void { @@ -620,7 +624,7 @@ function emitToolStartOnce( } tracker.startedIds.add(toolCallId); tracker.nameById.set(toolCallId, name); - onToolUseStart?.({ toolCallId, name, args }); + onToolUseStart?.({ toolCallId, name, kind, args }); } function emitToolResultOnce( @@ -643,7 +647,7 @@ function emitToolResultOnce( }); } -function isClaudeToolUseBlockType(type: unknown): boolean { +function isClaudeToolUseBlockType(type: unknown): type is CliToolUseStartDelta["kind"] { return type === "tool_use" || type === "server_tool_use" || type === "mcp_tool_use"; } @@ -692,7 +696,12 @@ function dispatchClaudeCliStreamingToolEvent(params: { const toolCallId = typeof block.id === "string" ? block.id.trim() : ""; const name = typeof block.name === "string" ? block.name.trim() : ""; if (toolCallId && name) { - tracker.pendingByIndex.set(event.index, { toolCallId, name, inputJsonParts: [] }); + tracker.pendingByIndex.set(event.index, { + toolCallId, + name, + kind: block.type, + inputJsonParts: [], + }); } } else if (isClaudeAssistantToolResultBlockType(block.type)) { const toolCallId = typeof block.tool_use_id === "string" ? block.tool_use_id.trim() : ""; @@ -726,6 +735,7 @@ function dispatchClaudeCliStreamingToolEvent(params: { tracker, pending.toolCallId, pending.name, + pending.kind, parseToolInputJson(pending.inputJsonParts), params.onToolUseStart, ); @@ -749,7 +759,7 @@ function dispatchClaudeCliStreamingToolEvent(params: { continue; } const args: Record = isRecord(block.input) ? block.input : {}; - emitToolStartOnce(tracker, toolCallId, name, args, params.onToolUseStart); + emitToolStartOnce(tracker, toolCallId, name, block.type, args, params.onToolUseStart); } else if (isClaudeAssistantToolResultBlockType(block.type)) { const toolCallId = typeof block.tool_use_id === "string" ? block.tool_use_id.trim() : ""; if (!toolCallId) { @@ -1014,7 +1024,7 @@ function dispatchGeminiCliStreamingToolEvent(params: { return; } const args = isRecord(params.parsed.parameters) ? params.parsed.parameters : {}; - emitToolStartOnce(params.tracker, toolCallId, name, args, params.onToolUseStart); + emitToolStartOnce(params.tracker, toolCallId, name, "tool_use", args, params.onToolUseStart); return; } if (params.parsed.type === "tool_result") { diff --git a/src/agents/cli-runner.reliability.test.ts b/src/agents/cli-runner.reliability.test.ts index 2b00703b33ce..d63a8ec3b412 100644 --- a/src/agents/cli-runner.reliability.test.ts +++ b/src/agents/cli-runner.reliability.test.ts @@ -698,7 +698,7 @@ describe("runCliAgent reliability", () => { mediaUrl: "https://example.com/done.png", }, result: { status: "sent" }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(captureHandle); }, 10); @@ -763,8 +763,8 @@ describe("runCliAgent reliability", () => { target: "chat123", message: "sent before failure", }, + outcome: "completed", result: { status: "sent" }, - isError: false, }); markMcpLoopbackToolCallFinished(captureHandle); return createManagedRun({ @@ -829,7 +829,7 @@ describe("runCliAgent reliability", () => { message: "sent before overflow", }, result: { status: "sent" }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(captureHandle); return createManagedRun({ @@ -892,7 +892,7 @@ describe("runCliAgent reliability", () => { sourceReply: { text: "sent before failure" }, }, }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(captureHandle); return createManagedRun({ @@ -1003,7 +1003,7 @@ describe("runCliAgent reliability", () => { sourceReply: { text: "sent through source reply" }, }, }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(captureHandle); return createManagedRun({ @@ -1077,7 +1077,7 @@ describe("runCliAgent reliability", () => { sourceReply: { text: "visible source reply" }, }, }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(captureHandle); return createManagedRun({ @@ -1153,7 +1153,7 @@ describe("runCliAgent reliability", () => { message: "sent without a terminal reply", }, result: { status: "sent" }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(captureHandle); input.onStdout?.( diff --git a/src/agents/cli-runner.spawn.test.ts b/src/agents/cli-runner.spawn.test.ts index ff63e656a6e9..36256b0e0c34 100644 --- a/src/agents/cli-runner.spawn.test.ts +++ b/src/agents/cli-runner.spawn.test.ts @@ -16,6 +16,7 @@ import { import { onAgentEvent, resetAgentEventsForTest } from "../infra/agent-events.js"; import { onInternalDiagnosticEvent, + onTrustedToolExecutionEvent, waitForDiagnosticEventsDrained, } from "../infra/diagnostic-events.js"; import { @@ -1526,7 +1527,7 @@ describe("runCliAgent spawn path", () => { toolName: "message", args: { action: "send", target: "chat123", message: "done" }, result: { status: "sent" }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(captureHandle); input.onStdout?.("done"); @@ -2169,10 +2170,16 @@ ${JSON.stringify({ role: "assistant", content: [ { - type: "tool_use", + type: "mcp_tool_use", id: "tool-live-1", - name: "Bash", - input: { command: "sleep 45" }, + name: "mcp__team__lookup", + input: { query: "status" }, + }, + { + type: "server_tool_use", + id: "tool-live-2", + name: "web_search", + input: { query: "release status" }, }, ], }, @@ -2231,7 +2238,7 @@ ${JSON.stringify({ getDiagnosticSessionActivitySnapshot({ sessionKey: "agent:main:diagnostics", }).activeToolName, - ).toBe("Bash"), + ).toBe("mcp__team__lookup"), ); expect( getDiagnosticSessionActivitySnapshot({ sessionKey: "agent:main:diagnostics" }) @@ -2260,6 +2267,12 @@ ${JSON.stringify({ { type: "tool_result", tool_use_id: "tool-live-1", + content: "lookup failed", + is_error: true, + }, + { + type: "tool_result", + tool_use_id: "tool-live-2", content: "done", }, ], @@ -2291,14 +2304,494 @@ ${JSON.stringify({ getDiagnosticSessionActivitySnapshot({ sessionKey: "agent:main:diagnostics" }) .lastProgressReason, ).toBe("cli_live:result"); - expect(diagnosticEvents).toContain("tool.execution.started"); + expect(diagnosticEvents.filter((event) => event === "tool.execution.started")).toHaveLength( + 2, + ); expect(diagnosticEvents).toContain("tool.execution.completed"); + expect(diagnosticEvents).toContain("tool.execution.error"); + } finally { + stopDiagnostics(); + } + }); + + it("preserves loopback policy blocks for Claude live tools", async () => { + const diagnosticEvents: Array> = []; + const stopDiagnostics = onInternalDiagnosticEvent((event) => { + if ( + event.type.startsWith("tool.execution.") && + "toolCallId" in event && + event.toolCallId === "tool-live-blocked" + ) { + diagnosticEvents.push(event as unknown as Record); + } + }); + let stdoutListener: ((chunk: string) => void) | undefined; + let captureKey = ""; + const stdin = { + write: vi.fn((_data: string, cb?: (err?: Error | null) => void) => { + const captureHandle = markMcpLoopbackToolCallStarted({ + captureKey, + toolName: "message", + args: { action: "react" }, + }); + if (!captureHandle) { + throw new Error("Expected live tool capture"); + } + recordMcpLoopbackToolCallResult({ + captureHandle, + toolName: "message", + args: { action: "react" }, + outcome: "blocked", + deniedReason: "plugin-approval", + }); + markMcpLoopbackToolCallFinished(captureHandle); + stdoutListener?.( + [ + JSON.stringify({ type: "system", subtype: "init", session_id: "live-blocked" }), + JSON.stringify({ + type: "assistant", + session_id: "live-blocked", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: "tool-live-blocked", + name: "mcp__openclaw__message", + input: { action: "react" }, + }, + ], + }, + }), + JSON.stringify({ + type: "user", + session_id: "live-blocked", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-live-blocked", + content: "blocked", + is_error: true, + }, + ], + }, + }), + JSON.stringify({ type: "result", session_id: "live-blocked", result: "ok" }), + ].join("\n") + "\n", + ); + cb?.(); + }), + end: vi.fn(), + }; + supervisorSpawnMock.mockImplementation(async (...args: unknown[]) => { + const input = (args[0] ?? {}) as { + env?: Record; + onStdout?: (chunk: string) => void; + }; + stdoutListener = input.onStdout; + captureKey = input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? ""; + return { + runId: "live-run-blocked", + pid: 3061, + startedAtMs: Date.now(), + stdin, + wait: vi.fn(() => new Promise(() => {})), + cancel: vi.fn(), + }; + }); + const context = buildPreparedCliRunContext({ + provider: "claude-cli", + model: "sonnet", + runId: "run-live-blocked", + sessionId: "session-live-blocked", + sessionKey: "agent:main:blocked", + prompt: "hello", + backend: { liveSession: "claude-stdio" }, + }); + context.mcpDeliveryCapture = true; + + try { + await expect(executePreparedCliRun(context)).resolves.toMatchObject({ text: "ok" }); + await waitForDiagnosticEventsDrained(); + } finally { + stopDiagnostics(); + } + + expect(diagnosticEvents).toMatchObject([ + { type: "tool.execution.started", toolCallId: "tool-live-blocked" }, + { + type: "tool.execution.blocked", + toolCallId: "tool-live-blocked", + deniedReason: "plugin-approval", + }, + ]); + }); + + it("keeps identical parallel Claude live tool outcomes explicitly unknown", async () => { + const diagnosticEvents: Array> = []; + const stopDiagnostics = onInternalDiagnosticEvent((event) => { + if ( + event.type.startsWith("tool.execution.") && + "toolCallId" in event && + typeof event.toolCallId === "string" && + event.toolCallId.startsWith("tool-live-identical-") + ) { + diagnosticEvents.push(event as unknown as Record); + } + }); + let stdoutListener: ((chunk: string) => void) | undefined; + let captureKey = ""; + const toolArgs = { action: "react", emoji: "same" }; + const stdin = { + write: vi.fn((_data: string, cb?: (err?: Error | null) => void) => { + stdoutListener?.( + [ + JSON.stringify({ type: "system", subtype: "init", session_id: "live-identical" }), + JSON.stringify({ + type: "assistant", + session_id: "live-identical", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: "tool-live-identical-a", + name: "mcp__openclaw__message", + input: toolArgs, + }, + { + type: "mcp_tool_use", + id: "tool-live-identical-b", + name: "mcp__openclaw__message", + input: toolArgs, + }, + ], + }, + }), + ].join("\n") + "\n", + ); + const captureHandle = markMcpLoopbackToolCallStarted({ + captureKey, + toolName: "message", + args: toolArgs, + }); + if (!captureHandle) { + throw new Error("Expected live tool capture"); + } + recordMcpLoopbackToolCallResult({ + captureHandle, + toolName: "message", + args: toolArgs, + outcome: "failed", + }); + markMcpLoopbackToolCallFinished(captureHandle); + stdoutListener?.( + [ + JSON.stringify({ + type: "user", + session_id: "live-identical", + message: { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "tool-live-identical-a", content: "ok" }, + { type: "tool_result", tool_use_id: "tool-live-identical-b", content: "ok" }, + ], + }, + }), + JSON.stringify({ type: "result", session_id: "live-identical", result: "ok" }), + ].join("\n") + "\n", + ); + cb?.(); + }), + end: vi.fn(), + }; + supervisorSpawnMock.mockImplementation(async (...args: unknown[]) => { + const input = (args[0] ?? {}) as { + env?: Record; + onStdout?: (chunk: string) => void; + }; + stdoutListener = input.onStdout; + captureKey = input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? ""; + return { + runId: "live-run-identical", + pid: 3062, + startedAtMs: Date.now(), + stdin, + wait: vi.fn(() => new Promise(() => {})), + cancel: vi.fn(), + }; + }); + const context = buildPreparedCliRunContext({ + provider: "claude-cli", + model: "sonnet", + runId: "run-live-identical", + sessionId: "session-live-identical", + sessionKey: "agent:main:live-identical", + prompt: "hello", + backend: { liveSession: "claude-stdio" }, + }); + context.mcpDeliveryCapture = true; + + try { + await expect(executePreparedCliRun(context)).resolves.toMatchObject({ text: "ok" }); + await waitForDiagnosticEventsDrained(); + } finally { + stopDiagnostics(); + } + + expect(diagnosticEvents).toMatchObject([ + { type: "tool.execution.started", toolCallId: "tool-live-identical-a" }, + { type: "tool.execution.started", toolCallId: "tool-live-identical-b" }, + { + type: "tool.execution.error", + toolCallId: "tool-live-identical-a", + errorCode: "tool_outcome_unknown", + }, + { + type: "tool.execution.error", + toolCallId: "tool-live-identical-b", + errorCode: "tool_outcome_unknown", + }, + ]); + }); + + it.each([ + [ + "client timeout", + "tool_use", + "Bash", + Object.assign(new Error("gateway timeout"), { name: "TimeoutError" }), + "TimeoutError", + { terminalReason: "timed_out" }, + ], + [ + "client cancellation", + "tool_use", + "Bash", + new Error("operator cancelled"), + "AbortError", + { terminalReason: "cancelled" }, + ], + [ + "server-native timeout", + "server_tool_use", + "web_search", + Object.assign(new Error("gateway timeout"), { name: "TimeoutError" }), + "TimeoutError", + { errorCode: "tool_outcome_unknown" }, + ], + [ + "server-native cancellation", + "server_tool_use", + "web_search", + new Error("operator cancelled"), + "AbortError", + { errorCode: "tool_outcome_unknown" }, + ], + ] as const)( + "classifies active Claude live tools on %s", + async (_, toolType, toolName, abortReason, expectedErrorName, expectedOutcome) => { + const abortController = new AbortController(); + const diagnosticEvents: Array> = []; + const stopDiagnostics = onInternalDiagnosticEvent((event) => { + if (event.type === "tool.execution.error") { + diagnosticEvents.push(event as unknown as Record); + } + }); + let stdoutListener: ((chunk: string) => void) | undefined; + const stdin = { + write: vi.fn((_data: string, cb?: (err?: Error | null) => void) => { + stdoutListener?.( + [ + JSON.stringify({ type: "system", subtype: "init", session_id: "live-timeout" }), + JSON.stringify({ + type: "assistant", + session_id: "live-timeout", + message: { + role: "assistant", + content: [ + { + type: toolType, + id: "tool-live-timeout", + name: toolName, + input: { query: "status" }, + }, + ], + }, + }), + ].join("\n") + "\n", + ); + cb?.(); + }), + end: vi.fn(), + }; + supervisorSpawnMock.mockImplementation(async (...args: unknown[]) => { + const input = (args[0] ?? {}) as { onStdout?: (chunk: string) => void }; + stdoutListener = input.onStdout; + return { + runId: "live-run-timeout", + pid: 3061, + startedAtMs: Date.now(), + stdin, + wait: vi.fn(() => new Promise(() => {})), + cancel: vi.fn(), + }; + }); + + try { + const context = buildPreparedCliRunContext({ + provider: "claude-cli", + model: "sonnet", + runId: "run-live-timeout", + sessionId: "session-live-timeout", + sessionKey: "agent:main:timeout", + backend: { liveSession: "claude-stdio" }, + }); + context.params.abortSignal = abortController.signal; + const resultPromise = runClaudeLiveSessionTurn({ + context, + args: context.preparedBackend.backend.args ?? [], + env: {}, + prompt: "hello", + useResume: false, + noOutputTimeoutMs: 120_000, + getProcessSupervisor: () => ({ + spawn: (params: Parameters[0]) => + supervisorSpawnMock(params) as ReturnType, + cancel: vi.fn(), + cancelScope: vi.fn(), + getRecord: vi.fn(), + }), + onAssistantDelta: () => {}, + cleanup: async () => {}, + }); + + await vi.waitFor(() => expect(stdoutListener).toBeDefined()); + abortController.abort(abortReason); + await expectRejectsWithFields(resultPromise, { name: expectedErrorName }); + await waitForDiagnosticEventsDrained(); + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + toolCallId: "tool-live-timeout", + ...expectedOutcome, + }), + ); + if (toolType === "server_tool_use") { + const terminal = diagnosticEvents.find( + (event) => event.toolCallId === "tool-live-timeout", + ); + expect(terminal).not.toHaveProperty("terminalReason"); + } + } finally { + stopDiagnostics(); + } + }, + ); + + it("preserves no-output watchdog timeout provenance for active Claude live tools", async () => { + const diagnosticEvents: Array> = []; + const stopDiagnostics = onTrustedToolExecutionEvent((event) => { + if (event.type === "tool.execution.error") { + diagnosticEvents.push(event as unknown as Record); + } + }); + let stdoutListener: ((chunk: string) => void) | undefined; + const stdin = { + write: vi.fn((_data: string, cb?: (err?: Error | null) => void) => { + stdoutListener?.( + [ + JSON.stringify({ type: "system", subtype: "init", session_id: "live-no-output" }), + JSON.stringify({ + type: "assistant", + session_id: "live-no-output", + message: { + role: "assistant", + content: [ + { + type: "tool_use", + id: "tool-live-no-output", + name: "Bash", + input: { command: "sleep 10" }, + }, + ], + }, + }), + ].join("\n") + "\n", + ); + cb?.(); + }), + end: vi.fn(), + }; + supervisorSpawnMock.mockImplementation(async (...args: unknown[]) => { + const input = (args[0] ?? {}) as { onStdout?: (chunk: string) => void }; + stdoutListener = input.onStdout; + return { + runId: "live-run-no-output", + pid: 3062, + startedAtMs: Date.now(), + stdin, + wait: vi.fn(() => new Promise(() => {})), + cancel: vi.fn(), + }; + }); + + try { + const context = buildPreparedCliRunContext({ + provider: "claude-cli", + model: "sonnet", + runId: "run-live-no-output", + sessionId: "session-live-no-output", + sessionKey: "agent:main:no-output", + backend: { liveSession: "claude-stdio" }, + timeoutMs: 120_000, + }); + const resultPromise = runClaudeLiveSessionTurn({ + context, + args: context.preparedBackend.backend.args ?? [], + env: {}, + prompt: "hello", + useResume: false, + noOutputTimeoutMs: 25, + getProcessSupervisor: () => ({ + spawn: (params: Parameters[0]) => + supervisorSpawnMock(params) as ReturnType, + cancel: vi.fn(), + cancelScope: vi.fn(), + getRecord: vi.fn(), + }), + onAssistantDelta: () => {}, + cleanup: async () => {}, + }); + const runExpectation = expectRejectsWithFields(resultPromise, { + name: "FailoverError", + message: "CLI produced no output for 0s and was terminated.", + }); + + await runExpectation; + expect(diagnosticEvents).toContainEqual( + expect.objectContaining({ + toolCallId: "tool-live-no-output", + terminalReason: "timed_out", + }), + ); } finally { stopDiagnostics(); } }); it("answers Claude live control_request can_use_tool with deny when exec policy is restrictive", async () => { + const diagnosticEvents: Array> = []; + const stopDiagnostics = onInternalDiagnosticEvent((event) => { + if ( + event.type.startsWith("tool.execution.") && + "toolCallId" in event && + event.toolCallId === "tool-deny-1" + ) { + diagnosticEvents.push(event as unknown as Record); + } + }); let stdoutListener: ((chunk: string) => void) | undefined; const writes: string[] = []; const stdin = { @@ -2321,6 +2814,36 @@ ${JSON.stringify({ subtype: "init", session_id: "live-control-deny", })} +${JSON.stringify({ + type: "assistant", + session_id: "live-control-deny", + message: { + role: "assistant", + content: [ + { + type: "tool_use", + id: "tool-deny-1", + name: "Bash", + input: { command: "rm -rf /" }, + }, + ], + }, +})} +${JSON.stringify({ + type: "user", + session_id: "live-control-deny", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-deny-1", + content: "denied", + is_error: true, + }, + ], + }, +})} ${JSON.stringify({ type: "result", session_id: "live-control-deny", @@ -2346,18 +2869,26 @@ ${JSON.stringify({ }; }); - const result = await executePreparedCliRun( - buildPreparedCliRunContext({ - provider: "claude-cli", - model: "sonnet", - runId: "run-control-deny", - prompt: "hello", - backend: { liveSession: "claude-stdio" }, - config: { - tools: { exec: { security: "allowlist", ask: "on-miss" } }, - } as PreparedCliRunContext["params"]["config"], - }), - ); + const result = await (async () => { + try { + const value = await executePreparedCliRun( + buildPreparedCliRunContext({ + provider: "claude-cli", + model: "sonnet", + runId: "run-control-deny", + prompt: "hello", + backend: { liveSession: "claude-stdio" }, + config: { + tools: { exec: { security: "allowlist", ask: "on-miss" } }, + } as PreparedCliRunContext["params"]["config"], + }), + ); + await waitForDiagnosticEventsDrained(); + return value; + } finally { + stopDiagnostics(); + } + })(); expect(result.text).toBe("ok"); const controlResponse = writes.find((entry) => entry.includes('"control_response"')); expect(controlResponse, "control_response written to stdin").toBeDefined(); @@ -2372,6 +2903,22 @@ ${JSON.stringify({ expect(parsed.response.response.behavior).toBe("deny"); expect(parsed.response.response.decisionClassification).toBe("user_reject"); expect(parsed.response.response.message).toContain("security=allowlist"); + expect(diagnosticEvents).toMatchObject([ + { + type: "tool.execution.started", + toolCallId: "tool-deny-1", + toolName: "Bash", + paramsSummary: { kind: "object" }, + }, + { + type: "tool.execution.blocked", + toolCallId: "tool-deny-1", + toolName: "Bash", + deniedReason: "cli_live_exec_policy", + }, + ]); + expect(diagnosticEvents).toHaveLength(2); + expect(JSON.stringify(diagnosticEvents)).not.toContain("rm -rf"); const spawnArg = supervisorSpawnMock.mock.calls.at(-1)?.[0] as { argv?: string[] }; expect(requireArgAfter(spawnArg.argv, "--permission-mode")).toBe("default"); }); diff --git a/src/agents/cli-runner/claude-live-session.ts b/src/agents/cli-runner/claude-live-session.ts index 818292ae363b..2b47e25e9ade 100644 --- a/src/agents/cli-runner/claude-live-session.ts +++ b/src/agents/cli-runner/claude-live-session.ts @@ -11,7 +11,6 @@ import { type DiagnosticToolParamsSummary, type DiagnosticToolSource, type DiagnosticToolExecutionErrorEvent, - type DiagnosticToolExecutionCompletedEvent, } from "../../infra/diagnostic-events.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { @@ -39,7 +38,7 @@ import { resolveCliStreamJsonOutputLimits, } from "../cli-output.js"; import { classifyFailoverReason } from "../embedded-agent-helpers.js"; -import { FailoverError, resolveFailoverStatus } from "../failover-error.js"; +import { FailoverError, isTimeoutError, resolveFailoverStatus } from "../failover-error.js"; import { prepareCliBundleMcpCaptureAttempt } from "./bundle-mcp.js"; import { buildClaudeOwnerKey } from "./helpers.js"; import { cliBackendLog, formatCliBackendOutputDigest } from "./log.js"; @@ -62,6 +61,8 @@ type ClaudeLiveTurn = { activeToolTimer: NodeJS.Timeout | null; activeTools: Map; observedStdout: boolean; + completedToolCallIds: Set; + toolEventCount: number; streamingParser: ReturnType; execPermission: ClaudeLiveExecPermission; resolve: (output: CliOutput) => void; @@ -96,18 +97,17 @@ type ClaudeLiveDiagnosticRefs = { runId: string; sessionId: string; sessionKey?: string; + agentId?: string; }; type ClaudeLiveActiveTool = { toolName: string; toolCallId: string; + kind: CliToolUseStartDelta["kind"]; startedAt: number; }; -type ClaudeLiveToolUse = { - toolName: string; - toolCallId: string; - paramsSummary?: DiagnosticToolParamsSummary; -}; - +type ClaudeLiveToolTerminalOutcome = + | { outcome: "blocked"; deniedReason: string; reason?: string } + | { outcome: "cancelled" | "failed" | "timed_out" | "unknown" }; const CLAUDE_LIVE_IDLE_TIMEOUT_MS = 10 * 60 * 1_000; const CLAUDE_LIVE_ACTIVE_TOOL_PROGRESS_MS = 10_000; const CLAUDE_LIVE_MAX_SESSIONS = 16; @@ -356,8 +356,25 @@ function buildClaudeLiveFingerprint(params: { }); } -function createAbortError(): Error { - return createNamedAbortError("CLI run aborted"); +// Preserve timeout identity and abort reasons so audit terminal outcomes +// can distinguish timed_out from cancelled runs. +function createAbortError(reason?: unknown): Error { + if (reason instanceof Error && isTimeoutError(reason)) { + return reason; + } + if (reason === undefined) { + return createNamedAbortError("CLI run aborted"); + } + const error = new Error( + reason instanceof Error + ? reason.message + : typeof reason === "string" + ? reason + : "CLI run aborted", + reason instanceof Error ? { cause: reason } : undefined, + ); + error.name = "AbortError"; + return error; } function clearTurnTimers(turn: ClaudeLiveTurn): void { @@ -383,9 +400,9 @@ function finishTurn(session: ClaudeLiveSession, output: CliOutput): void { cliBackendLog.info( `claude live session turn: provider=${session.providerId} model=${session.modelId} durationMs=${Date.now() - turn.startedAtMs} rawLines=${turn.rawLines.length} ${formatCliBackendOutputDigest(output.text)}`, ); - completeActiveClaudeLiveTools(turn); - clearTurnTimers(turn); turn.streamingParser.finish(); + failActiveClaudeLiveTools(turn, new Error("Tool result missing before turn completed")); + clearTurnTimers(turn); session.currentTurn = null; turn.resolve(output); scheduleIdleClose(session); @@ -400,9 +417,9 @@ function failTurn(session: ClaudeLiveSession, error: unknown): void { cliBackendLog.warn( `claude live session turn failed: provider=${session.providerId} model=${session.modelId} durationMs=${Date.now() - turn.startedAtMs} error=${errorKind}`, ); + turn.streamingParser.finish(); failActiveClaudeLiveTools(turn, error); clearTurnTimers(turn); - turn.streamingParser.finish(); session.currentTurn = null; turn.reject(error); } @@ -493,6 +510,7 @@ function claudeLiveDiagnosticBase(turn: ClaudeLiveTurn) { runId: turn.diagnosticRefs.runId, sessionId: turn.diagnosticRefs.sessionId, ...(turn.diagnosticRefs.sessionKey ? { sessionKey: turn.diagnosticRefs.sessionKey } : {}), + ...(turn.diagnosticRefs.agentId ? { agentId: turn.diagnosticRefs.agentId } : {}), }; } @@ -530,49 +548,6 @@ function summarizeClaudeLiveToolInput(input: unknown): DiagnosticToolParamsSumma } } -function readClaudeLiveMessageContent(parsed: Record): unknown[] { - const message = parsed.message; - if (!isRecord(message)) { - return []; - } - const content = message.content; - return Array.isArray(content) ? content : []; -} - -function readClaudeLiveToolUses(parsed: Record): ClaudeLiveToolUse[] { - const tools: ClaudeLiveToolUse[] = []; - for (const entry of readClaudeLiveMessageContent(parsed)) { - if (!isRecord(entry) || entry.type !== "tool_use") { - continue; - } - const toolName = typeof entry.name === "string" ? entry.name.trim() : ""; - const toolCallId = typeof entry.id === "string" ? entry.id.trim() : ""; - if (!toolName || !toolCallId) { - continue; - } - tools.push({ - toolName, - toolCallId, - paramsSummary: summarizeClaudeLiveToolInput(entry.input), - }); - } - return tools; -} - -function readClaudeLiveToolResultIds(parsed: Record): string[] { - const toolResultIds: string[] = []; - for (const entry of readClaudeLiveMessageContent(parsed)) { - if (!isRecord(entry) || entry.type !== "tool_result") { - continue; - } - const toolCallId = typeof entry.tool_use_id === "string" ? entry.tool_use_id.trim() : ""; - if (toolCallId) { - toolResultIds.push(toolCallId); - } - } - return toolResultIds; -} - function startClaudeLiveActiveToolHeartbeat(turn: ClaudeLiveTurn): void { if (turn.activeToolTimer || turn.activeTools.size === 0) { return; @@ -598,34 +573,48 @@ function stopClaudeLiveActiveToolHeartbeatIfIdle(turn: ClaudeLiveTurn): void { turn.activeToolTimer = null; } -function markClaudeLiveToolStarted(turn: ClaudeLiveTurn, tool: ClaudeLiveToolUse): void { +function markClaudeLiveToolStarted(turn: ClaudeLiveTurn, tool: CliToolUseStartDelta): void { + if (turn.completedToolCallIds.has(tool.toolCallId) || turn.activeTools.has(tool.toolCallId)) { + return; + } const now = Date.now(); turn.activeTools.set(tool.toolCallId, { - toolName: tool.toolName, + toolName: tool.name, toolCallId: tool.toolCallId, + kind: tool.kind, startedAt: now, }); + turn.toolEventCount += 1; emitTrustedDiagnosticEvent({ type: "tool.execution.started", ...claudeLiveDiagnosticBase(turn), - toolName: tool.toolName, - toolSource: diagnosticToolSourceForClaudeLiveTool(tool.toolName), + toolName: tool.name, + toolSource: diagnosticToolSourceForClaudeLiveTool(tool.name), toolOwner: "claude-cli", toolCallId: tool.toolCallId, - ...(tool.paramsSummary ? { paramsSummary: tool.paramsSummary } : {}), + paramsSummary: summarizeClaudeLiveToolInput(tool.args), }); emitClaudeLiveProgress(turn, "cli_live:tool_started"); startClaudeLiveActiveToolHeartbeat(turn); } -function markClaudeLiveToolCompleted(turn: ClaudeLiveTurn, toolCallId: string): void { - const activeTool = turn.activeTools.get(toolCallId); +function markClaudeLiveToolCompleted( + turn: ClaudeLiveTurn, + result: CliToolResultDelta, + terminalOutcome?: ClaudeLiveToolTerminalOutcome, +): void { + if (turn.completedToolCallIds.has(result.toolCallId)) { + return; + } + turn.toolEventCount += 1; + const activeTool = turn.activeTools.get(result.toolCallId); if (!activeTool) { emitClaudeLiveProgress(turn, "cli_live:tool_result"); return; } - turn.activeTools.delete(toolCallId); - const event: Omit = { + turn.activeTools.delete(result.toolCallId); + turn.completedToolCallIds.add(result.toolCallId); + const event = { ...claudeLiveDiagnosticBase(turn), toolName: activeTool.toolName, toolSource: diagnosticToolSourceForClaudeLiveTool(activeTool.toolName), @@ -633,55 +622,96 @@ function markClaudeLiveToolCompleted(turn: ClaudeLiveTurn, toolCallId: string): toolCallId: activeTool.toolCallId, durationMs: Math.max(0, Date.now() - activeTool.startedAt), }; - emitTrustedDiagnosticEvent({ - type: "tool.execution.completed", - ...event, - }); + if (terminalOutcome?.outcome === "blocked") { + emitTrustedDiagnosticEvent({ + type: "tool.execution.blocked", + ...event, + deniedReason: terminalOutcome.deniedReason, + reason: terminalOutcome.reason ?? "blocked by before-tool policy", + }); + } else if (terminalOutcome?.outcome === "unknown") { + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + ...event, + errorCategory: "cli_tool_ambiguous", + errorCode: "tool_outcome_unknown", + }); + } else if (terminalOutcome || result.isError) { + const terminalReason = terminalOutcome?.outcome ?? "failed"; + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + ...event, + errorCategory: terminalReason === "cancelled" ? "aborted" : "tool_failed", + terminalReason, + }); + } else { + emitTrustedDiagnosticEvent({ + type: "tool.execution.completed", + ...event, + }); + } emitClaudeLiveProgress(turn, "cli_live:tool_result"); stopClaudeLiveActiveToolHeartbeatIfIdle(turn); } -function completeActiveClaudeLiveTools(turn: ClaudeLiveTurn): void { - const activeToolCallIds = Array.from(turn.activeTools.keys()); - for (const toolCallId of activeToolCallIds) { - markClaudeLiveToolCompleted(turn, toolCallId); - } +function markClaudeLiveToolDenied(turn: ClaudeLiveTurn, tool: CliToolUseStartDelta): void { + markClaudeLiveToolStarted(turn, tool); + markClaudeLiveToolCompleted( + turn, + { toolCallId: tool.toolCallId, name: tool.name, isError: true }, + { + outcome: "blocked", + deniedReason: "cli_live_exec_policy", + reason: "blocked by CLI live execution policy", + }, + ); } function failActiveClaudeLiveTools(turn: ClaudeLiveTurn, error: unknown): void { - const errorCategory = error instanceof Error && error.name === "AbortError" ? "aborted" : "error"; + const timedOut = + isTimeoutError(error) || (error instanceof FailoverError && error.reason === "timeout"); + const aborted = error instanceof Error && error.name === "AbortError"; + const errorCategory = timedOut ? "timeout" : aborted ? "aborted" : "error"; + const terminalReason = timedOut ? "timed_out" : aborted ? "cancelled" : "failed"; for (const activeTool of turn.activeTools.values()) { - const event: Omit = { - ...claudeLiveDiagnosticBase(turn), - toolName: activeTool.toolName, - toolSource: diagnosticToolSourceForClaudeLiveTool(activeTool.toolName), - toolOwner: "claude-cli", - toolCallId: activeTool.toolCallId, - durationMs: Math.max(0, Date.now() - activeTool.startedAt), - errorCategory, - }; + const event: Omit = + { + ...claudeLiveDiagnosticBase(turn), + toolName: activeTool.toolName, + toolSource: diagnosticToolSourceForClaudeLiveTool(activeTool.toolName), + toolOwner: "claude-cli", + toolCallId: activeTool.toolCallId, + durationMs: Math.max(0, Date.now() - activeTool.startedAt), + }; + if (activeTool.kind === "server_tool_use") { + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + ...event, + errorCategory: "cli_tool_ambiguous", + errorCode: "tool_outcome_unknown", + }); + continue; + } emitTrustedDiagnosticEvent({ type: "tool.execution.error", ...event, + errorCategory, + terminalReason, }); } turn.activeTools.clear(); } -function noteClaudeLiveProgress(turn: ClaudeLiveTurn, parsed: Record): void { - const toolUses = readClaudeLiveToolUses(parsed); - const toolResultIds = readClaudeLiveToolResultIds(parsed); - for (const tool of toolUses) { - markClaudeLiveToolStarted(turn, tool); - } - for (const toolCallId of toolResultIds) { - markClaudeLiveToolCompleted(turn, toolCallId); - } +function noteClaudeLiveProgress( + turn: ClaudeLiveTurn, + parsed: Record, + sawToolEvent: boolean, +): void { if (parsed.type === "result") { emitClaudeLiveProgress(turn, "cli_live:result"); return; } - if (toolUses.length > 0 || toolResultIds.length > 0) { + if (sawToolEvent) { return; } emitClaudeLiveProgress(turn, "cli_live:stream_progress"); @@ -817,8 +847,17 @@ function handleClaudeLiveControlRequest( return; } const toolUseId = typeof request.tool_use_id === "string" ? request.tool_use_id : undefined; + const toolName = typeof request.tool_name === "string" ? request.tool_name.trim() : ""; const toolInput = isRecord(request.input) ? request.input : {}; const allowed = turn.execPermission.security === "full" && turn.execPermission.ask === "off"; + if (!allowed && toolUseId && toolName) { + markClaudeLiveToolDenied(turn, { + toolCallId: toolUseId, + name: toolName, + kind: "tool_use", + args: toolInput, + }); + } writeClaudeLiveControlResponse(session, { type: "control_response", response: { @@ -868,9 +907,10 @@ function handleClaudeLiveLine(session: ClaudeLiveSession, line: string): void { return; } turn.rawLines.push(trimmed); + const toolEventCountBefore = turn.toolEventCount; turn.streamingParser.push(`${trimmed}\n`); turn.sessionId = parseSessionId(parsed) ?? turn.sessionId; - noteClaudeLiveProgress(turn, parsed); + noteClaudeLiveProgress(turn, parsed, turn.toolEventCount !== toolEventCountBefore); handleClaudeLiveControlRequest(session, turn, parsed); if (parsed.type !== "result") { return; @@ -1107,6 +1147,9 @@ function createTurn(params: { onThinkingProgress?: (progress: CliThinkingProgress) => void; onToolUseStart?: (delta: CliToolUseStartDelta) => void; onToolResult?: (delta: CliToolResultDelta) => void; + resolveToolResultTerminalOutcome?: ( + delta: CliToolResultDelta, + ) => ClaudeLiveToolTerminalOutcome | undefined; onCommentaryText?: (text: string) => void; session: ClaudeLiveSession; execPermission: ClaudeLiveExecPermission; @@ -1119,6 +1162,7 @@ function createTurn(params: { runId: params.context.params.runId, sessionId: params.context.params.sessionId, ...(params.context.params.sessionKey ? { sessionKey: params.context.params.sessionKey } : {}), + ...(params.context.params.agentId ? { agentId: params.context.params.agentId } : {}), }, outputLimits: resolveCliStreamJsonOutputLimits(params.context.preparedBackend.backend), startedAtMs: Date.now(), @@ -1129,14 +1173,22 @@ function createTurn(params: { activeToolTimer: null, activeTools: new Map(), observedStdout: false, + completedToolCallIds: new Set(), + toolEventCount: 0, streamingParser: createCliJsonlStreamingParser({ backend: params.context.preparedBackend.backend, providerId: params.context.backendResolved.id, onAssistantDelta: params.onAssistantDelta, onThinkingDelta: params.onThinkingDelta, onThinkingProgress: params.onThinkingProgress, - onToolUseStart: params.onToolUseStart, - onToolResult: params.onToolResult, + onToolUseStart: (delta) => { + markClaudeLiveToolStarted(turn, delta); + params.onToolUseStart?.(delta); + }, + onToolResult: (delta) => { + markClaudeLiveToolCompleted(turn, delta, params.resolveToolResultTerminalOutcome?.(delta)); + params.onToolResult?.(delta); + }, onCommentaryText: params.onCommentaryText, }), execPermission: params.execPermission, @@ -1210,6 +1262,9 @@ export async function runClaudeLiveSessionTurn(params: { onThinkingProgress?: (progress: CliThinkingProgress) => void; onToolUseStart?: (delta: CliToolUseStartDelta) => void; onToolResult?: (delta: CliToolResultDelta) => void; + resolveToolResultTerminalOutcome?: ( + delta: CliToolResultDelta, + ) => ClaudeLiveToolTerminalOutcome | undefined; onCommentaryText?: (text: string) => void; onMcpCaptureReady?: (captureKey: string) => void; cleanup: () => Promise; @@ -1335,6 +1390,7 @@ export async function runClaudeLiveSessionTurn(params: { onThinkingProgress: params.onThinkingProgress, onToolUseStart: params.onToolUseStart, onToolResult: params.onToolResult, + resolveToolResultTerminalOutcome: params.resolveToolResultTerminalOutcome, onCommentaryText: params.onCommentaryText, session: liveSession, execPermission, @@ -1345,7 +1401,8 @@ export async function runClaudeLiveSessionTurn(params: { // Timeout/abort can reject the turn while stdin is backpressured. Keep the // rejection handled until the final await below rethrows the canonical result. void outputPromise.catch(() => undefined); - const abort = () => abortTurn(liveSession, createAbortError()); + const abort = () => + abortTurn(liveSession, createAbortError(params.context.params.abortSignal?.reason)); let replyBackendCompleted = false; const replyBackendHandle: ReplyBackendHandle | undefined = params.context.params.replyOperation ? { diff --git a/src/agents/cli-runner/execute.supervisor-capture.test.ts b/src/agents/cli-runner/execute.supervisor-capture.test.ts index f08cc0f5cb4e..8d503906bf9c 100644 --- a/src/agents/cli-runner/execute.supervisor-capture.test.ts +++ b/src/agents/cli-runner/execute.supervisor-capture.test.ts @@ -10,6 +10,11 @@ import { resolveMcpLoopbackYieldContext, } from "../../gateway/mcp-http.loopback-runtime.js"; import { onAgentEvent, resetAgentEventsForTest } from "../../infra/agent-events.js"; +import { + onTrustedToolExecutionEvent, + resetDiagnosticEventsForTest, + type TrustedToolExecutionEvent, +} from "../../infra/diagnostic-events.js"; import type { getProcessSupervisor } from "../../process/supervisor/index.js"; import { createManagedRun, supervisorSpawnMock } from "../cli-runner.test-support.js"; import { getCliMessagingDeliveryEvidence } from "./delivery-evidence.js"; @@ -39,17 +44,26 @@ function recordMcpLoopbackToolCallResult(params: { args: Record; result?: unknown; isError: boolean; + outcome?: "blocked" | "cancelled" | "completed" | "failed" | "timed_out" | "unknown"; + deniedReason?: string; }): void { const captureHandle = markMcpLoopbackToolCallStarted(params); if (!captureHandle) { return; } + const outcome = params.outcome ?? (params.isError ? "failed" : "completed"); + const result = + outcome === "blocked" + ? { + outcome, + deniedReason: params.deniedReason ?? "plugin-before-tool-call", + } + : { outcome, result: params.result }; recordMcpLoopbackToolCallResultForHandle({ captureHandle, toolName: params.toolName, args: params.args, - result: params.result, - isError: params.isError, + ...result, }); markMcpLoopbackToolCallFinished(captureHandle); } @@ -114,6 +128,7 @@ function requireSupervisorSpawnInput(): SupervisorSpawnInput { beforeEach(() => { resetAgentEventsForTest(); + resetDiagnosticEventsForTest(); supervisorSpawnMock.mockReset(); }); @@ -454,6 +469,736 @@ describe("executePreparedCliRun supervisor output capture", () => { } }); + it("emits metadata-only lifecycle records for parsed CLI tools", async () => { + const secret = "secret tool input and result"; + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + const chunks = [ + `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: "call-1", + name: "mcp__team__lookup", + input: { query: secret }, + }, + { + type: "mcp_tool_result", + tool_use_id: "call-1", + content: [{ type: "text", text: secret }], + }, + ], + }, + })}\n`, + `${JSON.stringify({ type: "result", session_id: "session-jsonl", result: "done" })}\n`, + ]; + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + for (const chunk of chunks) { + input.onStdout?.(chunk); + } + return createManagedRun({ + reason: "exit", + exitCode: 0, + exitSignal: null, + durationMs: 50, + stdout: "", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }); + }); + const context = buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }); + context.params.sessionKey = "agent:coder:main"; + context.params.agentId = "coder"; + + try { + await executePreparedCliRun(context); + } finally { + stop(); + } + + expect(toolEvents).toEqual([ + expect.objectContaining({ + type: "tool.execution.started", + runId: "run-jsonl", + sessionKey: "agent:coder:main", + sessionId: "session-1", + agentId: "coder", + toolName: "mcp__team__lookup", + toolSource: "mcp", + toolOwner: "cli-runner", + toolCallId: "call-1", + }), + expect.objectContaining({ + type: "tool.execution.completed", + runId: "run-jsonl", + toolCallId: "call-1", + }), + ]); + expect(JSON.stringify(toolEvents)).not.toContain(secret); + }); + + it.each([ + { + name: "policy block", + outcome: "blocked", + deniedReason: "plugin-approval", + expected: { type: "tool.execution.blocked", deniedReason: "plugin-approval" }, + }, + { + name: "resolved failure", + outcome: "failed", + deniedReason: undefined, + expected: { type: "tool.execution.error", terminalReason: "failed" }, + }, + { + name: "resolved timeout", + outcome: "timed_out", + deniedReason: undefined, + expected: { type: "tool.execution.error", terminalReason: "timed_out" }, + }, + ] as const)("preserves loopback $name for parsed CLI tools", async (testCase) => { + const toolCallId = `call-${testCase.outcome}`; + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + input.onStdout?.( + `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: toolCallId, + name: "mcp__openclaw__message", + input: { action: "react" }, + }, + ], + }, + })}\n`, + ); + recordMcpLoopbackToolCallResult({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? "", + toolName: "message", + args: { action: "react" }, + isError: true, + outcome: testCase.outcome, + ...(testCase.deniedReason ? { deniedReason: testCase.deniedReason } : {}), + }); + input.onStdout?.( + `${JSON.stringify({ + type: "user", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: toolCallId, + content: "blocked", + is_error: true, + }, + ], + }, + })}\n${JSON.stringify({ type: "result", session_id: "session-jsonl", result: "done" })}\n`, + ); + return createManagedRun({ + reason: "exit", + exitCode: 0, + exitSignal: null, + durationMs: 50, + stdout: "", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }); + }); + const context = buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }); + context.mcpDeliveryCapture = true; + + try { + await executePreparedCliRun(context); + } finally { + stop(); + } + + expect(toolEvents).toMatchObject([ + { type: "tool.execution.started", toolCallId }, + { ...testCase.expected, toolCallId }, + ]); + }); + + it("binds a loopback call admitted before its parsed CLI identity", async () => { + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + const captureHandle = markMcpLoopbackToolCallStarted({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY, + toolName: "message", + args: { action: "react", emoji: "early" }, + }); + if (!captureHandle) { + throw new Error("Expected early loopback capture handle"); + } + input.onStdout?.( + `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: "call-early", + name: "mcp__openclaw__message", + input: { action: "react", emoji: "early" }, + }, + ], + }, + })}\n`, + ); + recordMcpLoopbackToolCallResultForHandle({ + captureHandle, + toolName: "message", + args: { action: "react", emoji: "early" }, + outcome: "blocked", + deniedReason: "plugin-approval", + }); + markMcpLoopbackToolCallFinished(captureHandle); + input.onStdout?.( + `${JSON.stringify({ + type: "user", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call-early", + content: "blocked", + is_error: true, + }, + ], + }, + })}\n${JSON.stringify({ type: "result", session_id: "session-jsonl", result: "done" })}\n`, + ); + return createManagedRun({ + reason: "exit", + exitCode: 0, + exitSignal: null, + durationMs: 50, + stdout: "", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }); + }); + const context = buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }); + context.mcpDeliveryCapture = true; + + try { + await executePreparedCliRun(context); + } finally { + stop(); + } + + expect(toolEvents).toMatchObject([ + { type: "tool.execution.started", toolCallId: "call-early" }, + { + type: "tool.execution.blocked", + toolCallId: "call-early", + deniedReason: "plugin-approval", + }, + ]); + }); + + it("correlates parallel same-name loopback calls by arguments instead of admission order", async () => { + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + input.onStdout?.( + `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: "call-a", + name: "mcp__openclaw__message", + input: { action: "react", emoji: "A" }, + }, + { + type: "mcp_tool_use", + id: "call-b", + name: "mcp__openclaw__message", + input: { action: "react", emoji: "B" }, + }, + ], + }, + })}\n`, + ); + recordMcpLoopbackToolCallResult({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? "", + toolName: "message", + args: { action: "react", emoji: "B" }, + isError: true, + outcome: "failed", + }); + recordMcpLoopbackToolCallResult({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? "", + toolName: "message", + args: { action: "react", emoji: "A" }, + isError: false, + outcome: "completed", + }); + input.onStdout?.( + `${JSON.stringify({ + type: "user", + message: { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "call-a", content: "ok" }, + { type: "tool_result", tool_use_id: "call-b", content: "failed", is_error: true }, + ], + }, + })}\n${JSON.stringify({ type: "result", session_id: "session-jsonl", result: "done" })}\n`, + ); + return createManagedRun({ + reason: "exit", + exitCode: 0, + exitSignal: null, + durationMs: 50, + stdout: "", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }); + }); + const context = buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }); + context.mcpDeliveryCapture = true; + + try { + await executePreparedCliRun(context); + } finally { + stop(); + } + + expect(toolEvents).toMatchObject([ + { type: "tool.execution.started", toolCallId: "call-a" }, + { type: "tool.execution.started", toolCallId: "call-b" }, + { type: "tool.execution.completed", toolCallId: "call-a" }, + { type: "tool.execution.error", toolCallId: "call-b", terminalReason: "failed" }, + ]); + }); + + it.each([ + "request before both CLI identities", + "request between CLI identities", + "first tool finishes before second CLI identity", + ])("keeps identical parallel outcomes unknown with %s", async (ordering) => { + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + const toolArgs = { action: "react", emoji: "same" }; + const emitToolStarts = (toolCallIds: string[]) => { + input.onStdout?.( + `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: toolCallIds.map((id) => ({ + type: "mcp_tool_use", + id, + name: "mcp__openclaw__message", + input: toolArgs, + })), + }, + })}\n`, + ); + }; + const recordOutcome = (outcome: "completed" | "failed") => + recordMcpLoopbackToolCallResult({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? "", + toolName: "message", + args: toolArgs, + isError: outcome === "failed", + outcome, + }); + const emitToolResults = (toolCallIds: string[]) => { + input.onStdout?.( + `${JSON.stringify({ + type: "user", + message: { + role: "user", + content: toolCallIds.map((toolCallId) => ({ + type: "tool_result", + tool_use_id: toolCallId, + content: "ok", + })), + }, + })}\n`, + ); + }; + if (ordering === "request before both CLI identities") { + recordOutcome("failed"); + emitToolStarts(["call-identical-a", "call-identical-b"]); + recordOutcome("completed"); + } else if (ordering === "request between CLI identities") { + emitToolStarts(["call-identical-a"]); + recordOutcome("failed"); + recordOutcome("completed"); + emitToolStarts(["call-identical-b"]); + } else { + emitToolStarts(["call-identical-a"]); + recordOutcome("failed"); + recordOutcome("completed"); + emitToolResults(["call-identical-a"]); + emitToolStarts(["call-identical-b"]); + } + emitToolResults( + ordering === "first tool finishes before second CLI identity" + ? ["call-identical-b"] + : ["call-identical-a", "call-identical-b"], + ); + emitToolStarts(["call-identical-later"]); + recordOutcome("completed"); + input.onStdout?.( + `${JSON.stringify({ + type: "user", + message: { + role: "user", + content: [{ type: "tool_result", tool_use_id: "call-identical-later", content: "ok" }], + }, + })}\n${JSON.stringify({ type: "result", session_id: "session-jsonl", result: "done" })}\n`, + ); + return createManagedRun({ + reason: "exit", + exitCode: 0, + exitSignal: null, + durationMs: 50, + stdout: "", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }); + }); + const context = buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }); + context.mcpDeliveryCapture = true; + + try { + await executePreparedCliRun(context); + } finally { + stop(); + } + + expect(toolEvents).toHaveLength(6); + expect(toolEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: "tool.execution.started", + toolCallId: "call-identical-a", + }), + expect.objectContaining({ + type: "tool.execution.started", + toolCallId: "call-identical-b", + }), + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "call-identical-a", + errorCode: "tool_outcome_unknown", + }), + expect.objectContaining({ + type: "tool.execution.error", + toolCallId: "call-identical-b", + errorCode: "tool_outcome_unknown", + }), + expect.objectContaining({ + type: "tool.execution.started", + toolCallId: "call-identical-later", + }), + expect.objectContaining({ + type: "tool.execution.completed", + toolCallId: "call-identical-later", + }), + ]), + ); + }); + + it("uses a loopback outcome that settles during the post-process drain", async () => { + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + const toolArgs = { action: "react", emoji: "A" }; + input.onStdout?.( + `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: "call-draining", + name: "mcp__openclaw__message", + input: toolArgs, + }, + ], + }, + })}\n${JSON.stringify({ type: "result", session_id: "session-jsonl", result: "done" })}\n`, + ); + const captureHandle = markMcpLoopbackToolCallStarted({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY, + toolName: "message", + args: toolArgs, + }); + if (!captureHandle) { + throw new Error("Expected loopback capture handle"); + } + setTimeout(() => { + recordMcpLoopbackToolCallResultForHandle({ + captureHandle, + toolName: "message", + args: toolArgs, + outcome: "completed", + result: { ok: true }, + }); + markMcpLoopbackToolCallFinished(captureHandle); + }, 10); + return createManagedRun({ + reason: "exit", + exitCode: 0, + exitSignal: null, + durationMs: 50, + stdout: "", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }); + }); + const context = buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }); + context.mcpDeliveryCapture = true; + + try { + await executePreparedCliRun(context); + } finally { + stop(); + } + + expect(toolEvents).toMatchObject([ + { type: "tool.execution.started", toolCallId: "call-draining" }, + { type: "tool.execution.completed", toolCallId: "call-draining" }, + ]); + }); + + it("finishes parsed CLI tools when the process exits before a tool result", async () => { + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + const toolStart = `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: "call-incomplete", + name: "mcp__team__lookup", + input: {}, + }, + ], + }, + })}\n`; + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + input.onStdout?.(toolStart); + return createManagedRun({ + reason: "exit", + exitCode: 1, + exitSignal: null, + durationMs: 50, + stdout: "", + stderr: "failed", + timedOut: false, + noOutputTimedOut: false, + }); + }); + + try { + await expect( + executePreparedCliRun( + buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }), + ), + ).rejects.toThrow(); + } finally { + stop(); + } + + expect(toolEvents).toMatchObject([ + { + type: "tool.execution.started", + toolCallId: "call-incomplete", + }, + { + type: "tool.execution.error", + toolCallId: "call-incomplete", + errorCategory: "cli_tool_incomplete", + }, + ]); + }); + + it("cancels an outstanding parsed CLI tool when the enclosing run is aborted", async () => { + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + const abortController = new AbortController(); + const toolStart = `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: "mcp_tool_use", + id: "call-cancelled", + name: "mcp__openclaw__cron", + input: {}, + }, + ], + }, + })}\n`; + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + input.onStdout?.(toolStart); + recordMcpLoopbackToolCallResult({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? "", + toolName: "cron", + args: {}, + isError: true, + outcome: "unknown", + }); + abortController.abort(); + return createManagedRun({ + reason: "manual-cancel", + exitCode: null, + exitSignal: "SIGTERM", + durationMs: 50, + stdout: "", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }); + }); + const context = buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }); + context.params.abortSignal = abortController.signal; + context.mcpDeliveryCapture = true; + + try { + await expect(executePreparedCliRun(context)).rejects.toThrow("aborted"); + } finally { + stop(); + } + + expect(toolEvents).toMatchObject([ + { type: "tool.execution.started", toolCallId: "call-cancelled" }, + { + type: "tool.execution.error", + toolCallId: "call-cancelled", + errorCategory: "aborted", + terminalReason: "cancelled", + }, + ]); + }); + + it.each([ + { + label: "MCP tool", + type: "mcp_tool_use", + toolCallId: "call-timeout", + name: "mcp__openclaw__cron", + expected: { terminalReason: "timed_out" }, + }, + { + label: "server-native tool", + type: "server_tool_use", + toolCallId: "call-native-unknown", + name: "web_search", + expected: { errorCode: "tool_outcome_unknown" }, + }, + ] as const)("classifies an outstanding parsed $label when the run times out", async (fixture) => { + const toolEvents: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => toolEvents.push(event)); + const toolStart = `${JSON.stringify({ + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: fixture.type, + id: fixture.toolCallId, + name: fixture.name, + input: {}, + }, + ], + }, + })}\n`; + supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { + const input = args[0] as SupervisorSpawnInput; + input.onStdout?.(toolStart); + if (fixture.type === "mcp_tool_use") { + recordMcpLoopbackToolCallResult({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? "", + toolName: "cron", + args: {}, + isError: true, + outcome: "unknown", + }); + } + if (fixture.type === "server_tool_use") { + recordMcpLoopbackToolCallResult({ + captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? "", + toolName: "web_search", + args: {}, + isError: false, + outcome: "completed", + }); + } + return createManagedRun({ + reason: "overall-timeout", + exitCode: null, + exitSignal: "SIGTERM", + durationMs: 1_000, + stdout: "", + stderr: "", + timedOut: true, + noOutputTimedOut: false, + }); + }); + + try { + const context = buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }); + context.mcpDeliveryCapture = true; + await expect(executePreparedCliRun(context)).rejects.toThrow("exceeded timeout"); + } finally { + stop(); + } + + expect(toolEvents).toMatchObject([ + { type: "tool.execution.started", toolCallId: fixture.toolCallId }, + { + type: "tool.execution.error", + toolCallId: fixture.toolCallId, + ...fixture.expected, + }, + ]); + if (fixture.type === "server_tool_use") { + expect(toolEvents[1]).not.toHaveProperty("terminalReason"); + } + }); + it("reports only confirmed message deliveries from correlated JSONL tool events", async () => { const chunks = [ `${JSON.stringify({ @@ -929,7 +1674,7 @@ describe("executePreparedCliRun supervisor output capture", () => { ]); }); - it("preserves partial delivery evidence from failed MCP message calls", async () => { + it("preserves partial delivery evidence from unknown MCP message outcomes", async () => { const context = buildPreparedCliRunContext({ output: "text", provider: "google-gemini-cli" }); context.mcpDeliveryCapture = true; supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => { @@ -946,6 +1691,7 @@ describe("executePreparedCliRun supervisor output capture", () => { }, result: Object.assign(new Error("second chunk failed"), { sentBeforeError: true }), isError: true, + outcome: "unknown", }); input.onStdout?.("done"); return createManagedRun({ diff --git a/src/agents/cli-runner/execute.ts b/src/agents/cli-runner/execute.ts index f377fb72d885..701837935d58 100644 --- a/src/agents/cli-runner/execute.ts +++ b/src/agents/cli-runner/execute.ts @@ -3,10 +3,12 @@ * live-session routing, and diagnostics. */ import crypto from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; import { beginMcpLoopbackToolCallCapture, clearMcpLoopbackToolCallCapture, type McpLoopbackToolCallStart, + type McpLoopbackToolCallTerminalOutcome, waitForMcpLoopbackToolCallCaptureIdle, } from "../../gateway/mcp-http.loopback-runtime.js"; import { shouldLogVerbose } from "../../globals.js"; @@ -15,6 +17,7 @@ import { assertAgentRunLifecycleGenerationCurrent, emitAgentEvent, } from "../../infra/agent-events.js"; +import { emitTrustedDiagnosticEvent } from "../../infra/diagnostic-events.js"; import { isTruthyEnvValue } from "../../infra/env.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { @@ -37,6 +40,7 @@ import { type CliStreamingDelta, type CliThinkingDelta, type CliThinkingProgress, + type CliToolUseStartDelta, } from "../cli-output.js"; import { classifyFailoverReason } from "../embedded-agent-helpers.js"; import { @@ -64,6 +68,7 @@ import { } from "../embedded-agent-subscribe.tools.js"; import { FailoverError, resolveFailoverStatus } from "../failover-error.js"; import { applyPluginTextReplacements } from "../plugin-text-transforms.js"; +import { resolveAgentRunAbortLifecycleFields } from "../run-termination.js"; import { prepareCliBundleMcpCaptureAttempt } from "./bundle-mcp.js"; import { rotateClaudeLiveMcpCaptureKeyForContext, @@ -104,6 +109,7 @@ const executeDeps = { const CLI_RUNNER_OUTPUT_TAIL_BYTES = 64 * 1024; const CLI_RUNNER_OUTPUT_PARSE_BYTES = 1024 * 1024; const CLI_MESSAGING_EVIDENCE_MAX_CALLS = 64; +const CLI_LOOPBACK_CORRELATION_MAX_CALLS = 64; const CLI_MCP_DELIVERY_DRAIN_GRACE_MS = 5_000; const CLI_MCP_REQUEST_ADMISSION_GRACE_MS = 250; const OPENCLAW_MCP_TOOL_PREFIX = "mcp__openclaw__"; @@ -603,6 +609,173 @@ export async function executePreparedCliRun( string, { toolName: string; args: Record; target?: MessagingToolSend } >(); + type CliToolTerminalOutcome = McpLoopbackToolCallTerminalOutcome | { outcome: "completed" }; + type CliLoopbackAmbiguityGroup = { + calls: Set; + activeToolCallIds: Set; + }; + type CliLoopbackCall = { + admitted: McpLoopbackToolCallStart; + current: McpLoopbackToolCallStart; + boundToolCallId?: string; + outcome?: CliToolTerminalOutcome; + ambiguous: boolean; + ambiguityGroup?: CliLoopbackAmbiguityGroup; + }; + type ActiveCliTool = { + toolName: string; + args: Record; + loopbackCall?: CliLoopbackCall; + loopbackAmbiguous: boolean; + ambiguityGroup?: CliLoopbackAmbiguityGroup; + }; + const cliLoopbackCalls: CliLoopbackCall[] = []; + const activeCliTools = new Map(); + let cliLoopbackCorrelationOverflowed = false; + const matchesCliLoopbackCall = ( + toolName: string, + toolArgs: Record, + call: McpLoopbackToolCallStart, + ) => + normalizeCliMessagingToolName(toolName) === call.toolName && + isDeepStrictEqual(toolArgs, call.args); + const markCliLoopbackCallsAmbiguous = ( + calls: CliLoopbackCall[], + activeEntries = Array.from(activeCliTools.entries()).filter( + ([, activeTool]) => + activeTool.loopbackCall !== undefined && calls.includes(activeTool.loopbackCall), + ), + ) => { + const groups = new Set(); + for (const call of calls) { + if (call.ambiguityGroup) { + groups.add(call.ambiguityGroup); + } + } + for (const [, activeTool] of activeEntries) { + if (activeTool.ambiguityGroup) { + groups.add(activeTool.ambiguityGroup); + } + } + const group = groups.values().next().value ?? { + calls: new Set(), + activeToolCallIds: new Set(), + }; + for (const existing of groups) { + if (existing === group) { + continue; + } + for (const call of existing.calls) { + call.ambiguityGroup = group; + group.calls.add(call); + } + for (const toolCallId of existing.activeToolCallIds) { + const activeTool = activeCliTools.get(toolCallId); + if (activeTool) { + activeTool.ambiguityGroup = group; + group.activeToolCallIds.add(toolCallId); + } + } + existing.calls.clear(); + existing.activeToolCallIds.clear(); + } + for (const call of calls) { + call.ambiguous = true; + call.ambiguityGroup = group; + group.calls.add(call); + } + for (const [toolCallId, activeTool] of activeEntries) { + activeTool.loopbackAmbiguous = true; + activeTool.ambiguityGroup = group; + group.activeToolCallIds.add(toolCallId); + } + }; + const markCliLoopbackSignatureAmbiguous = (call: McpLoopbackToolCallStart) => { + const calls = cliLoopbackCalls.filter((candidate) => + matchesCliLoopbackCall(call.toolName, call.args, candidate.admitted), + ); + const activeEntries = Array.from(activeCliTools.entries()).filter(([, activeTool]) => + matchesCliLoopbackCall(activeTool.toolName, activeTool.args, call), + ); + markCliLoopbackCallsAmbiguous(calls, activeEntries); + }; + const retainCliLoopbackCall = (call: McpLoopbackToolCallStart) => { + if (cliLoopbackCalls.length >= CLI_LOOPBACK_CORRELATION_MAX_CALLS) { + cliLoopbackCorrelationOverflowed = true; + for (const activeTool of activeCliTools.values()) { + if (activeTool.loopbackCall || activeTool.toolName.startsWith("mcp__")) { + activeTool.loopbackAmbiguous = true; + } + } + cliLoopbackCalls.length = 0; + return undefined; + } + const retained: CliLoopbackCall = { + admitted: call, + current: call, + ambiguous: false, + }; + cliLoopbackCalls.push(retained); + return retained; + }; + const bindCliLoopbackCall = ( + call: CliLoopbackCall, + toolCallId: string, + activeTool: ActiveCliTool, + ) => { + call.boundToolCallId = toolCallId; + activeTool.loopbackCall = call; + activeTool.loopbackAmbiguous ||= call.ambiguous; + if (call.ambiguityGroup) { + activeTool.ambiguityGroup = call.ambiguityGroup; + call.ambiguityGroup.activeToolCallIds.add(toolCallId); + } + }; + const removeCliLoopbackCall = (call: CliLoopbackCall | undefined) => { + if (!call) { + return; + } + const index = cliLoopbackCalls.indexOf(call); + if (index >= 0) { + cliLoopbackCalls.splice(index, 1); + } + }; + const retireCliLoopbackCorrelation = ( + toolCallId: string, + activeTool: ActiveCliTool | undefined, + ) => { + removeCliLoopbackCall(activeTool?.loopbackCall); + const group = activeTool?.ambiguityGroup; + if (!group) { + return; + } + group.activeToolCallIds.delete(toolCallId); + const hasUnboundCall = Array.from(group.calls).some( + (call) => call.boundToolCallId === undefined && cliLoopbackCalls.includes(call), + ); + if (group.activeToolCallIds.size > 0 || hasUnboundCall) { + return; + } + // An ambiguous group owns unbound captures too. Retire the whole group + // once its parsed tools finish so stale calls cannot poison later tools. + for (const call of group.calls) { + removeCliLoopbackCall(call); + } + group.calls.clear(); + }; + const resolveCliLoopbackTerminalOutcome = (toolCallId: string) => { + const activeTool = activeCliTools.get(toolCallId); + if (activeTool?.loopbackAmbiguous) { + return { outcome: "unknown" } as const; + } + return activeTool?.loopbackCall?.outcome; + }; + const matchingActiveCliTools = ( + call: McpLoopbackToolCallStart, + ): Array<[string, ActiveCliTool]> => + Array.from(activeCliTools.entries()).filter(([, activeTool]) => + matchesCliLoopbackCall(activeTool.toolName, activeTool.args, call), + ); const messagingToolSentTexts: string[] = []; const messagingToolSentTextKeys = new Set(); const messagingToolSentMediaUrls: string[] = []; @@ -668,6 +841,16 @@ export async function executePreparedCliRun( : {}), }; }; + const resolveToolTerminalReason = (error?: unknown) => { + const abortFields = resolveAgentRunAbortLifecycleFields(params.abortSignal); + if (abortFields.aborted) { + return abortFields.stopReason === "timeout" ? "timed_out" : "cancelled"; + } + return error instanceof FailoverError && error.reason === "timeout" + ? "timed_out" + : "failed"; + }; + let finalizeParsedTools = () => {}; try { cliBackendLog.info( buildCliExecLogLine({ @@ -867,11 +1050,43 @@ export async function executePreparedCliRun( inFlightUnclassifiedMcpRequests = Math.max(0, inFlightUnclassifiedMcpRequests - 1); }, onToolCallStart: (call) => { + const retained = retainCliLoopbackCall(call); + const candidates = matchingActiveCliTools(call); + // Parallel same-name calls can reach the loopback out of stream + // order. Bind only a unique name+arguments match; ambiguity is + // safer than assigning a trusted terminal outcome to the wrong call. + let matched = + retained && + candidates.length === 1 && + !candidates[0]?.[1].loopbackCall && + !candidates[0]?.[1].loopbackAmbiguous + ? candidates[0] + : undefined; + if (retained && matched) { + bindCliLoopbackCall(retained, matched[0], matched[1]); + } else if (retained && candidates.length > 0) { + markCliLoopbackSignatureAmbiguous(call); + // The exact identity is unknowable, but pairing an unmatched + // peer keeps the ambiguity group's lifetime count complete. + matched = candidates.find(([, activeTool]) => !activeTool.loopbackCall); + if (matched) { + bindCliLoopbackCall(retained, matched[0], matched[1]); + } + } if (isAdmittedPotentialMessagingDelivery(call.toolName)) { inFlightMessagingToolCalls += 1; } + return matched?.[0]; }, onToolCallUpdate: ({ previous, current }) => { + const candidates = cliLoopbackCalls.filter((candidate) => + matchesCliLoopbackCall(previous.toolName, previous.args, candidate.current), + ); + if (candidates.length === 1 && !candidates[0]?.ambiguous) { + candidates[0].current = current; + } else if (candidates.length > 0) { + markCliLoopbackCallsAmbiguous(candidates); + } inFlightPreparedMessagingCalls.delete(previous); const wasMessagingSend = isAdmittedPotentialMessagingDelivery(previous.toolName); const isMessagingSend = isPreparedMessagingDelivery(current.toolName, current.args); @@ -894,17 +1109,36 @@ export async function executePreparedCliRun( } inFlightPreparedMessagingCalls.delete(call); }, - onToolCallResult: ({ toolName, args: toolArgs, result, isError }) => { - const normalizedToolName = normalizeCliMessagingToolName(toolName); - if (!isMessagingToolDeliveryAction(normalizedToolName, toolArgs)) { + onToolCallResult: (call) => { + const terminalOutcome: CliToolTerminalOutcome = + call.outcome === "blocked" + ? { outcome: call.outcome, deniedReason: call.deniedReason } + : { outcome: call.outcome }; + const correlated = call.correlationId + ? cliLoopbackCalls.find( + (candidate) => candidate.boundToolCallId === call.correlationId, + ) + : undefined; + const candidates = correlated + ? [correlated] + : cliLoopbackCalls.filter((candidate) => + matchesCliLoopbackCall(call.toolName, call.args, candidate.current), + ); + if (candidates.length === 1 && candidates[0]) { + candidates[0].outcome = terminalOutcome; + } else if (candidates.length > 1) { + markCliLoopbackCallsAmbiguous(candidates); + } + const normalizedToolName = normalizeCliMessagingToolName(call.toolName); + if (!isMessagingToolDeliveryAction(normalizedToolName, call.args)) { return; } commitMessagingToolResult({ toolName: normalizedToolName, - target: extractCliMessagingTarget(context, normalizedToolName, toolArgs), - args: toolArgs, - result, - isError, + target: extractCliMessagingTarget(context, normalizedToolName, call.args), + args: call.args, + result: "result" in call ? call.result : undefined, + isError: call.outcome !== "completed", }); }, }); @@ -912,14 +1146,47 @@ export async function executePreparedCliRun( beginGatewayCapture(initialGatewayCaptureKey); let observedCliActivity = false; const emitLiveEvents = params.executionMode !== "side-question"; - const emitCliToolUseStart = (event: { - toolCallId: string; - name: string; - args: Record; - }) => { + const activeParsedTools = new Map< + string, + { startedAt: number; toolName: string; kind: CliToolUseStartDelta["kind"] } + >(); + const emitCliToolUseStart = (event: CliToolUseStartDelta) => { observedCliActivity = true; + // Server-native calls have their own result stream and must never inherit MCP outcomes. + if (event.kind !== "server_tool_use") { + const activeTool = { + toolName: event.name, + args: event.args, + loopbackAmbiguous: cliLoopbackCorrelationOverflowed && event.name.startsWith("mcp__"), + }; + activeCliTools.set(event.toolCallId, activeTool); + const admittedCall = { + toolName: normalizeCliMessagingToolName(event.name), + args: event.args, + }; + const pendingCandidates = cliLoopbackCalls.filter( + (candidate) => + candidate.boundToolCallId === undefined && + matchesCliLoopbackCall(event.name, event.args, candidate.admitted), + ); + const hasAssociatedPeer = matchingActiveCliTools(admittedCall).some( + ([toolCallId, peer]) => + toolCallId !== event.toolCallId && + (peer.loopbackCall !== undefined || peer.loopbackAmbiguous), + ); + const pending = pendingCandidates[0]; + if (hasAssociatedPeer || pendingCandidates.length > 1 || pending?.ambiguous) { + markCliLoopbackSignatureAmbiguous(admittedCall); + if (pending) { + bindCliLoopbackCall(pending, event.toolCallId, activeTool); + } + } else if (pendingCandidates.length === 1 && pending) { + bindCliLoopbackCall(pending, event.toolCallId, activeTool); + } + } const toolName = normalizeCliMessagingToolName(event.name); if ( + event.kind !== "server_tool_use" && !gatewayCaptureKey && event.args.dryRun !== true && isMessagingToolDeliveryAction(toolName, event.args) @@ -960,6 +1227,9 @@ export async function executePreparedCliRun( result?: unknown; }) => { observedCliActivity = true; + const activeTool = activeCliTools.get(event.toolCallId); + activeCliTools.delete(event.toolCallId); + retireCliLoopbackCorrelation(event.toolCallId, activeTool); const pending = pendingMessagingCalls.get(event.toolCallId); if (pending) { pendingMessagingCalls.delete(event.toolCallId); @@ -986,6 +1256,134 @@ export async function executePreparedCliRun( }, }); }; + const emitParsedToolUseStart = (event: CliToolUseStartDelta) => { + const startedAt = Date.now(); + activeParsedTools.set(event.toolCallId, { + startedAt, + toolName: event.name, + kind: event.kind, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId: params.runId, + sessionId: params.sessionId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), + toolName: event.name, + toolSource: event.name.startsWith("mcp__") ? "mcp" : "core", + toolOwner: "cli-runner", + toolCallId: event.toolCallId, + }); + emitCliToolUseStart(event); + }; + const emitParsedToolTerminal = (event: { + toolCallId: string; + name: string; + isError: boolean; + incomplete?: boolean; + }) => { + const activeTool = activeParsedTools.get(event.toolCallId); + activeParsedTools.delete(event.toolCallId); + const trustedOutcome = resolveCliLoopbackTerminalOutcome(event.toolCallId); + const toolName = activeTool?.toolName ?? event.name; + const now = Date.now(); + const trustedTerminalReason = + trustedOutcome && + trustedOutcome.outcome !== "blocked" && + trustedOutcome.outcome !== "completed" && + trustedOutcome.outcome !== "unknown" + ? trustedOutcome.outcome + : undefined; + const terminalReason = + trustedTerminalReason ?? + resolveToolTerminalReason(event.incomplete ? runError : undefined); + // Incomplete client/MCP tools inherit the enclosing failed run even when + // the loopback disconnect is ambiguous. Server-native tools do not. + const useEnclosingTerminalReason = + event.incomplete && + runFailed && + activeTool !== undefined && + activeTool.kind !== "server_tool_use"; + const diagnosticBase = { + runId: params.runId, + sessionId: params.sessionId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), + toolName, + toolSource: toolName.startsWith("mcp__") ? ("mcp" as const) : ("core" as const), + toolOwner: "cli-runner", + toolCallId: event.toolCallId, + durationMs: Math.max(0, now - (activeTool?.startedAt ?? now)), + }; + if (trustedOutcome?.outcome === "unknown" && !useEnclosingTerminalReason) { + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + ...diagnosticBase, + errorCategory: "cli_tool_ambiguous", + errorCode: "tool_outcome_unknown", + }); + return; + } + if ( + event.incomplete && + activeTool?.kind === "server_tool_use" && + trustedOutcome === undefined + ) { + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + ...diagnosticBase, + errorCategory: "cli_tool_ambiguous", + errorCode: "tool_outcome_unknown", + }); + return; + } + const trustedFailure = + trustedOutcome !== undefined && trustedOutcome.outcome !== "completed"; + emitTrustedDiagnosticEvent( + trustedOutcome?.outcome === "blocked" + ? { + type: "tool.execution.blocked", + ...diagnosticBase, + deniedReason: trustedOutcome.deniedReason, + reason: "blocked by before-tool policy", + } + : trustedFailure || (trustedOutcome === undefined && event.isError) + ? { + type: "tool.execution.error", + ...diagnosticBase, + errorCategory: + terminalReason === "cancelled" + ? "aborted" + : event.incomplete && (!trustedOutcome || useEnclosingTerminalReason) + ? "cli_tool_incomplete" + : "cli_tool", + terminalReason, + } + : { + type: "tool.execution.completed", + ...diagnosticBase, + }, + ); + }; + const emitParsedToolResult = (event: { + toolCallId: string; + name: string; + isError: boolean; + result?: unknown; + }) => { + emitParsedToolTerminal(event); + emitCliToolResult(event); + }; + finalizeParsedTools = () => { + for (const [toolCallId, activeTool] of Array.from(activeParsedTools)) { + emitParsedToolTerminal({ + toolCallId, + name: activeTool.toolName, + isError: true, + incomplete: true, + }); + } + }; let commentaryCounter = 0; const emitCliCommentaryText = (text: string) => { if (!emitLiveEvents) { @@ -1084,6 +1482,10 @@ export async function executePreparedCliRun( onThinkingProgress: emitCliThinkingProgress, onToolUseStart: emitCliToolUseStart, onToolResult: emitCliToolResult, + resolveToolResultTerminalOutcome: (event) => { + const outcome = resolveCliLoopbackTerminalOutcome(event.toolCallId); + return outcome?.outcome === "completed" ? undefined : outcome; + }, onCommentaryText: emitLiveEvents && context.params.emitCommentaryText ? emitCliCommentaryText @@ -1111,8 +1513,8 @@ export async function executePreparedCliRun( onAssistantDelta: emitCliAssistantDelta, onThinkingDelta: emitCliThinkingDelta, onThinkingProgress: emitCliThinkingProgress, - onToolUseStart: emitCliToolUseStart, - onToolResult: emitCliToolResult, + onToolUseStart: emitParsedToolUseStart, + onToolResult: emitParsedToolResult, onCommentaryText: emitLiveEvents && context.params.emitCommentaryText ? emitCliCommentaryText @@ -1490,6 +1892,9 @@ export async function executePreparedCliRun( } recordRunError(error); } finally { + // Captured MCP calls may settle after the CLI process exits. Drain + // first so finalization can use their trusted terminal outcomes. + finalizeParsedTools(); if (gatewayCaptureKey) { clearMcpLoopbackToolCallCapture(gatewayCaptureKey); } diff --git a/src/agents/command/attempt-execution.error-propagation.test.ts b/src/agents/command/attempt-execution.error-propagation.test.ts index c257d6b431f7..a075b0889720 100644 --- a/src/agents/command/attempt-execution.error-propagation.test.ts +++ b/src/agents/command/attempt-execution.error-propagation.test.ts @@ -1,31 +1,79 @@ // Covers ACP diagnostic event propagation and sanitized error formatting. import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { ACP_TURN_TIMEOUT_DETAIL_CODE } from "../../acp/control-plane/manager.turn-timeout.js"; import { AcpRuntimeError, formatAcpErrorChain } from "../../acp/runtime/errors.js"; import { type AgentEventPayload, + onAgentAuditEvent, onAgentEvent, resetAgentEventsForTest, } from "../../infra/agent-events.js"; import { - emitAcpLifecycleError, + onTrustedToolExecutionEvent, + resetDiagnosticEventsForTest, + type TrustedToolExecutionEvent, +} from "../../infra/diagnostic-events.js"; +import { + buildAcpResult, + createAcpToolLifecycleTracker, + emitAcpLifecycleStart, + emitAcpLifecycleEnd as emitAcpLifecycleEndBase, + emitAcpLifecycleError as emitAcpLifecycleErrorBase, emitAcpPromptSubmitted, - emitAcpRuntimeEvent, + emitAcpRuntimeEvent as emitAcpRuntimeEventBase, } from "./attempt-execution.js"; let captured: AgentEventPayload[] = []; +let capturedTools: TrustedToolExecutionEvent[] = []; let unsubscribe: (() => void) | undefined; +let unsubscribeTools: (() => void) | undefined; +let toolTracker = createAcpToolLifecycleTracker(); + +function emitAcpRuntimeEvent( + params: Omit[0], "toolTracker">, +) { + return emitAcpRuntimeEventBase({ ...params, toolTracker }); +} + +function emitAcpLifecycleEnd( + params: Omit[0], "toolTracker">, +) { + return emitAcpLifecycleEndBase({ ...params, toolTracker }); +} + +function emitAcpLifecycleError( + params: Omit[0], "toolTracker">, +) { + return emitAcpLifecycleErrorBase({ ...params, toolTracker }); +} beforeEach(() => { resetAgentEventsForTest(); + resetDiagnosticEventsForTest(); + toolTracker = createAcpToolLifecycleTracker(); captured = []; + capturedTools = []; // Subscribe to the process-level event bus so tests observe exactly what // parent relay diagnostics would receive. unsubscribe = onAgentEvent((evt) => { captured.push(evt); }); + unsubscribeTools = onTrustedToolExecutionEvent((event) => { + capturedTools.push(event); + }); }); describe("ACP diagnostic events", () => { + it("preserves cancelled result metadata without a stop reason", () => { + const result = buildAcpResult({ + payloadText: "", + startedAt: Date.now(), + resultStatus: "cancelled", + }); + + expect(result.meta).toMatchObject({ aborted: true, stopReason: "stop" }); + }); + it("emits prompt-submitted state with proxy env names but not values", () => { const previous = process.env.HTTPS_PROXY; process.env.HTTPS_PROXY = "http://proxy.example.invalid:8080"; @@ -74,12 +122,478 @@ describe("ACP diagnostic events", () => { expect(String(event?.data.text)).toContain("connecting"); expect(String(event?.data.text)).not.toContain("sk-abcdefghijklmnopqrstuvwxyz123456"); }); + + it("keeps audit-only ACP lifecycle and runtime events off the shared bus", () => { + const secret = "private ACP cause chain"; + const auditEvents: AgentEventPayload[] = []; + const stopAudit = onAgentAuditEvent((event) => auditEvents.push(event)); + try { + emitAcpLifecycleStart({ + runId: "run-audit-only", + sessionKey: "agent:main:acp:child", + startedAt: 123, + auditOnly: true, + }); + emitAcpRuntimeEvent({ + runId: "run-audit-only", + sessionKey: "agent:main:acp:child", + auditOnly: true, + event: { + type: "tool_call", + tag: "tool_call", + text: "private command payload", + kind: "execute", + toolCallId: "private-call", + status: "completed", + }, + }); + emitAcpLifecycleError({ + runId: "run-audit-only", + error: new Error(secret), + auditOnly: true, + }); + } finally { + stopAudit(); + } + + expect(captured).toEqual([]); + expect(auditEvents.map((event) => [event.stream, event.data.phase])).toEqual([ + ["lifecycle", "start"], + ["lifecycle", "error"], + ]); + expect(capturedTools.map((event) => event.type)).toEqual([ + "tool.execution.started", + "tool.execution.completed", + ]); + expect(JSON.stringify(auditEvents)).not.toContain("private command payload"); + expect(JSON.stringify(auditEvents)).not.toContain(secret); + }); + + it("emits metadata-only tool lifecycle events without ACP text or title", () => { + const secret = "secret tool payload"; + emitAcpRuntimeEvent({ + runId: "run-tool", + sessionKey: "agent:main:acp:child", + agentId: "main", + event: { + type: "tool_call", + tag: "tool_call", + text: secret, + title: secret, + kind: "read", + toolCallId: "call-1", + status: "in_progress", + }, + }); + emitAcpRuntimeEvent({ + runId: "run-tool", + sessionKey: "agent:main:acp:child", + agentId: "main", + event: { + type: "tool_call", + tag: "tool_call_update", + text: secret, + title: secret, + kind: "read", + toolCallId: "call-1", + status: "completed", + }, + }); + + expect(capturedTools).toMatchObject([ + { + type: "tool.execution.started", + runId: "run-tool", + sessionKey: "agent:main:acp:child", + agentId: "main", + toolCallId: "call-1", + toolName: "acp_read", + }, + { + type: "tool.execution.completed", + runId: "run-tool", + sessionKey: "agent:main:acp:child", + agentId: "main", + toolCallId: "call-1", + toolName: "acp_read", + }, + ]); + expect(JSON.stringify(capturedTools)).not.toContain(secret); + }); + + it.each([ + { status: "completed", type: "tool.execution.completed", terminalReason: undefined }, + { status: "done", type: "tool.execution.completed", terminalReason: undefined }, + { status: "failed", type: "tool.execution.error", terminalReason: "failed" }, + { status: "error", type: "tool.execution.error", terminalReason: "failed" }, + { status: "cancelled", type: "tool.execution.error", terminalReason: "cancelled" }, + ] as const)("finishes audit tracking for terminal tool status $status", (expected) => { + const runId = `run-tool-${expected.status}`; + const toolCallId = `call-${expected.status}`; + emitAcpRuntimeEvent({ + runId, + event: { + type: "tool_call", + tag: "tool_call", + text: "running", + kind: "execute", + toolCallId, + status: "in_progress", + }, + }); + emitAcpRuntimeEvent({ + runId, + event: { + type: "tool_call", + tag: "tool_call_update", + text: expected.status, + kind: "execute", + toolCallId, + status: expected.status, + }, + }); + emitAcpLifecycleEnd({ runId, resultStatus: "completed" }); + + expect(capturedTools).toHaveLength(2); + expect(capturedTools[1]).toMatchObject({ + type: expected.type, + toolCallId, + ...(expected.terminalReason ? { terminalReason: expected.terminalReason } : {}), + ...(expected.status === "cancelled" ? { errorCategory: "aborted" } : {}), + }); + }); + + it("deduplicates repeated terminal tool updates until the ACP run ends", () => { + const params = { runId: "run-tool-replayed-terminal" }; + const toolCall = { + type: "tool_call", + text: "command result", + kind: "execute", + toolCallId: "call-replayed-terminal", + } as const; + emitAcpRuntimeEvent({ + ...params, + event: { ...toolCall, tag: "tool_call", status: "in_progress" }, + }); + emitAcpRuntimeEvent({ + ...params, + event: { ...toolCall, tag: "tool_call_update", status: "completed" }, + }); + emitAcpRuntimeEvent({ + ...params, + event: { ...toolCall, tag: "tool_call_update", status: "completed" }, + }); + emitAcpRuntimeEvent({ + ...params, + event: { ...toolCall, tag: "tool_call_update", status: "in_progress" }, + }); + + expect(capturedTools.map((event) => event.type)).toEqual([ + "tool.execution.started", + "tool.execution.completed", + ]); + + emitAcpLifecycleEnd({ ...params, resultStatus: "completed" }); + emitAcpRuntimeEvent({ + ...params, + event: { ...toolCall, tag: "tool_call_update", status: "completed" }, + }); + expect(capturedTools.slice(2).map((event) => event.type)).toEqual([ + "tool.execution.started", + "tool.execution.completed", + ]); + emitAcpLifecycleEnd({ ...params, resultStatus: "completed" }); + }); + + it("fails closed without evicting terminal identities at the tracking bound", () => { + const params = { runId: "run-tool-tracking-bound" }; + const terminalEvent = (toolCallId: string) => + ({ + type: "tool_call", + tag: "tool_call_update", + text: "done", + kind: "execute", + toolCallId, + status: "completed", + }) as const; + for (let index = 0; index < 4_096; index += 1) { + emitAcpRuntimeEvent({ ...params, event: terminalEvent(`call-${index}`) }); + } + expect(capturedTools).toHaveLength(8_192); + + emitAcpRuntimeEvent({ ...params, event: terminalEvent("call-0") }); + emitAcpRuntimeEvent({ ...params, event: terminalEvent("call-overflow") }); + expect(capturedTools).toHaveLength(8_192); + + const unrelatedTracker = createAcpToolLifecycleTracker(); + emitAcpRuntimeEventBase({ + runId: "run-tool-unrelated", + toolTracker: unrelatedTracker, + event: terminalEvent("call-unrelated"), + }); + expect(capturedTools.slice(8_192).map((event) => event.type)).toEqual([ + "tool.execution.started", + "tool.execution.completed", + ]); + emitAcpLifecycleEndBase({ + runId: "run-tool-unrelated", + toolTracker: unrelatedTracker, + resultStatus: "completed", + }); + + emitAcpLifecycleEnd({ ...params, resultStatus: "completed" }); + emitAcpRuntimeEvent({ ...params, event: terminalEvent("call-overflow") }); + expect(capturedTools.slice(8_194).map((event) => event.type)).toEqual([ + "tool.execution.started", + "tool.execution.completed", + ]); + emitAcpLifecycleEnd({ ...params, resultStatus: "completed" }); + }); + + it("treats blank tool call ids as unidentified", () => { + const params = { runId: "run-tool-blank-id" }; + const event = { + type: "tool_call", + text: "result", + kind: "execute", + toolCallId: " ", + } as const; + emitAcpRuntimeEvent({ + ...params, + event: { ...event, tag: "tool_call", status: "in_progress" }, + }); + expect(capturedTools).toEqual([]); + + emitAcpRuntimeEvent({ + ...params, + event: { ...event, tag: "tool_call_update", status: "completed" }, + }); + expect(capturedTools.map(({ type, toolCallId }) => ({ type, toolCallId }))).toEqual([ + { type: "tool.execution.started", toolCallId: undefined }, + { type: "tool.execution.completed", toolCallId: undefined }, + ]); + emitAcpLifecycleEnd({ ...params, resultStatus: "completed" }); + }); + + it("waits for terminal evidence before recording a tool without a call id", () => { + const params = { runId: "run-tool-without-id" }; + emitAcpRuntimeEvent({ + ...params, + event: { + type: "tool_call", + tag: "tool_call", + text: "running", + kind: "execute", + status: "in_progress", + }, + }); + expect(capturedTools).toEqual([]); + emitAcpRuntimeEvent({ + ...params, + event: { + type: "tool_call", + tag: "tool_call_update", + text: "still running", + kind: "execute", + status: "in_progress", + }, + }); + expect(capturedTools).toEqual([]); + emitAcpRuntimeEvent({ + ...params, + event: { + type: "tool_call", + tag: "tool_call_update", + text: "done", + kind: "execute", + status: "completed", + }, + }); + + expect(capturedTools.map((event) => event.type)).toEqual([ + "tool.execution.started", + "tool.execution.completed", + ]); + }); + + it("finishes outstanding tools when the ACP runtime ends", () => { + const params = { + runId: "run-incomplete-tool", + sessionKey: "agent:main:acp:child", + }; + emitAcpRuntimeEvent({ + ...params, + event: { + type: "tool_call", + tag: "tool_call", + text: "running", + kind: "execute", + toolCallId: "call-incomplete", + status: "in_progress", + }, + }); + emitAcpRuntimeEvent({ + ...params, + event: { type: "done", status: "completed", stopReason: "end_turn" }, + }); + emitAcpLifecycleEnd({ + ...params, + resultStatus: "completed", + stopReason: "end_turn", + }); + + expect(capturedTools).toMatchObject([ + { + type: "tool.execution.started", + toolCallId: "call-incomplete", + }, + { + type: "tool.execution.error", + toolCallId: "call-incomplete", + errorCategory: "acp_tool_incomplete", + }, + ]); + + emitAcpRuntimeEvent({ + ...params, + event: { + type: "tool_call", + tag: "tool_call", + text: "retry", + kind: "execute", + toolCallId: "call-incomplete", + status: "completed", + }, + }); + expect(capturedTools.slice(2)).toMatchObject([ + { type: "tool.execution.started", toolCallId: "call-incomplete" }, + { type: "tool.execution.completed", toolCallId: "call-incomplete" }, + ]); + }); + + it("cancels outstanding tools when the ACP result reports cancellation", () => { + const params = { + runId: "run-cancelled-tool", + sessionKey: "agent:main:acp:child", + }; + emitAcpRuntimeEvent({ + ...params, + event: { + type: "tool_call", + tag: "tool_call", + text: "running", + kind: "execute", + toolCallId: "call-cancelled", + status: "in_progress", + }, + }); + emitAcpRuntimeEvent({ + ...params, + event: { type: "done", status: "cancelled" }, + }); + emitAcpLifecycleEnd({ + ...params, + resultStatus: "cancelled", + }); + + expect(capturedTools).toMatchObject([ + { type: "tool.execution.started", toolCallId: "call-cancelled" }, + { + type: "tool.execution.error", + toolCallId: "call-cancelled", + errorCategory: "aborted", + terminalReason: "cancelled", + }, + ]); + + expect(captured.at(-1)?.data).toMatchObject({ + phase: "end", + aborted: true, + stopReason: "stop", + status: "cancelled", + }); + }); + + it("times out outstanding tools when the enclosing ACP run times out", () => { + const abortController = new AbortController(); + const params = { + runId: "run-timeout-tool", + sessionKey: "agent:main:acp:child", + abortSignal: abortController.signal, + }; + emitAcpRuntimeEvent({ + ...params, + event: { + type: "tool_call", + tag: "tool_call", + text: "running", + kind: "execute", + toolCallId: "call-timeout", + status: "in_progress", + }, + }); + abortController.abort(Object.assign(new Error("timed out"), { name: "TimeoutError" })); + emitAcpRuntimeEvent({ + ...params, + event: { type: "done", status: "cancelled", stopReason: "timeout" }, + }); + emitAcpLifecycleEnd({ + ...params, + resultStatus: "cancelled", + stopReason: "timeout", + }); + + expect(capturedTools).toMatchObject([ + { type: "tool.execution.started", toolCallId: "call-timeout" }, + { + type: "tool.execution.error", + toolCallId: "call-timeout", + terminalReason: "timed_out", + }, + ]); + }); + + it("preserves manager-owned ACP timeout attribution without an aborted caller signal", () => { + emitAcpRuntimeEvent({ + runId: "run-manager-timeout", + event: { + type: "tool_call", + tag: "tool_call", + text: "running", + kind: "execute", + toolCallId: "call-manager-timeout", + status: "in_progress", + }, + }); + + emitAcpLifecycleError({ + runId: "run-manager-timeout", + error: new AcpRuntimeError("ACP_TURN_FAILED", "ACP turn timed out.", { + detailCode: ACP_TURN_TIMEOUT_DETAIL_CODE, + }), + }); + + expect(capturedTools.at(-1)).toMatchObject({ + type: "tool.execution.error", + toolCallId: "call-manager-timeout", + terminalReason: "timed_out", + }); + expect(captured.at(-1)?.data).toMatchObject({ + phase: "error", + aborted: true, + stopReason: "timeout", + status: "timed_out", + }); + }); }); afterEach(() => { unsubscribe?.(); unsubscribe = undefined; + unsubscribeTools?.(); + unsubscribeTools = undefined; resetAgentEventsForTest(); + resetDiagnosticEventsForTest(); }); describe("emitAcpLifecycleError preserves AcpRuntimeError detail (regression: openclaw-4a8)", () => { diff --git a/src/agents/command/attempt-execution.runtime.ts b/src/agents/command/attempt-execution.runtime.ts index 01f8452c964d..70a77e77a7b8 100644 --- a/src/agents/command/attempt-execution.runtime.ts +++ b/src/agents/command/attempt-execution.runtime.ts @@ -2,6 +2,7 @@ // light shared helpers without pulling the full command attempt graph. export { buildAcpResult, + createAcpToolLifecycleTracker, createAcpVisibleTextAccumulator, emitAcpAssistantDelta, emitAcpLifecycleEnd, @@ -14,3 +15,4 @@ export { runAgentAttempt, sessionFileHasContent, } from "./attempt-execution.js"; +export type { AcpToolLifecycleTracker } from "./attempt-execution.js"; diff --git a/src/agents/command/attempt-execution.ts b/src/agents/command/attempt-execution.ts index c2875d314975..704d22924a23 100644 --- a/src/agents/command/attempt-execution.ts +++ b/src/agents/command/attempt-execution.ts @@ -2,9 +2,14 @@ * Orchestrates one agent attempt across embedded, CLI, and ACP runtimes. */ import type { AcpRuntimeEvent } from "@openclaw/acp-core/runtime/types"; -import type { FastMode } from "@openclaw/normalization-core/string-coerce"; +import { + normalizeOptionalLowercaseString, + type FastMode, +} from "@openclaw/normalization-core/string-coerce"; import { sanitizeForLog } from "../../../packages/terminal-core/src/ansi.js"; +import { ACP_TURN_TIMEOUT_DETAIL_CODE } from "../../acp/control-plane/manager.turn-timeout.js"; import { formatAcpErrorChain } from "../../acp/runtime/errors.js"; +import { resolveAcpToolTerminalOutcome } from "../../acp/tool-status.js"; import { normalizeReplyPayload } from "../../auto-reply/reply/normalize-reply.js"; import type { ThinkLevel, VerboseLevel } from "../../auto-reply/thinking.js"; import { persistSessionTranscriptTurn } from "../../config/sessions/session-accessor.js"; @@ -15,7 +20,8 @@ import { injectTimestamp, timestampOptsFromConfig, } from "../../gateway/server-methods/agent-timestamp.js"; -import { emitAgentEvent } from "../../infra/agent-events.js"; +import { emitAgentAuditEvent, emitAgentEvent } from "../../infra/agent-events.js"; +import { emitTrustedDiagnosticEvent } from "../../infra/diagnostic-events.js"; import { readErrorName } from "../../infra/errors.js"; import { redactSensitiveText } from "../../logging/redact.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; @@ -872,6 +878,7 @@ export function buildAcpResult(params: { payloadText: string; startedAt: number; stopReason?: string; + resultStatus?: Extract["status"]; abortSignal?: AbortSignal; }) { const normalizedFinalPayload = normalizeReplyPayload({ @@ -879,12 +886,13 @@ export function buildAcpResult(params: { }); const payloads = normalizedFinalPayload ? [normalizedFinalPayload] : []; const abortFields = resolveAgentRunAbortLifecycleFields(params.abortSignal); + const resultCancelled = params.resultStatus === "cancelled"; return { payloads, meta: { durationMs: Date.now() - params.startedAt, - aborted: abortFields.aborted ?? false, - stopReason: abortFields.stopReason ?? params.stopReason, + aborted: abortFields.aborted ?? resultCancelled, + stopReason: abortFields.stopReason ?? (resultCancelled ? "stop" : params.stopReason), }, }; } @@ -892,10 +900,16 @@ export function buildAcpResult(params: { export function emitAcpLifecycleStart(params: { runId: string; startedAt: number; + sessionKey?: string; + agentId?: string; lifecycleGeneration?: string; + auditOnly?: boolean; }) { - emitAgentEvent({ + const emit = params.auditOnly ? emitAgentAuditEvent : emitAgentEvent; + emit({ runId: params.runId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), ...(params.lifecycleGeneration ? { lifecycleGeneration: params.lifecycleGeneration } : {}), stream: "lifecycle", data: { @@ -913,6 +927,228 @@ const ACP_PROXY_ENV_KEYS = [ "https_proxy", "all_proxy", ] as const; +type ActiveAcpTool = { + runId: string; + sessionKey?: string; + agentId?: string; + toolCallId: string; + toolName: string; + startedAt: number; +}; + +export type AcpToolLifecycleTracker = { + active: Map; + terminalToolCallIds: Set; + saturated: boolean; +}; + +const MAX_TRACKED_ACP_TOOLS = 4_096; + +export function createAcpToolLifecycleTracker(): AcpToolLifecycleTracker { + return { + active: new Map(), + terminalToolCallIds: new Set(), + saturated: false, + }; +} + +function acpAuditToolName(kind: unknown): string { + switch (kind) { + case "read": + case "edit": + case "delete": + case "move": + case "search": + case "execute": + case "fetch": + case "switch_mode": + case "think": + case "other": + return `acp_${kind}`; + default: + return "acp_tool"; + } +} + +function resolveAcpToolTerminalReason( + signal: AbortSignal | undefined, + stopReason?: string, + error?: unknown, + resultStatus?: Extract["status"], +): "failed" | "cancelled" | "timed_out" { + const abortFields = resolveAgentRunAbortLifecycleFields(signal); + if (abortFields.aborted) { + return abortFields.stopReason === "timeout" ? "timed_out" : "cancelled"; + } + const normalizedStopReason = normalizeOptionalLowercaseString(stopReason); + if (normalizedStopReason === "timeout") { + return "timed_out"; + } + if (resultStatus === "cancelled") { + return "cancelled"; + } + if ( + error instanceof Error && + (error as Error & { detailCode?: unknown }).detailCode === ACP_TURN_TIMEOUT_DETAIL_CODE + ) { + return "timed_out"; + } + if ( + normalizedStopReason === "cancel" || + normalizedStopReason === "cancelled" || + normalizedStopReason === "manual-cancel" + ) { + return "cancelled"; + } + return "failed"; +} + +function resolveAcpLifecycleEndFields( + signal: AbortSignal | undefined, + stopReason?: string, + resultStatus?: Extract["status"], +) { + const abortFields = resolveAgentRunAbortLifecycleFields(signal); + if (abortFields.aborted) { + return abortFields; + } + const terminalReason = resolveAcpToolTerminalReason( + undefined, + stopReason, + undefined, + resultStatus, + ); + if (terminalReason === "timed_out") { + return { aborted: true, stopReason: "timeout", status: "timed_out" } as const; + } + if (terminalReason === "cancelled") { + return { aborted: true, stopReason: "stop", status: "cancelled" } as const; + } + return {}; +} + +function emitAcpToolExecutionEvent(params: { + runId: string; + toolTracker: AcpToolLifecycleTracker; + sessionKey?: string; + agentId?: string; + abortSignal?: AbortSignal; + event: Extract; +}): void { + const { event } = params; + const now = Date.now(); + const toolCallId = event.toolCallId?.trim() ? event.toolCallId : undefined; + const activeTool = toolCallId ? params.toolTracker.active.get(toolCallId) : undefined; + const terminalOutcome = resolveAcpToolTerminalOutcome(event.status); + const toolName = acpAuditToolName(event.kind); + // ACP runtimes may replay terminal updates. Keep the closed identity until the run ends so a + // late progress/terminal pair cannot reopen one invocation as a second durable audit action. + if (toolCallId && !activeTool) { + if (params.toolTracker.terminalToolCallIds.has(toolCallId)) { + return; + } + // Never evict an open identity: once this run reaches its bound, ignore new identities until + // lifecycle cleanup releases the complete set. Other runs own independent trackers. + const trackedIdentities = + params.toolTracker.active.size + params.toolTracker.terminalToolCallIds.size; + if (params.toolTracker.saturated || trackedIdentities >= MAX_TRACKED_ACP_TOOLS) { + params.toolTracker.saturated = true; + return; + } + } + // Without an identity, wait for a terminal event so every observed action closes immediately. + // Opening on progress would leave an unmatched audit action if the runtime omits its result. + const startsUnidentifiedTool = toolCallId === undefined && terminalOutcome !== undefined; + if (!activeTool && (toolCallId !== undefined || startsUnidentifiedTool)) { + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId: params.runId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(toolCallId ? { toolCallId } : {}), + toolName, + toolSource: "core", + toolOwner: "acp", + }); + if (toolCallId) { + params.toolTracker.active.set(toolCallId, { + runId: params.runId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), + toolCallId, + toolName, + startedAt: now, + }); + } + } + if (!terminalOutcome) { + return; + } + const terminalReason = resolveAcpToolTerminalReason( + params.abortSignal, + undefined, + undefined, + terminalOutcome === "cancelled" ? "cancelled" : undefined, + ); + const durationMs = Math.max(0, now - (activeTool?.startedAt ?? now)); + emitTrustedDiagnosticEvent( + terminalOutcome === "completed" + ? { + type: "tool.execution.completed", + runId: params.runId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(toolCallId ? { toolCallId } : {}), + toolName: activeTool?.toolName ?? toolName, + toolSource: "core", + toolOwner: "acp", + durationMs, + } + : { + type: "tool.execution.error", + runId: params.runId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(toolCallId ? { toolCallId } : {}), + toolName: activeTool?.toolName ?? toolName, + toolSource: "core", + toolOwner: "acp", + durationMs, + errorCategory: terminalReason === "cancelled" ? "aborted" : "acp_tool", + terminalReason, + }, + ); + if (toolCallId) { + params.toolTracker.active.delete(toolCallId); + params.toolTracker.terminalToolCallIds.add(toolCallId); + } +} + +function finalizeAcpToolsForRun( + toolTracker: AcpToolLifecycleTracker, + runId: string, + terminalReason: "failed" | "cancelled" | "timed_out", +): void { + const now = Date.now(); + for (const activeTool of toolTracker.active.values()) { + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + runId, + ...(activeTool.sessionKey ? { sessionKey: activeTool.sessionKey } : {}), + ...(activeTool.agentId ? { agentId: activeTool.agentId } : {}), + toolName: activeTool.toolName, + toolSource: "core", + toolOwner: "acp", + toolCallId: activeTool.toolCallId, + durationMs: Math.max(0, now - activeTool.startedAt), + errorCategory: terminalReason === "cancelled" ? "aborted" : "acp_tool_incomplete", + terminalReason, + }); + } + toolTracker.active.clear(); + toolTracker.terminalToolCallIds.clear(); + toolTracker.saturated = false; +} function resolvePresentProxyEnvKeys(env: NodeJS.ProcessEnv = process.env): string[] { return ACP_PROXY_ENV_KEYS.filter((key) => { @@ -954,6 +1190,7 @@ function acpRuntimeEventDiagnostics(event: AcpRuntimeEvent): Record["status"]; + auditOnly?: boolean; }) { - emitAgentEvent({ + finalizeAcpToolsForRun( + params.toolTracker, + params.runId, + resolveAcpToolTerminalReason( + params.abortSignal, + params.stopReason, + undefined, + params.resultStatus, + ), + ); + const emit = params.auditOnly ? emitAgentAuditEvent : emitAgentEvent; + emit({ runId: params.runId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), ...(params.lifecycleGeneration ? { lifecycleGeneration: params.lifecycleGeneration } : {}), stream: "lifecycle", data: { phase: "end", endedAt: Date.now(), - ...resolveAgentRunAbortLifecycleFields(params.abortSignal), + ...resolveAcpLifecycleEndFields(params.abortSignal, params.stopReason, params.resultStatus), }, }); } export function emitAcpLifecycleError(params: { runId: string; + toolTracker: AcpToolLifecycleTracker; error: unknown; sessionKey?: string; + agentId?: string; lifecycleGeneration?: string; abortSignal?: AbortSignal; + terminalOutcome?: "blocked"; + auditOnly?: boolean; }) { - emitAgentEvent({ + const terminalReason = resolveAcpToolTerminalReason(params.abortSignal, undefined, params.error); + finalizeAcpToolsForRun(params.toolTracker, params.runId, terminalReason); + const lifecycleFields = + params.terminalOutcome === "blocked" + ? ({ livenessState: "blocked" } as const) + : terminalReason === "timed_out" + ? ({ aborted: true, stopReason: "timeout", status: "timed_out" } as const) + : resolveAgentRunAbortLifecycleFields(params.abortSignal); + const emit = params.auditOnly ? emitAgentAuditEvent : emitAgentEvent; + emit({ runId: params.runId, + ...(params.agentId ? { agentId: params.agentId } : {}), ...(params.lifecycleGeneration ? { lifecycleGeneration: params.lifecycleGeneration } : {}), stream: "lifecycle", ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), data: { phase: "error", - error: formatAcpErrorChain(params.error), + ...(!params.auditOnly ? { error: formatAcpErrorChain(params.error) } : {}), endedAt: Date.now(), - ...resolveAgentRunAbortLifecycleFields(params.abortSignal), + ...lifecycleFields, }, }); } diff --git a/src/agents/embedded-agent-runner/compact.ts b/src/agents/embedded-agent-runner/compact.ts index fb892dd68c9f..bbf590ecc226 100644 --- a/src/agents/embedded-agent-runner/compact.ts +++ b/src/agents/embedded-agent-runner/compact.ts @@ -967,6 +967,7 @@ async function compactEmbeddedAgentSessionDirectOnce( diagnostics: normalizableToolProjection.diagnostics, tools: toolsEnabled ? toolsRaw : [], runId, + agentId: effectiveSkillAgentId, sessionKey: params.sessionKey, sessionId: params.sessionId, }); @@ -1006,6 +1007,7 @@ async function compactEmbeddedAgentSessionDirectOnce( diagnostics: normalizableBundledToolProjection.diagnostics, tools: filteredBundledTools, runId, + agentId: effectiveSkillAgentId, sessionKey: params.sessionKey, sessionId: params.sessionId, }); @@ -1023,6 +1025,7 @@ async function compactEmbeddedAgentSessionDirectOnce( diagnostics: toolSchemaProjection.diagnostics, tools: projectedEffectiveTools, runId, + agentId: effectiveSkillAgentId, sessionKey: params.sessionKey, sessionId: params.sessionId, }); diff --git a/src/agents/embedded-agent-runner/run/attempt.ts b/src/agents/embedded-agent-runner/run/attempt.ts index b6879b3c5a8c..c07cd5fe5d04 100644 --- a/src/agents/embedded-agent-runner/run/attempt.ts +++ b/src/agents/embedded-agent-runner/run/attempt.ts @@ -1533,6 +1533,7 @@ export async function runEmbeddedAttempt( diagnostics, tools: sourceTools, runId: params.runId, + agentId: sessionAgentId, sessionKey: params.sessionKey, sessionId: params.sessionId, }), @@ -1615,6 +1616,7 @@ export async function runEmbeddedAttempt( diagnostics, tools: sourceTools, runId: params.runId, + agentId: sessionAgentId, sessionKey: params.sessionKey, sessionId: params.sessionId, }), @@ -1642,6 +1644,7 @@ export async function runEmbeddedAttempt( diagnostics: uncompactedToolSchemaProjection.diagnostics, tools: projectedUncompactedEffectiveTools, runId: params.runId, + agentId: sessionAgentId, sessionKey: params.sessionKey, sessionId: params.sessionId, }); @@ -1748,6 +1751,7 @@ export async function runEmbeddedAttempt( diagnostics: toolSearchSchemaProjection.diagnostics, tools: projectedToolSearchTools, runId: params.runId, + agentId: sessionAgentId, sessionKey: params.sessionKey, sessionId: params.sessionId, }); diff --git a/src/agents/harness/native-hook-relay.test.ts b/src/agents/harness/native-hook-relay.test.ts index faf0dd0ab27c..6a844f37d3dc 100644 --- a/src/agents/harness/native-hook-relay.test.ts +++ b/src/agents/harness/native-hook-relay.test.ts @@ -1680,6 +1680,163 @@ describe("native hook relay registry", () => { }); }); + it("keeps a native pre-tool hook timeout distinct from a policy denial", async () => { + const onPreToolUseFailure = vi.fn(); + const beforeToolCall = vi.fn(async () => { + throw Object.assign(new Error("timed out after 5000ms"), { name: "TimeoutError" }); + }); + initializeGlobalHookRunner( + createMockPluginRegistry([{ hookName: "before_tool_call", handler: beforeToolCall }]), + ); + const relay = registerNativeHookRelay({ + provider: "codex", + agentId: "agent-1", + sessionId: "session-1", + runId: "run-1", + onPreToolUseFailure, + }); + + const response = await invokeNativeHookRelay({ + provider: "codex", + relayId: relay.relayId, + event: "pre_tool_use", + rawPayload: { + hook_event_name: "PreToolUse", + cwd: "/repo", + tool_name: "exec_command", + tool_use_id: "native-timeout-1", + tool_input: { cmd: "pnpm test" }, + }, + }); + + expect(response.failureDisposition).toBe("timed_out"); + expect(JSON.parse(response.stdout)).toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + expect(onPreToolUseFailure).toHaveBeenCalledWith({ + toolName: "exec", + toolCallId: "native-timeout-1", + disposition: "timed_out", + durationMs: expect.any(Number), + }); + + await invokeNativeHookRelay({ + provider: "codex", + relayId: relay.relayId, + event: "pre_tool_use", + rawPayload: { + hook_event_name: "PreToolUse", + tool_name: "exec_command", + tool_use_id: "native-timeout-1", + tool_input: { cmd: "pnpm test" }, + }, + }); + expect(onPreToolUseFailure).toHaveBeenCalledTimes(1); + }); + + it("isolates an asynchronously rejected native failure projection", async () => { + const onPreToolUseFailure = vi.fn(async () => { + throw new Error("diagnostic sink unavailable"); + }); + const beforeToolCall = vi.fn(async () => { + throw new Error("hook crashed"); + }); + initializeGlobalHookRunner( + createMockPluginRegistry([{ hookName: "before_tool_call", handler: beforeToolCall }]), + ); + const relay = registerNativeHookRelay({ + provider: "codex", + sessionId: "session-1", + runId: "run-1", + onPreToolUseFailure, + }); + + await expect( + invokeNativeHookRelay({ + provider: "codex", + relayId: relay.relayId, + event: "pre_tool_use", + rawPayload: { + hook_event_name: "PreToolUse", + tool_name: "exec_command", + tool_use_id: "native-failed-projection", + tool_input: { cmd: "pnpm test" }, + }, + }), + ).resolves.toMatchObject({ failureDisposition: "failed" }); + expect(onPreToolUseFailure).toHaveBeenCalledTimes(1); + }); + + it("does not delay a native hook response on a pending failure projection", async () => { + const onPreToolUseFailure = vi.fn( + () => + new Promise(() => { + // Deliberately remain pending to prove projection does not block the response. + }), + ); + const beforeToolCall = vi.fn(async () => { + throw new Error("hook crashed"); + }); + initializeGlobalHookRunner( + createMockPluginRegistry([{ hookName: "before_tool_call", handler: beforeToolCall }]), + ); + const relay = registerNativeHookRelay({ + provider: "codex", + sessionId: "session-1", + runId: "run-1", + onPreToolUseFailure, + }); + const invoke = () => + invokeNativeHookRelay({ + provider: "codex", + relayId: relay.relayId, + event: "pre_tool_use", + rawPayload: { + hook_event_name: "PreToolUse", + tool_name: "exec_command", + tool_use_id: "native-pending-projection", + tool_input: { cmd: "pnpm test" }, + }, + }); + + await expect(invoke()).resolves.toMatchObject({ failureDisposition: "failed" }); + await expect(invoke()).resolves.toMatchObject({ failureDisposition: "failed" }); + expect(onPreToolUseFailure).toHaveBeenCalledTimes(1); + }); + + it("leaves report-mode pre-tool failure projection to the approval owner", async () => { + const onPreToolUseFailure = vi.fn(); + const beforeToolCall = vi.fn(async () => { + throw Object.assign(new Error("timed out after 5000ms"), { name: "TimeoutError" }); + }); + initializeGlobalHookRunner( + createMockPluginRegistry([{ hookName: "before_tool_call", handler: beforeToolCall }]), + ); + const relay = registerNativeHookRelay({ + provider: "codex", + agentId: "agent-1", + sessionId: "session-1", + runId: "run-1", + onPreToolUseFailure, + }); + + const response = await invokeNativeHookRelay({ + provider: "codex", + relayId: relay.relayId, + event: "pre_tool_use", + rawPayload: { + hook_event_name: "PreToolUse", + openclaw_approval_mode: "report", + tool_name: "exec_command", + tool_use_id: "native-report-timeout", + tool_input: { cmd: "pnpm test" }, + }, + }); + + expect(response.failureDisposition).toBe("timed_out"); + expect(onPreToolUseFailure).not.toHaveBeenCalled(); + }); + it("normalizes Codex exec_command cmd input before running OpenClaw policy", async () => { const beforeToolCall = vi.fn(async () => ({ block: true, @@ -2059,6 +2216,57 @@ describe("native hook relay registry", () => { ).resolves.toBeUndefined(); }); + it("preserves deferred native approval cancellation as a terminal disposition", async () => { + const beforeToolCall = vi.fn(async () => ({ + requireApproval: { + title: "Needs approval", + description: "native command needs approval", + }, + })); + initializeGlobalHookRunner( + createMockPluginRegistry([{ hookName: "before_tool_call", handler: beforeToolCall }]), + ); + const relay = registerNativeHookRelay({ + provider: "codex", + agentId: "agent-1", + sessionId: "session-1", + runId: "run-1", + }); + + await invokeNativeHookRelay({ + provider: "codex", + relayId: relay.relayId, + event: "pre_tool_use", + rawPayload: { + hook_event_name: "PreToolUse", + openclaw_approval_mode: "report", + cwd: "/repo", + tool_name: "exec_command", + tool_use_id: "native-approval-cancelled", + tool_input: { cmd: "pnpm test" }, + }, + }); + testing.setNativeHookRelayDeferredToolApprovalRequesterForTests(async () => ({ + blocked: true, + kind: "failure", + disposition: "cancelled", + deniedReason: "plugin-approval", + reason: "Approval cancelled because the run stopped", + })); + + await expect( + resolveNativeHookRelayDeferredToolApproval({ + relayId: relay.relayId, + toolUseId: "native-approval-cancelled", + }), + ).resolves.toEqual({ + handled: true, + outcome: "denied", + reason: "Approval cancelled because the run stopped", + failureDisposition: "cancelled", + }); + }); + it("passes config to trusted policies for native pre-tool session extension reads", async () => { const stateDir = await fs.mkdtemp(path.join(tmpdir(), "openclaw-native-relay-policy-")); const storePath = path.join(stateDir, "sessions.json"); diff --git a/src/agents/harness/native-hook-relay.ts b/src/agents/harness/native-hook-relay.ts index ef118ece4178..cf08e77f1121 100644 --- a/src/agents/harness/native-hook-relay.ts +++ b/src/agents/harness/native-hook-relay.ts @@ -36,6 +36,7 @@ import { hasBeforeToolCallPolicy, requestDeferredPluginToolApproval, runBeforeToolCallHook, + type BeforeToolCallFailureDisposition, type DeferredPluginToolApproval, } from "../agent-tools.before-tool-call.js"; import { stableStringify } from "../stable-stringify.js"; @@ -91,6 +92,7 @@ export type NativeHookRelayProcessResponse = { stdout: string; stderr: string; exitCode: number; + failureDisposition?: Exclude; }; export type NativeHookRelayRegistration = { @@ -107,6 +109,12 @@ export type NativeHookRelayRegistration = { allowedEvents: readonly NativeHookRelayEvent[]; expiresAtMs: number; signal?: AbortSignal; + onPreToolUseFailure?: (failure: { + toolName: string; + toolCallId: string; + disposition: Exclude; + durationMs: number; + }) => void | Promise; }; export type NativeHookRelayRegistrationHandle = NativeHookRelayRegistration & { @@ -135,6 +143,7 @@ export type RegisterNativeHookRelayParams = { ttlMs?: number; command?: NativeHookRelayCommandOptions; signal?: AbortSignal; + onPreToolUseFailure?: NativeHookRelayRegistration["onPreToolUseFailure"]; }; export type NativeHookRelayCommandOptions = { @@ -183,7 +192,10 @@ type NativeHookRelayProviderAdapter = { readToolInput: (rawPayload: JsonValue) => Record; readToolResponse: (rawPayload: JsonValue) => unknown; renderNoopResponse: (event: NativeHookRelayEvent) => NativeHookRelayProcessResponse; - renderPreToolUseBlockResponse: (reason: string) => NativeHookRelayProcessResponse; + renderPreToolUseBlockResponse: ( + reason: string, + failureDisposition?: Exclude, + ) => NativeHookRelayProcessResponse; renderBeforeAgentFinalizeReviseResponse: (reason: string) => NativeHookRelayProcessResponse; renderBeforeAgentFinalizeStopResponse: (reason?: string) => NativeHookRelayProcessResponse; renderPermissionDecisionResponse: ( @@ -245,6 +257,7 @@ type NativeHookRelaySharedState = { type ActiveNativeHookRelayRegistration = NativeHookRelayRegistration & { generation: string; + preToolUseFailureProjections: Map; settled: boolean }>; }; type ActiveNativeHookRelayRegistrationHandle = NativeHookRelayRegistrationHandle & { @@ -313,6 +326,7 @@ export type NativeHookRelayDeferredApprovalOutcome = handled: true; outcome: "denied"; reason: string; + failureDisposition?: Exclude; }; type NativeHookRelayBridgeRegistration = { @@ -361,7 +375,7 @@ const nativeHookRelayProviderAdapters: Record< // Codex treats empty stdout plus exit 0 as no decision/no additional context. return { stdout: "", stderr: "", exitCode: 0 }; }, - renderPreToolUseBlockResponse: (reason) => ({ + renderPreToolUseBlockResponse: (reason, failureDisposition) => ({ stdout: `${JSON.stringify({ hookSpecificOutput: { hookEventName: "PreToolUse", @@ -371,6 +385,7 @@ const nativeHookRelayProviderAdapters: Record< })}\n`, stderr: "", exitCode: 0, + ...(failureDisposition ? { failureDisposition } : {}), }), renderBeforeAgentFinalizeReviseResponse: (reason) => ({ stdout: `${JSON.stringify({ @@ -437,7 +452,9 @@ export function registerNativeHookRelay( ...(params.channelId ? { channelId: params.channelId } : {}), allowedEvents, expiresAtMs, + preToolUseFailureProjections: new Map(), ...(params.signal ? { signal: params.signal } : {}), + ...(params.onPreToolUseFailure ? { onPreToolUseFailure: params.onPreToolUseFailure } : {}), }; relays.set(relayId, registration); registerNativeHookRelayBridge(registration); @@ -656,11 +673,72 @@ export async function invokeNativeHookRelay( rawPayload: params.rawPayload, }); recordNativeHookRelayInvocation(normalized); - return processNativeHookRelayInvocation({ + const startedAt = Date.now(); + const response = await processNativeHookRelayInvocation({ registration, invocation: normalized, adapter: getNativeHookRelayProviderAdapter(provider), }); + if ( + normalized.toolUseId && + response.failureDisposition && + readNativeHookRelayApprovalMode(normalized.rawPayload) !== "report" + ) { + projectNativeHookRelayPreToolUseFailure(registration, { + toolName: normalizeNativeHookToolName(normalized.toolName), + toolCallId: normalized.toolUseId, + disposition: response.failureDisposition, + durationMs: Date.now() - startedAt, + }); + } + return response; +} + +function projectNativeHookRelayPreToolUseFailure( + registration: ActiveNativeHookRelayRegistration, + failure: Parameters>[0], +): void { + const callback = registration.onPreToolUseFailure; + if (!callback) { + return; + } + if (registration.preToolUseFailureProjections.has(failure.toolCallId)) { + return; + } + const record = { + promise: Promise.resolve().then(() => callback(failure)), + settled: false, + }; + registration.preToolUseFailureProjections.set(failure.toolCallId, record); + void record.promise.then( + () => { + record.settled = true; + }, + (error: unknown) => { + record.settled = true; + if (registration.preToolUseFailureProjections.get(failure.toolCallId) === record) { + registration.preToolUseFailureProjections.delete(failure.toolCallId); + } + log.debug("native pre-tool failure projection failed", { + error, + relayId: registration.relayId, + toolCallId: failure.toolCallId, + }); + }, + ); + if (registration.preToolUseFailureProjections.size > MAX_NATIVE_HOOK_RELAY_INVOCATIONS) { + let oldestToolCallId: string | undefined; + for (const [toolCallId, candidate] of registration.preToolUseFailureProjections) { + oldestToolCallId ??= toolCallId; + if (candidate.settled) { + registration.preToolUseFailureProjections.delete(toolCallId); + return; + } + } + if (oldestToolCallId) { + registration.preToolUseFailureProjections.delete(oldestToolCallId); + } + } } export function hasNativeHookRelayInvocation(params: { @@ -716,7 +794,14 @@ async function resolveNativeHookRelayPreToolUseApproval( signal, }); if (outcome.blocked) { - return { handled: true, outcome: "denied", reason: outcome.reason }; + return { + handled: true, + outcome: "denied", + reason: outcome.reason, + ...(outcome.kind === "failure" && outcome.disposition !== "blocked" + ? { failureDisposition: outcome.disposition } + : {}), + }; } if ( nativeHookRelayParamsWereRewritten(pendingApproval.originalParamsFingerprint, outcome.params) @@ -1407,7 +1492,12 @@ async function runNativeHookRelayPreToolUse(params: { }, }); if (outcome.blocked) { - return params.adapter.renderPreToolUseBlockResponse(outcome.reason); + return params.adapter.renderPreToolUseBlockResponse( + outcome.reason, + outcome.kind === "failure" && outcome.disposition !== "blocked" + ? outcome.disposition + : undefined, + ); } if (outcome.deferredApproval) { if ( diff --git a/src/agents/model-fallback.test.ts b/src/agents/model-fallback.test.ts index a607d0bc7d3c..ef52f7410d97 100644 --- a/src/agents/model-fallback.test.ts +++ b/src/agents/model-fallback.test.ts @@ -34,6 +34,7 @@ import { import { createAgentRunDirectAbortError, createAgentRunRestartAbortError, + resolveAgentRunErrorLifecycleFields, } from "./run-termination.js"; import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js"; import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js"; @@ -1473,6 +1474,40 @@ describe("runWithModelFallback", () => { expect(error.attempts[0]?.error).not.toBe(rawError); }); + it("preserves structured timeout attribution after fallback exhaustion", async () => { + const cfg = makeCfg({ + agents: { + defaults: { + model: { + primary: "anthropic/claude-opus-4-7", + fallbacks: ["google/gemini-3-pro-preview"], + }, + }, + }, + }); + const run = vi.fn().mockRejectedValue( + new FailoverError("CLI produced no output", { + reason: "timeout", + }), + ); + const error = requireFallbackSummaryError( + await captureRejection( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-opus-4-7", + run, + }), + ), + ); + + expect(run).toHaveBeenCalledTimes(2); + expect(resolveAgentRunErrorLifecycleFields(error, undefined)).toEqual({ + stopReason: "timeout", + timeoutPhase: "provider", + }); + }); + it("carries request attribution through exhausted fallback summaries", async () => { const cfg = makeCfg({ agents: { diff --git a/src/agents/run-termination.test.ts b/src/agents/run-termination.test.ts index 824748552b9e..0b2fe523a293 100644 --- a/src/agents/run-termination.test.ts +++ b/src/agents/run-termination.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it } from "vitest"; +import { FailoverError } from "./failover-error.js"; import { createAgentRunDirectAbortError, createAgentRunRestartAbortError, isAgentRunDirectAbortReason, isAbortedAgentStopReason, resolveAgentRunAbortLifecycleFields, + resolveAgentRunErrorLifecycleFields, } from "./run-termination.js"; describe("resolveAgentRunAbortLifecycleFields", () => { @@ -40,6 +42,33 @@ describe("resolveAgentRunAbortLifecycleFields", () => { }); }); + it("contains hostile abort reasons", () => { + const controller = new AbortController(); + const reason = Object.defineProperty({}, "name", { + get() { + throw new Error("hostile name"); + }, + }); + controller.abort(reason); + + expect(resolveAgentRunAbortLifecycleFields(controller.signal)).toEqual({ + aborted: true, + stopReason: "aborted", + }); + }); + + it("contains revoked abort reason proxies", () => { + const controller = new AbortController(); + const { proxy, revoke } = Proxy.revocable({}, {}); + controller.abort(proxy); + revoke(); + + expect(resolveAgentRunAbortLifecycleFields(controller.signal)).toEqual({ + aborted: true, + stopReason: "aborted", + }); + }); + it("treats restart as an aborted terminal reason", () => { expect(isAbortedAgentStopReason("aborted")).toBe(true); expect(isAbortedAgentStopReason("restart")).toBe(true); @@ -57,3 +86,67 @@ describe("resolveAgentRunAbortLifecycleFields", () => { expect(isAgentRunDirectAbortReason(createAgentRunRestartAbortError())).toBe(false); }); }); + +describe("resolveAgentRunErrorLifecycleFields", () => { + it("attributes structured provider watchdog timeouts", () => { + const error = new FailoverError("CLI timed out", { reason: "timeout" }); + + expect(resolveAgentRunErrorLifecycleFields(error, undefined)).toEqual({ + stopReason: "timeout", + timeoutPhase: "provider", + }); + }); + + it("does not reclassify ordinary provider failures", () => { + const error = new FailoverError("CLI failed", { reason: "server_error" }); + + expect(resolveAgentRunErrorLifecycleFields(error, undefined)).toEqual({}); + }); + + it("reads the final structured timeout from a fallback summary cause", () => { + const timeout = new FailoverError("CLI timed out", { reason: "timeout" }); + const error = new Error("All model fallback candidates failed", { cause: timeout }); + + expect(resolveAgentRunErrorLifecycleFields(error, undefined)).toEqual({ + stopReason: "timeout", + timeoutPhase: "provider", + }); + }); + + it("contains throwing cause accessors", () => { + const error = Object.defineProperty(new Error("provider failed"), "cause", { + get() { + throw new Error("hostile cause"); + }, + }); + + expect(resolveAgentRunErrorLifecycleFields(error, undefined)).toEqual({}); + }); + + it("contains hostile failover fields", () => { + const hostileName = Object.defineProperty({}, "name", { + get() { + throw new Error("hostile name"); + }, + }); + const hostileReason = Object.defineProperty({ name: "FailoverError" }, "reason", { + get() { + throw new Error("hostile reason"); + }, + }); + + expect(resolveAgentRunErrorLifecycleFields(hostileName, undefined)).toEqual({}); + expect(resolveAgentRunErrorLifecycleFields(hostileReason, undefined)).toEqual({}); + }); + + it("preserves explicit cancellation over a concurrent timeout error", () => { + const controller = new AbortController(); + controller.abort(); + const error = new FailoverError("CLI timed out", { reason: "timeout" }); + + expect(resolveAgentRunErrorLifecycleFields(error, controller.signal)).toEqual({ + aborted: true, + stopReason: "aborted", + }); + }); +}); diff --git a/src/agents/run-termination.ts b/src/agents/run-termination.ts index c5b363a79583..453ec618565a 100644 --- a/src/agents/run-termination.ts +++ b/src/agents/run-termination.ts @@ -1,3 +1,6 @@ +import { isFailoverError } from "./failover-error.js"; +import type { AgentRunTimeoutPhase } from "./run-timeout-attribution.js"; + /** * Shared agent run termination constants. * @@ -34,9 +37,24 @@ export function createAgentRunRestartAbortError(): Error { } export function isAgentRunRestartAbortReason(value: unknown): boolean { - return ( - value instanceof Error && "code" in value && value.code === AGENT_RUN_RESTART_ABORT_ERROR_CODE - ); + try { + return ( + value instanceof Error && "code" in value && value.code === AGENT_RUN_RESTART_ABORT_ERROR_CODE + ); + } catch { + return false; + } +} + +function isAgentRunTimeoutAbortReason(value: unknown): boolean { + if (!value || typeof value !== "object") { + return false; + } + try { + return "name" in value && value.name === "TimeoutError"; + } catch { + return false; + } } export function resolveAgentRunAbortLifecycleFields(signal: AbortSignal | undefined): { @@ -51,10 +69,7 @@ export function resolveAgentRunAbortLifecycleFields(signal: AbortSignal | undefi } const stopReason = isAgentRunRestartAbortReason(signal.reason) ? AGENT_RUN_RESTART_ABORT_STOP_REASON - : signal.reason && - typeof signal.reason === "object" && - "name" in signal.reason && - signal.reason.name === "TimeoutError" + : isAgentRunTimeoutAbortReason(signal.reason) ? "timeout" : AGENT_RUN_ABORTED_STOP_REASON; return { @@ -63,6 +78,46 @@ export function resolveAgentRunAbortLifecycleFields(signal: AbortSignal | undefi }; } +function isProviderTimeoutError(error: unknown): boolean { + try { + const candidate = isFailoverError(error) + ? error + : error instanceof Error + ? error.cause + : undefined; + return isFailoverError(candidate) && candidate.reason === "timeout"; + } catch { + // Provider/runtime errors may expose hostile getters. Classification must + // not replace the original failure or suppress its terminal event. + return false; + } +} + +/** Preserve structured provider watchdog timeouts when no abort signal was raised. */ +export function resolveAgentRunErrorLifecycleFields( + error: unknown, + signal: AbortSignal | undefined, +): { + aborted?: true; + stopReason?: + | typeof AGENT_RUN_ABORTED_STOP_REASON + | typeof AGENT_RUN_RESTART_ABORT_STOP_REASON + | "timeout"; + timeoutPhase?: AgentRunTimeoutPhase; +} { + const abortFields = resolveAgentRunAbortLifecycleFields(signal); + if (abortFields.aborted) { + return abortFields; + } + if (!isProviderTimeoutError(error)) { + return {}; + } + return { + stopReason: "timeout", + timeoutPhase: "provider", + }; +} + /** Returns whether a stop reason is the stable aborted-run reason. */ export function isAbortedAgentStopReason( value: unknown, diff --git a/src/agents/subagent-announce-output.test.ts b/src/agents/subagent-announce-output.test.ts index 85c859e39591..aa581b4244ee 100644 --- a/src/agents/subagent-announce-output.test.ts +++ b/src/agents/subagent-announce-output.test.ts @@ -372,6 +372,26 @@ describe("applySubagentWaitOutcome", () => { }); }); + it("treats abandoned ok wait snapshots as incomplete failures", () => { + const applied = applySubagentWaitOutcome({ + wait: { + status: "ok", + startedAt: 100, + endedAt: 150, + livenessState: "abandoned", + }, + outcome: undefined, + }); + + expect(applied.outcome).toEqual({ + status: "error", + error: "Agent run ended before producing a complete result.", + startedAt: 100, + endedAt: 150, + elapsedMs: 50, + }); + }); + it("keeps provider hard timeouts stronger than blocked wait metadata", () => { const applied = applySubagentWaitOutcome({ wait: { @@ -394,7 +414,7 @@ describe("applySubagentWaitOutcome", () => { }); }); - it("keeps rpc timeout wait snapshots as timeout outcomes", () => { + it("keeps explicit cancellation distinct from timeout outcomes", () => { const applied = applySubagentWaitOutcome({ wait: { status: "timeout", @@ -406,7 +426,8 @@ describe("applySubagentWaitOutcome", () => { }); expect(applied.outcome).toEqual({ - status: "timeout", + status: "error", + error: "subagent run terminated", startedAt: 100, endedAt: 150, elapsedMs: 50, diff --git a/src/agents/subagent-announce-output.ts b/src/agents/subagent-announce-output.ts index 4712e8b748a7..316c40a6c311 100644 --- a/src/agents/subagent-announce-output.ts +++ b/src/agents/subagent-announce-output.ts @@ -291,7 +291,11 @@ export function applySubagentWaitOutcome(params: { outcome = { status: "timeout" }; } else if (terminalOutcome?.reason === "aborted" || terminalOutcome?.reason === "cancelled") { outcome = { status: "error", error: "subagent run terminated" }; - } else if (terminalOutcome?.reason === "blocked" || terminalOutcome?.reason === "failed") { + } else if ( + terminalOutcome?.reason === "blocked" || + terminalOutcome?.reason === "abandoned" || + terminalOutcome?.reason === "failed" + ) { outcome = { status: "error", error: terminalOutcome.error ?? waitError }; } else if (terminalOutcome?.reason === "completed") { outcome = { status: "ok" }; diff --git a/src/agents/subagent-registry.test.ts b/src/agents/subagent-registry.test.ts index 09ec356a0687..57eb73326fb8 100644 --- a/src/agents/subagent-registry.test.ts +++ b/src/agents/subagent-registry.test.ts @@ -1841,7 +1841,7 @@ describe("subagent registry seam flow", () => { expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); }); - it("records terminal agent.wait timeouts even before session store timing is persisted", async () => { + it("records explicit agent.wait cancellation before session timing is persisted", async () => { mocks.callGateway.mockImplementation(async (request: { method?: string }) => { if (request.method === "agent.wait") { return { @@ -1870,14 +1870,23 @@ describe("subagent registry seam flow", () => { cleanup: "keep", }); + // Main defers timed-out lifecycle completion behind a retry grace timer. + await vi.advanceTimersByTimeAsync(20_000); + await waitForFast(() => { const run = mod .listSubagentRunsForRequester("agent:main:main") .find((entry) => entry.runId === "run-terminal-timeout"); expect(run?.endedAt).toBe(222); - expectRecordFields(run?.outcome, { status: "timeout" }, "terminal timeout outcome"); + expectRecordFields( + run?.outcome, + { status: "error", error: "subagent run terminated" }, + "terminal cancellation outcome", + ); }); - expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); + // Announce delivery for wait-terminal completions is owned by main's + // cancellation-evidence reconciliation and covered by its own flows; + // this test pins the audit-relevant ordering (outcome + endedAt) only. }); it("caps terminal agent.wait timeouts to the explicit run deadline", async () => { @@ -4239,78 +4248,94 @@ describe("subagent registry seam flow", () => { expect(run?.cleanupCompletedAt).toBeTypeOf("number"); }); - it("announces blocked lifecycle end events as errors instead of success", async () => { - mocks.callGateway.mockImplementation(async (request: { method?: string }) => { - if (request.method === "agent.wait") { - return { status: "pending" }; - } - return {}; - }); - - mod.registerSubagentRun({ + it.each([ + { + livenessState: "blocked", runId: "run-blocked-end", - childSessionKey: "agent:main:subagent:child", - requesterSessionKey: "agent:main:main", - requesterDisplayKey: "main", task: "overflow task", - cleanup: "keep", - expectsCompletionMessage: true, - }); + error: "Context overflow: prompt too large for the model.", + }, + { + livenessState: "abandoned", + runId: "run-abandoned-end", + task: "incomplete tool chain", + error: "Agent run ended before producing a complete result.", + }, + ] as const)( + "announces $livenessState lifecycle end events as errors instead of success", + async ({ livenessState, runId, task, error }) => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => { + if (request.method === "agent.wait") { + return { status: "pending" }; + } + return {}; + }); - const lastOnAgentEventCall = mocks.onAgentEvent.mock.calls[ - mocks.onAgentEvent.mock.calls.length - 1 - ] as unknown as - | [(evt: { runId: string; stream: string; data: Record }) => void] - | undefined; - const lifecycleHandler = lastOnAgentEventCall?.[0]; - expect(lifecycleHandler).toBeTypeOf("function"); + mod.registerSubagentRun({ + runId, + childSessionKey: "agent:main:subagent:child", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task, + cleanup: "keep", + expectsCompletionMessage: true, + }); - lifecycleHandler?.({ - runId: "run-blocked-end", - stream: "lifecycle", - data: { - phase: "start", - startedAt: 10, - }, - }); - lifecycleHandler?.({ - runId: "run-blocked-end", - stream: "lifecycle", - data: { - phase: "end", - startedAt: 10, - endedAt: 20, - livenessState: "blocked", - error: "Context overflow: prompt too large for the model.", - }, - }); + const lastOnAgentEventCall = mocks.onAgentEvent.mock.calls[ + mocks.onAgentEvent.mock.calls.length - 1 + ] as unknown as + | [(evt: { runId: string; stream: string; data: Record }) => void] + | undefined; + const lifecycleHandler = lastOnAgentEventCall?.[0]; + expect(lifecycleHandler).toBeTypeOf("function"); - await waitForFast(() => { - expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); - }); - const announceParams = expectRecordFields( - getMockCallArg(mocks.runSubagentAnnounceFlow, 0, 0, "blocked announce"), - { childRunId: "run-blocked-end" }, - "blocked announce params", - ); - expectRecordFields( - announceParams.outcome, - { - status: "error", - error: "Context overflow: prompt too large for the model.", - startedAt: 10, - endedAt: 20, - elapsedMs: 10, - }, - "blocked announce outcome", - ); + lifecycleHandler?.({ + runId, + stream: "lifecycle", + data: { + phase: "start", + startedAt: 10, + }, + }); + lifecycleHandler?.({ + runId, + stream: "lifecycle", + data: { + phase: "end", + startedAt: 10, + endedAt: 20, + livenessState, + ...(livenessState === "blocked" ? { error } : { replayInvalid: true }), + }, + }); - const run = mod - .listSubagentRunsForRequester("agent:main:main") - .find((entry) => entry.runId === "run-blocked-end"); - expect(run?.endedReason).toBe("subagent-error"); - expect(run?.outcome?.status).toBe("error"); - }); + await waitForFast(() => { + expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); + }); + const announceParams = expectRecordFields( + getMockCallArg(mocks.runSubagentAnnounceFlow, 0, 0, `${livenessState} announce`), + { childRunId: runId }, + `${livenessState} announce params`, + ); + expectRecordFields( + announceParams.outcome, + { + status: "error", + error, + startedAt: 10, + endedAt: 20, + elapsedMs: 10, + }, + `${livenessState} announce outcome`, + ); + + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === runId); + expect(run?.endedReason).toBe("subagent-error"); + expect(run?.outcome?.status).toBe("error"); + }, + ); it("publishes aborted lifecycle end events only after killed reconciliation", async () => { mocks.callGateway.mockImplementation(async (request: { method?: string }) => { diff --git a/src/agents/subagent-registry.ts b/src/agents/subagent-registry.ts index 2d763edf58de..57c15d6da8c2 100644 --- a/src/agents/subagent-registry.ts +++ b/src/agents/subagent-registry.ts @@ -11,7 +11,12 @@ import type { ContextEngine, SubagentEndReason } from "../context-engine/types.j import { callGateway } from "../gateway/call.js"; import { getAgentRunContext, onAgentEvent } from "../infra/agent-events.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; -import { formatBlockedLivenessError, isBlockedLivenessState } from "../shared/agent-liveness.js"; +import { + formatAbandonedLivenessError, + formatBlockedLivenessError, + isAbandonedLivenessState, + isBlockedLivenessState, +} from "../shared/agent-liveness.js"; import { createLazyImportLoader, createLazyPromiseLoader } from "../shared/lazy-promise.js"; import { importRuntimeModule } from "../shared/runtime-import.js"; import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js"; @@ -1496,7 +1501,9 @@ function ensureListener() { }); return; } - if (isBlockedLivenessState(livenessState)) { + const blocked = isBlockedLivenessState(livenessState); + const abandoned = isAbandonedLivenessState(livenessState); + if (blocked || abandoned) { clearPendingLifecycleError(evt.runId); clearPendingLifecycleTimeout(evt.runId); const blockedParams = { @@ -1504,7 +1511,9 @@ function ensureListener() { endedAt, outcome: { status: "error" as const, - error: formatBlockedLivenessError(error), + error: blocked + ? formatBlockedLivenessError(error) + : formatAbandonedLivenessError(error), }, reason: SUBAGENT_ENDED_REASON_ERROR, sendFarewell: true, @@ -1512,7 +1521,10 @@ function ensureListener() { triggerCleanup: true, startedAt, }; - await completeSubagentRunWithRecovery(blockedParams, "lifecycle-blocked-event"); + await completeSubagentRunWithRecovery( + blockedParams, + blocked ? "lifecycle-blocked-event" : "lifecycle-abandoned-event", + ); return; } if (evt.data?.aborted) { diff --git a/src/agents/tool-result-error.test.ts b/src/agents/tool-result-error.test.ts new file mode 100644 index 000000000000..a5c2fad13fd3 --- /dev/null +++ b/src/agents/tool-result-error.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { + resolveToolExecutionErrorKind, + resolveToolResultFailureKind, +} from "./tool-result-error.js"; + +describe("resolveToolExecutionErrorKind", () => { + it("recognizes structured timeout identities", () => { + expect( + resolveToolExecutionErrorKind( + Object.assign(new Error("deadline elapsed"), { name: "TimeoutError" }), + ), + ).toBe("timed_out"); + expect(resolveToolExecutionErrorKind({ code: "ETIMEDOUT" })).toBe("timed_out"); + expect(resolveToolExecutionErrorKind({ reason: "timeout" })).toBe("timed_out"); + }); + + it("does not infer timeout from validation text", () => { + expect(resolveToolExecutionErrorKind(new Error("timeoutMs must be a positive number"))).toBe( + "failed", + ); + }); + + it("contains hostile error fields", () => { + const hostile = Object.defineProperty({}, "name", { + get() { + throw new Error("name getter escaped"); + }, + }); + expect(resolveToolExecutionErrorKind(hostile)).toBe("failed"); + }); +}); + +describe("resolveToolResultFailureKind", () => { + it("contains hostile structured result fields", () => { + const hostileDetails = new Proxy( + {}, + { + has() { + throw new Error("details field check escaped"); + }, + get() { + throw new Error("details field getter escaped"); + }, + }, + ); + const hostileResult = Object.defineProperty({}, "details", { + get() { + throw new Error("details getter escaped"); + }, + }); + + expect(resolveToolResultFailureKind({ details: hostileDetails })).toBeUndefined(); + expect(resolveToolResultFailureKind(hostileResult)).toBeUndefined(); + }); +}); diff --git a/src/agents/tool-result-error.ts b/src/agents/tool-result-error.ts index cf2e2014e9cd..f2b367319537 100644 --- a/src/agents/tool-result-error.ts +++ b/src/agents/tool-result-error.ts @@ -1,24 +1,86 @@ import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce"; +import { formatErrorMessage } from "../infra/errors.js"; + +const TOOL_TIMEOUT_ERROR_CODES = new Set([ + "ERR_TIMEOUT", + "ESOCKETTIMEDOUT", + "ETIMEDOUT", + "UND_ERR_BODY_TIMEOUT", + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", +]); + +function readToolErrorField(error: object, key: string): unknown { + try { + return key in error ? (error as Record)[key] : undefined; + } catch { + return undefined; + } +} + +function hasStructuredToolTimeoutIdentity(error: unknown): boolean { + const pending = [error]; + const seen = new Set(); + while (pending.length > 0 && seen.size < 8) { + const current = pending.shift(); + if (!current || typeof current !== "object" || seen.has(current)) { + continue; + } + seen.add(current); + const name = readToolErrorField(current, "name"); + if (name === "TimeoutError") { + return true; + } + const code = readToolErrorField(current, "code"); + if (typeof code === "string" && TOOL_TIMEOUT_ERROR_CODES.has(code.trim().toUpperCase())) { + return true; + } + for (const key of ["reason", "status"] as const) { + const value = readToolErrorField(current, key); + const normalized = normalizeOptionalLowercaseString(value); + if (normalized === "timeout" || normalized === "timed_out") { + return true; + } + if (value && typeof value === "object") { + pending.push(value); + } + } + const cause = readToolErrorField(current, "cause"); + if (cause && typeof cause === "object") { + pending.push(cause); + } + } + return false; +} export function readToolResultDetails(result: unknown): Record | undefined { if (!result || typeof result !== "object") { return undefined; } - const record = result as Record; - return record.details && typeof record.details === "object" && !Array.isArray(record.details) - ? (record.details as Record) - : undefined; + try { + const details = readToolErrorField(result, "details"); + return details && typeof details === "object" && !Array.isArray(details) + ? (details as Record) + : undefined; + } catch { + return undefined; + } } export function readToolResultStatus(result: unknown): string | undefined { - return normalizeOptionalLowercaseString(readToolResultDetails(result)?.status); + const details = readToolResultDetails(result); + return normalizeOptionalLowercaseString( + details ? readToolErrorField(details, "status") : undefined, + ); } export function isToolResultError(result: unknown): boolean { const details = readToolResultDetails(result); const normalized = readToolResultStatus(result); - const explicitlySuccessful = details?.ok === true || details?.success === true; - if (details?.ok === false || details?.success === false) { + const ok = details ? readToolErrorField(details, "ok") : undefined; + const success = details ? readToolErrorField(details, "success") : undefined; + const explicitlySuccessful = ok === true || success === true; + if (ok === false || success === false) { return true; } const hasFailureStatus = @@ -41,9 +103,62 @@ export function isToolResultError(result: unknown): boolean { if (hasFailureStatus && !explicitlySuccessful) { return true; } - if (details?.timedOut === true || Boolean(details?.error)) { + const timedOut = details ? readToolErrorField(details, "timedOut") : undefined; + const error = details ? readToolErrorField(details, "error") : undefined; + if (timedOut === true || Boolean(error)) { return true; } - const exitCode = details?.exitCode; + const exitCode = details ? readToolErrorField(details, "exitCode") : undefined; return typeof exitCode === "number" && Number.isFinite(exitCode) && exitCode !== 0; } + +export type ToolResultFailureKind = "blocked" | "cancelled" | "failed" | "timed_out"; + +/** Classify a thrown tool error without inferring cancellation from message text. */ +export function resolveToolExecutionErrorKind(error: unknown): "failed" | "timed_out" { + try { + return hasStructuredToolTimeoutIdentity(error) ? "timed_out" : "failed"; + } catch { + return "failed"; + } +} + +/** Format a redacted tool error without allowing hostile getters to escape observability. */ +export function formatToolExecutionErrorMessage(error: unknown, fallback: string): string { + try { + return formatErrorMessage(error) || fallback; + } catch { + return fallback; + } +} + +/** Classify a resolved structured tool result through the shared terminal contract. */ +export function resolveToolResultFailureKind(result: unknown): ToolResultFailureKind | undefined { + if (!isToolResultError(result)) { + return undefined; + } + const status = readToolResultStatus(result); + if ( + status === "blocked" || + status === "denied" || + status === "forbidden" || + status === "disabled" || + status === "approval-unavailable" + ) { + return "blocked"; + } + const details = readToolResultDetails(result); + const timedOut = details ? readToolErrorField(details, "timedOut") : undefined; + if (timedOut === true || status === "timeout" || status === "timed_out") { + return "timed_out"; + } + if ( + status === "aborted" || + status === "cancelled" || + status === "canceled" || + status === "killed" + ) { + return "cancelled"; + } + return "failed"; +} diff --git a/src/agents/tool-schema-quarantine.test.ts b/src/agents/tool-schema-quarantine.test.ts index acbe6e1b1d74..55bc318d908e 100644 --- a/src/agents/tool-schema-quarantine.test.ts +++ b/src/agents/tool-schema-quarantine.test.ts @@ -1,6 +1,10 @@ // Tool schema quarantine tests cover diagnostic logging for unreadable runtime // tool entries without touching the broken tool object again. import { afterEach, describe, expect, it } from "vitest"; +import { + onTrustedToolExecutionEvent, + type TrustedToolExecutionEvent, +} from "../infra/diagnostic-events.js"; import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; import { withStateDirEnv } from "../test-helpers/state-dir-env.js"; import { @@ -16,6 +20,8 @@ afterEach(() => { describe("runtime tool schema quarantine logging", () => { it("does not re-read unreadable tool entries while logging diagnostics", () => { + const events: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => events.push(event)); const tools = new Proxy([] as AnyAgentTool[], { get(target, property, receiver) { if (property === "0") { @@ -25,19 +31,32 @@ describe("runtime tool schema quarantine logging", () => { }, }); - expect(() => - logRuntimeToolSchemaQuarantine({ - diagnostics: [ - { - toolName: "tool[0]", - toolIndex: 0, - violations: ["tool[0] is unreadable"], - }, - ], - tools, + try { + expect(() => + logRuntimeToolSchemaQuarantine({ + diagnostics: [ + { + toolName: "tool[0]", + toolIndex: 0, + violations: ["tool[0] is unreadable"], + }, + ], + tools, + runId: "run-fuzzplugin-unreadable-tool", + agentId: "main", + }), + ).not.toThrow(); + } finally { + stop(); + } + expect(events).toMatchObject([ + { + type: "tool.execution.blocked", runId: "run-fuzzplugin-unreadable-tool", - }), - ).not.toThrow(); + agentId: "main", + toolName: "tool[0]", + }, + ]); }); it("clears this process's persisted quarantine after the tool schema recovers", async () => { @@ -60,6 +79,7 @@ describe("runtime tool schema quarantine logging", () => { }, ], runId: "run-recovered-tool", + agentId: "main", }); expect(listPersistedRuntimeToolSchemaQuarantines()).toEqual([]); diff --git a/src/agents/tool-schema-quarantine.ts b/src/agents/tool-schema-quarantine.ts index 51fe61432077..32766f0d7ffc 100644 --- a/src/agents/tool-schema-quarantine.ts +++ b/src/agents/tool-schema-quarantine.ts @@ -78,6 +78,7 @@ export function logRuntimeToolSchemaQuarantine(params: { diagnostics: readonly RuntimeToolSchemaDiagnostic[]; tools: readonly AnyAgentTool[]; runId: string; + agentId: string; sessionKey?: string; sessionId?: string; }): void { @@ -96,6 +97,7 @@ export function logRuntimeToolSchemaQuarantine(params: { emitTrustedDiagnosticEvent({ type: "tool.execution.blocked", runId: params.runId, + agentId: params.agentId, ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), ...(params.sessionId ? { sessionId: params.sessionId } : {}), toolName: diagnostic.toolName, diff --git a/src/audit/agent-event-audit.ts b/src/audit/agent-event-audit.ts new file mode 100644 index 000000000000..8a578c4f76d9 --- /dev/null +++ b/src/audit/agent-event-audit.ts @@ -0,0 +1,439 @@ +/** Redaction-safe projection from live agent events into durable audit metadata. */ +import { createHash } from "node:crypto"; +import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion"; +import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce"; +import { + AGENT_RUN_TERMINAL_RETRY_GRACE_MS, + buildAgentRunTerminalOutcome, + mergeAgentRunTerminalOutcome, + type AgentRunTerminalOutcome, +} from "../agents/agent-run-terminal-outcome.js"; +import { normalizeAgentRunTimeoutPhase } from "../agents/run-timeout-attribution.js"; +import { isAllowedToolCallName } from "../agents/tool-call-shared.js"; +import type { AgentEventPayload } from "../infra/agent-events.js"; +import type { TrustedToolExecutionEvent } from "../infra/diagnostic-events.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; +import { parseAgentSessionKey } from "../routing/session-key.js"; +import type { + AuditEventErrorCode, + AuditEventInput, + AuditEventStatus, +} from "./audit-event-types.js"; +import { createAuditEventWriter, type AuditEventWriter } from "./audit-event-writer.js"; + +const runProvenance = new Map< + string, + { actorType: "agent" | "system"; agentId: string; sessionKey?: string; sessionId?: string } +>(); +const MAX_TRACKED_RUN_PROVENANCE = 1_024; +const log = createSubsystemLogger("audit/events"); +let persistenceFailureWarned = false; + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function auditToolName(value: unknown): string | undefined { + const toolName = nonEmptyString(value)?.trim(); + if (!toolName) { + return undefined; + } + // Tool lifecycle producers include provider-controlled streams. Preserve + // only the compact model-facing name contract at the durable boundary. + return isAllowedToolCallName(toolName, null) ? toolName : "unknown"; +} + +function auditToolCallId(value: unknown): string | undefined { + const toolCallId = nonEmptyString(value); + if (!toolCallId) { + return undefined; + } + // Call ids remain useful for correlation, but their provider-owned bytes + // are not operator metadata and must never enter the ledger verbatim. + return `sha256:${createHash("sha256").update(toolCallId).digest("hex")}`; +} + +function rememberRunProvenance( + runId: string, + provenance: { + actorType: "agent" | "system"; + agentId: string; + sessionKey?: string; + sessionId?: string; + }, +): void { + runProvenance.delete(runId); + runProvenance.set(runId, provenance); + while (runProvenance.size > MAX_TRACKED_RUN_PROVENANCE) { + const oldestRunId = runProvenance.keys().next().value; + if (oldestRunId === undefined) { + break; + } + runProvenance.delete(oldestRunId); + } +} + +function resolveProvenance( + runId: string, + event: { agentId?: unknown; sessionKey?: unknown; sessionId?: unknown }, +) { + const remembered = runProvenance.get(runId); + const sessionKey = nonEmptyString(event.sessionKey) ?? remembered?.sessionKey; + const sessionId = nonEmptyString(event.sessionId) ?? remembered?.sessionId; + const eventAgentId = nonEmptyString(event.agentId); + const sessionAgentId = sessionKey ? parseAgentSessionKey(sessionKey)?.agentId : undefined; + const agentId = eventAgentId ?? sessionAgentId ?? remembered?.agentId ?? "unknown"; + const actorType = eventAgentId || sessionAgentId ? "agent" : (remembered?.actorType ?? "system"); + return { actorType, agentId, sessionKey, sessionId }; +} + +function resolveToolProvenance( + runId: string, + event: { agentId?: unknown; sessionKey?: unknown; sessionId?: unknown }, +) { + const observed = resolveProvenance(runId, event); + const remembered = runProvenance.get(runId); + if (!remembered) { + return observed; + } + // Tool diagnostics may use an execution sandbox key. Lifecycle start owns + // the canonical run identity; tool metadata only fills missing session fields. + return { + ...remembered, + sessionKey: remembered.sessionKey ?? observed.sessionKey, + sessionId: remembered.sessionId ?? observed.sessionId, + }; +} + +function classifyRunTerminal( + data: Record, + phase: "end" | "error", +): { + outcome: AgentRunTerminalOutcome; + status: AuditEventStatus; + errorCode?: AuditEventErrorCode; +} { + const stopReason = nonEmptyString(data.stopReason); + const timeoutPhase = normalizeAgentRunTimeoutPhase(data.timeoutPhase); + const terminalStatus = normalizeOptionalLowercaseString(data.status); + const explicitlyTimedOut = + stopReason === "timeout" || + timeoutPhase !== undefined || + terminalStatus === "timeout" || + terminalStatus === "timed_out"; + const explicitlyCancelled = + !explicitlyTimedOut && + (data.aborted === true || + stopReason === "aborted" || + terminalStatus === "cancelled" || + terminalStatus === "canceled" || + terminalStatus === "aborted"); + // The terminal helper accepts wait statuses, so normalize explicit lifecycle + // cancellation to its canonical stop signal without persisting the raw reason. + const outcomeStopReason = explicitlyCancelled && !explicitlyTimedOut ? "stop" : stopReason; + const outcome = buildAgentRunTerminalOutcome({ + status: explicitlyTimedOut + ? "timeout" + : phase === "error" + ? "error" + : explicitlyCancelled + ? "error" + : "ok", + stopReason: outcomeStopReason, + livenessState: data.livenessState, + timeoutPhase, + providerStarted: data.providerStarted, + startedAt: data.startedAt, + endedAt: data.endedAt, + }); + if (outcome.reason === "cancelled" || outcome.reason === "aborted") { + return { outcome, status: "cancelled", errorCode: "run_cancelled" }; + } + if (outcome.reason === "hard_timeout" || outcome.reason === "timed_out") { + return { outcome, status: "timed_out", errorCode: "run_timed_out" }; + } + if (outcome.reason === "blocked") { + return { outcome, status: "blocked", errorCode: "run_blocked" }; + } + return outcome.reason === "completed" + ? { outcome, status: "succeeded" } + : { outcome, status: "failed", errorCode: "run_failed" }; +} + +type AgentAuditProjection = { + input: AuditEventInput; + terminal?: { outcome: AgentRunTerminalOutcome; phase: "end" | "error" }; +}; + +function projectAgentEvent(event: AgentEventPayload): AgentAuditProjection | undefined { + const runId = nonEmptyString(event.runId); + const phase = nonEmptyString(event.data.phase); + if (!runId || !phase) { + return undefined; + } + const provenance = resolveProvenance(runId, event); + if (event.stream === "lifecycle" && phase === "start") { + rememberRunProvenance(runId, provenance); + return { + input: { + sourceSequence: event.seq, + occurredAt: asDateTimestampMs(event.data.startedAt) ?? event.ts, + kind: "agent_run", + action: "agent.run.started", + status: "started", + actorType: provenance.actorType, + actorId: provenance.agentId, + agentId: provenance.agentId, + ...(provenance.sessionKey ? { sessionKey: provenance.sessionKey } : {}), + ...(provenance.sessionId ? { sessionId: provenance.sessionId } : {}), + runId, + }, + }; + } + if (event.stream === "lifecycle" && (phase === "end" || phase === "error")) { + rememberRunProvenance(runId, provenance); + const { outcome, ...terminal } = classifyRunTerminal(event.data, phase); + return { + input: { + sourceSequence: event.seq, + occurredAt: asDateTimestampMs(event.data.endedAt) ?? event.ts, + kind: "agent_run", + action: "agent.run.finished", + ...terminal, + actorType: provenance.actorType, + actorId: provenance.agentId, + agentId: provenance.agentId, + ...(provenance.sessionKey ? { sessionKey: provenance.sessionKey } : {}), + ...(provenance.sessionId ? { sessionId: provenance.sessionId } : {}), + runId, + }, + terminal: { outcome, phase }, + }; + } + return undefined; +} + +/** Return a metadata-only audit input for supported run lifecycle events. */ +export function projectAgentEventToAudit(event: AgentEventPayload): AuditEventInput | undefined { + return projectAgentEvent(event)?.input; +} + +/** Project the complete trusted tool-execution lifecycle without private diagnostic content. */ +export function projectToolExecutionEventToAudit( + event: TrustedToolExecutionEvent, +): AuditEventInput | undefined { + // Schema quarantine describes tool availability before invocation. Without + // a call identity it must not become a durable tool-action claim. + if ( + event.type === "tool.execution.blocked" && + event.deniedReason === "unsupported_tool_schema" && + !nonEmptyString(event.toolCallId) + ) { + return undefined; + } + const runId = nonEmptyString(event.runId); + const toolName = auditToolName(event.toolName); + if (!runId || !toolName) { + return undefined; + } + const toolCallId = auditToolCallId(event.toolCallId); + const provenance = resolveToolProvenance(runId, event); + const errorCategory = + event.type === "tool.execution.error" + ? normalizeOptionalLowercaseString(event.errorCategory) + : undefined; + const terminalReason = event.type === "tool.execution.error" ? event.terminalReason : undefined; + const diagnosticErrorCode = + event.type === "tool.execution.error" + ? normalizeOptionalLowercaseString(event.errorCode) + : undefined; + // Modern producers set terminalReason explicitly; errorCategory is only a + // legacy fallback and must not override a definitive timeout or failure. + const toolCancelled = + terminalReason === "cancelled" || + (terminalReason === undefined && + (errorCategory === "aborted" || + errorCategory === "aborterror" || + errorCategory === "cancelled" || + errorCategory === "canceled")); + const toolTimedOut = terminalReason === "timed_out"; + // Unknown is an explicit dependency boundary, not a failed-run inference. + // Keep it authoritative when enclosing run provenance says cancel or timeout. + const terminal = + event.type === "tool.execution.started" + ? { status: "started" as const } + : event.type === "tool.execution.completed" + ? { status: "succeeded" as const } + : event.type === "tool.execution.blocked" + ? { status: "blocked" as const, errorCode: "tool_blocked" as const } + : diagnosticErrorCode === "tool_outcome_unknown" + ? { status: "unknown" as const, errorCode: "tool_outcome_unknown" as const } + : toolCancelled + ? { status: "cancelled" as const, errorCode: "tool_cancelled" as const } + : toolTimedOut + ? { status: "timed_out" as const, errorCode: "tool_timed_out" as const } + : { status: "failed" as const, errorCode: "tool_failed" as const }; + return { + sourceSequence: event.seq, + occurredAt: asDateTimestampMs(event.sourceTimestampMs) ?? event.ts, + kind: "tool_action", + action: + event.type === "tool.execution.started" ? "tool.action.started" : "tool.action.finished", + ...terminal, + actorType: provenance.actorType, + actorId: provenance.agentId, + agentId: provenance.agentId, + ...(provenance.sessionKey ? { sessionKey: provenance.sessionKey } : {}), + ...(provenance.sessionId ? { sessionId: provenance.sessionId } : {}), + runId, + ...(toolCallId ? { toolCallId } : {}), + toolName, + }; +} + +/** Create the Gateway-owned non-blocking audit projection and persistence handle. */ +export function createAgentEventAuditRecorder(options?: { + writer?: AuditEventWriter; + stateDir?: string; + terminalSettleMs?: number; +}): { + record: (event: AgentEventPayload) => void; + recordTool: (event: TrustedToolExecutionEvent) => void; + stop: () => Promise; +} { + const writer = + options?.writer ?? + createAuditEventWriter({ + ...(options?.stateDir ? { stateDir: options.stateDir } : {}), + onError: (error) => { + if (!persistenceFailureWarned) { + persistenceFailureWarned = true; + log.warn(`audit event persistence failed: ${error}`); + } + }, + }); + type PendingTerminal = NonNullable & { + input: AuditEventInput; + timer: ReturnType; + }; + const terminalSettleMs = Math.max( + 0, + Math.floor(options?.terminalSettleMs ?? AGENT_RUN_TERMINAL_RETRY_GRACE_MS), + ); + const pendingTerminals = new Map(); + const openRunInstances = new Set(); + const settledRunInstances = new Set(); + + const rememberSettled = (runInstance: string) => { + settledRunInstances.delete(runInstance); + settledRunInstances.add(runInstance); + if (settledRunInstances.size > MAX_TRACKED_RUN_PROVENANCE) { + const oldest = settledRunInstances.values().next().value; + if (oldest !== undefined) { + settledRunInstances.delete(oldest); + } + } + }; + const clearPending = (runInstance: string) => { + const pending = pendingTerminals.get(runInstance); + if (!pending) { + return; + } + clearTimeout(pending.timer); + pendingTerminals.delete(runInstance); + }; + const flushPending = (runInstance: string) => { + const pending = pendingTerminals.get(runInstance); + if (!pending) { + return; + } + clearPending(runInstance); + openRunInstances.delete(runInstance); + if (writer.record(pending.input)) { + rememberSettled(runInstance); + } + }; + const scheduleTerminal = (runInstance: string, incoming: Omit) => { + const existing = pendingTerminals.get(runInstance); + let selected = incoming; + if (existing) { + // A bare cleanup end can follow a definitive error without a retry start. + // Otherwise use the shared sticky timeout/cancellation merge contract. + const cleanupAfterError = + existing.phase === "error" && + incoming.phase === "end" && + incoming.outcome.reason === "completed"; + if (cleanupAfterError) { + selected = existing; + } else { + const merged = mergeAgentRunTerminalOutcome(existing.outcome, incoming.outcome); + selected = merged === existing.outcome ? existing : incoming; + } + clearTimeout(existing.timer); + } + const timer = setTimeout(() => flushPending(runInstance), terminalSettleMs); + timer.unref?.(); + pendingTerminals.delete(runInstance); + pendingTerminals.set(runInstance, { ...selected, timer }); + if (pendingTerminals.size > MAX_TRACKED_RUN_PROVENANCE) { + const oldest = pendingTerminals.keys().next().value; + if (oldest !== undefined) { + flushPending(oldest); + } + } + }; + + return { + record: (event) => { + const projection = projectAgentEvent(event); + if (!projection) { + return; + } + const runInstance = `${event.lifecycleGeneration ?? "unknown"}\0${event.runId}`; + if (!projection.terminal) { + const alreadyOpen = openRunInstances.has(runInstance); + clearPending(runInstance); + settledRunInstances.delete(runInstance); + if (alreadyOpen) { + return; + } + // Retry starts cancel a provisional terminal for the same logical run. + // Keep the original start so one run cannot acquire unmatched starts. + openRunInstances.add(runInstance); + writer.record(projection.input); + return; + } + if (settledRunInstances.has(runInstance)) { + return; + } + if ( + projection.terminal.outcome.reason === "completed" && + !pendingTerminals.has(runInstance) + ) { + openRunInstances.delete(runInstance); + if (writer.record(projection.input)) { + rememberSettled(runInstance); + } + return; + } + scheduleTerminal(runInstance, { input: projection.input, ...projection.terminal }); + }, + recordTool: (event) => { + const input = projectToolExecutionEventToAudit(event); + if (input) { + writer.record(input); + } + }, + stop: async () => { + for (const runInstance of pendingTerminals.keys()) { + flushPending(runInstance); + } + await writer.stop(); + }, + }; +} + +export function resetAgentEventAuditForTest(): void { + runProvenance.clear(); + persistenceFailureWarned = false; +} diff --git a/src/audit/audit-config.test.ts b/src/audit/audit-config.test.ts new file mode 100644 index 000000000000..aedd97f815b2 --- /dev/null +++ b/src/audit/audit-config.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { isAuditLedgerEnabled } from "./audit-config.js"; + +describe("isAuditLedgerEnabled", () => { + it("defaults to enabled without config or audit section", () => { + expect(isAuditLedgerEnabled(undefined)).toBe(true); + expect(isAuditLedgerEnabled({})).toBe(true); + expect(isAuditLedgerEnabled({ audit: {} })).toBe(true); + }); + + it("stays enabled on explicit true", () => { + expect(isAuditLedgerEnabled({ audit: { enabled: true } })).toBe(true); + }); + + it("disables only on explicit false", () => { + expect(isAuditLedgerEnabled({ audit: { enabled: false } })).toBe(false); + }); +}); diff --git a/src/audit/audit-config.ts b/src/audit/audit-config.ts new file mode 100644 index 000000000000..61690545dff7 --- /dev/null +++ b/src/audit/audit-config.ts @@ -0,0 +1,11 @@ +/** Resolves whether the metadata-only audit ledger records new events. */ +import type { OpenClawConfig } from "../config/types.openclaw.js"; + +/** + * The ledger is on by default: an audit trail enabled only after an incident + * cannot explain the incident. `audit.enabled: false` stops new writes; + * existing records remain readable through `audit.list` until they expire. + */ +export function isAuditLedgerEnabled(cfg: OpenClawConfig | undefined): boolean { + return cfg?.audit?.enabled !== false; +} diff --git a/src/audit/audit-event-store.ts b/src/audit/audit-event-store.ts new file mode 100644 index 000000000000..6233016287bd --- /dev/null +++ b/src/audit/audit-event-store.ts @@ -0,0 +1,205 @@ +/** SQLite persistence and stable cursor queries for metadata-only audit events. */ +import { randomUUID } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import type { Insertable, Selectable } from "kysely"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { normalizeSqliteNumber } from "../infra/sqlite-number.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import type { + AuditEventInput, + AuditEventListFilters, + AuditEventListPage, + AuditEventRecord, +} from "./audit-event-types.js"; + +type AuditEventsTable = OpenClawStateKyselyDatabase["audit_events"]; +type AuditDatabase = Pick; +type AuditEventRow = Selectable; + +const AUDIT_EVENT_RETENTION_MS = 30 * 24 * 60 * 60_000; +const AUDIT_EVENT_MAX_ROWS = 100_000; + +function getAuditKysely(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} + +function rowToAuditEvent(row: AuditEventRow): AuditEventRecord { + return { + sequence: normalizeSqliteNumber(row.sequence) ?? 0, + eventId: row.event_id, + sourceSequence: normalizeSqliteNumber(row.source_sequence) ?? 0, + occurredAt: normalizeSqliteNumber(row.occurred_at) ?? 0, + kind: row.kind as AuditEventRecord["kind"], + action: row.action as AuditEventRecord["action"], + status: row.status as AuditEventRecord["status"], + ...(row.error_code + ? { errorCode: row.error_code as NonNullable } + : {}), + actorType: row.actor_type as AuditEventRecord["actorType"], + actorId: row.actor_id, + agentId: row.agent_id, + ...(row.session_key ? { sessionKey: row.session_key } : {}), + ...(row.session_id ? { sessionId: row.session_id } : {}), + runId: row.run_id, + ...(row.tool_call_id ? { toolCallId: row.tool_call_id } : {}), + ...(row.tool_name ? { toolName: row.tool_name } : {}), + redaction: "metadata_only", + }; +} + +function bindAuditEvent(input: AuditEventInput): Insertable { + return { + event_id: randomUUID(), + source_id: `${input.runId}:${input.sourceSequence}:${input.occurredAt}:${input.action}`, + source_sequence: input.sourceSequence, + occurred_at: input.occurredAt, + kind: input.kind, + action: input.action, + status: input.status, + error_code: input.errorCode ?? null, + actor_type: input.actorType, + actor_id: input.actorId, + agent_id: input.agentId, + session_key: input.sessionKey ?? null, + session_id: input.sessionId ?? null, + run_id: input.runId, + tool_call_id: input.toolCallId ?? null, + tool_name: input.toolName ?? null, + }; +} + +function pruneAuditEventsAfterInsert(db: DatabaseSync, now: number): void { + const kysely = getAuditKysely(db); + executeSqliteQuerySync( + db, + kysely.deleteFrom("audit_events").where("occurred_at", "<", now - AUDIT_EVENT_RETENTION_MS), + ); + const overflowRow = executeSqliteQueryTakeFirstSync( + db, + kysely + .selectFrom("audit_events") + .select("sequence") + .orderBy("sequence", "desc") + .offset(AUDIT_EVENT_MAX_ROWS) + .limit(1), + ); + const sequenceCutoff = overflowRow ? normalizeSqliteNumber(overflowRow.sequence) : undefined; + if (sequenceCutoff !== undefined) { + executeSqliteQuerySync( + db, + kysely.deleteFrom("audit_events").where("sequence", "<=", sequenceCutoff), + ); + } +} + +/** Persist one projected event idempotently and prune fixed retention bounds. */ +export function recordAuditEvent( + input: AuditEventInput, + options: OpenClawStateDatabaseOptions = {}, +): AuditEventRecord | undefined { + return runOpenClawStateWriteTransaction(({ db }) => { + const insert = executeSqliteQuerySync( + db, + getAuditKysely(db) + .insertInto("audit_events") + .values(bindAuditEvent(input)) + .onConflict((conflict) => conflict.column("source_id").doNothing()), + ); + const insertedSequence = insert.insertId ? Number(insert.insertId) : undefined; + if (insertedSequence === undefined) { + return undefined; + } + pruneAuditEventsAfterInsert(db, Date.now()); + const row = executeSqliteQueryTakeFirstSync( + db, + getAuditKysely(db) + .selectFrom("audit_events") + .selectAll() + .where("sequence", "=", insertedSequence), + ); + return row ? rowToAuditEvent(row) : undefined; + }, options); +} + +/** List newest-first records using a stable sequence cursor. */ +export function listAuditEvents(params: { + filters?: AuditEventListFilters; + cursor?: number; + limit: number; + now?: number; + database?: OpenClawStateDatabaseOptions; +}): AuditEventListPage { + const { db } = openOpenClawStateDatabase(params.database); + const filters = params.filters ?? {}; + const retainedAfter = (params.now ?? Date.now()) - AUDIT_EVENT_RETENTION_MS; + let query = getAuditKysely(db) + .selectFrom("audit_events") + .selectAll() + .where("occurred_at", ">=", retainedAfter); + if (params.cursor !== undefined) { + query = query.where("sequence", "<", params.cursor); + } + if (filters.agentId) { + query = query.where("agent_id", "=", filters.agentId); + } + if (filters.sessionKey) { + query = query.where("session_key", "=", filters.sessionKey); + } + if (filters.runId) { + query = query.where("run_id", "=", filters.runId); + } + if (filters.kind) { + query = query.where("kind", "=", filters.kind); + } + if (filters.status) { + query = query.where("status", "=", filters.status); + } + if (filters.after !== undefined) { + query = query.where("occurred_at", ">=", filters.after); + } + if (filters.before !== undefined) { + query = query.where("occurred_at", "<=", filters.before); + } + const rows = executeSqliteQuerySync( + db, + query.orderBy("sequence", "desc").limit(params.limit + 1), + ).rows; + const hasMore = rows.length > params.limit; + const pageRows = hasMore ? rows.slice(0, params.limit) : rows; + const events = pageRows.map(rowToAuditEvent); + return { + events, + ...(hasMore && events.length > 0 ? { nextCursor: events[events.length - 1]?.sequence } : {}), + }; +} + +/** Delete expired metadata during Gateway startup and periodic worker maintenance. */ +export function pruneExpiredAuditEvents( + params: { + now?: number; + database?: OpenClawStateDatabaseOptions; + } = {}, +): void { + runOpenClawStateWriteTransaction(({ db }) => { + executeSqliteQuerySync( + db, + getAuditKysely(db) + .deleteFrom("audit_events") + .where("occurred_at", "<", (params.now ?? Date.now()) - AUDIT_EVENT_RETENTION_MS), + ); + }, params.database); +} + +export const auditEventStoreLimits = { + maxRows: AUDIT_EVENT_MAX_ROWS, + retentionMs: AUDIT_EVENT_RETENTION_MS, +} as const; diff --git a/src/audit/audit-event-types.ts b/src/audit/audit-event-types.ts new file mode 100644 index 000000000000..ce91193ef891 --- /dev/null +++ b/src/audit/audit-event-types.ts @@ -0,0 +1,71 @@ +/** Metadata-only durable audit contract for agent runs and tool actions. */ + +export type AuditEventKind = "agent_run" | "tool_action"; + +export type AuditEventAction = + | "agent.run.started" + | "agent.run.finished" + | "tool.action.started" + | "tool.action.finished"; + +export type AuditEventStatus = + | "started" + | "succeeded" + | "failed" + | "cancelled" + | "timed_out" + | "blocked" + | "unknown"; + +export type AuditEventErrorCode = + | "run_failed" + | "run_cancelled" + | "run_timed_out" + | "run_blocked" + | "tool_failed" + | "tool_cancelled" + | "tool_timed_out" + | "tool_blocked" + | "tool_outcome_unknown"; + +export type AuditEventActorType = "agent" | "system"; + +/** Durable columns accepted from trusted lifecycle projection. */ +export type AuditEventInput = { + sourceSequence: number; + occurredAt: number; + kind: AuditEventKind; + action: AuditEventAction; + status: AuditEventStatus; + errorCode?: AuditEventErrorCode; + actorType: AuditEventActorType; + actorId: string; + agentId: string; + sessionKey?: string; + sessionId?: string; + runId: string; + toolCallId?: string; + toolName?: string; +}; + +/** Public record returned by the bounded operator read surface. */ +export type AuditEventRecord = AuditEventInput & { + sequence: number; + eventId: string; + redaction: "metadata_only"; +}; + +export type AuditEventListFilters = { + agentId?: string; + sessionKey?: string; + runId?: string; + kind?: AuditEventKind; + status?: AuditEventStatus; + after?: number; + before?: number; +}; + +export type AuditEventListPage = { + events: AuditEventRecord[]; + nextCursor?: number; +}; diff --git a/src/audit/audit-event-writer.test.ts b/src/audit/audit-event-writer.test.ts new file mode 100644 index 000000000000..60932a95e776 --- /dev/null +++ b/src/audit/audit-event-writer.test.ts @@ -0,0 +1,58 @@ +import { afterAll, afterEach, describe, expect, it } from "vitest"; +import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; +import { + closeOpenClawStateDatabaseForTest, + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { listAuditEvents } from "./audit-event-store.js"; +import type { AuditEventInput } from "./audit-event-types.js"; +import { createAuditEventWriter, testApi } from "./audit-event-writer.js"; + +const tempDirs: string[] = []; + +function input(): AuditEventInput { + return { + sourceSequence: 1, + occurredAt: Date.now(), + kind: "agent_run", + action: "agent.run.started", + status: "started", + actorType: "agent", + actorId: "main", + agentId: "main", + runId: "run-1", + }; +} + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); + +afterAll(() => { + cleanupTempDirs(tempDirs); +}); + +describe("audit event worker", () => { + it("keeps shutdown beyond the supported SQLite contention window", () => { + expect(testApi.auditWriterShutdownTimeoutMs).toBeGreaterThan(OPENCLAW_SQLITE_BUSY_TIMEOUT_MS); + }); + + it("returns immediately under SQLite contention and flushes before stop", async () => { + const stateDir = makeTempDir(tempDirs, "openclaw-audit-writer-"); + const database = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const errors: string[] = []; + const writer = createAuditEventWriter({ stateDir, onError: (error) => errors.push(error) }); + await writer.ready; + const { db } = openOpenClawStateDatabase(database); + db.exec("BEGIN IMMEDIATE"); + const startedAt = performance.now(); + expect(writer.record(input())).toBe(true); + expect(performance.now() - startedAt).toBeLessThan(250); + db.exec("ROLLBACK"); + + await writer.stop(); + expect(errors).toEqual([]); + expect(listAuditEvents({ database, limit: 10 }).events).toHaveLength(1); + }); +}); diff --git a/src/audit/audit-event-writer.ts b/src/audit/audit-event-writer.ts new file mode 100644 index 000000000000..68e7beb38d4d --- /dev/null +++ b/src/audit/audit-event-writer.ts @@ -0,0 +1,192 @@ +/** Non-blocking worker-thread writer for Gateway audit metadata. */ +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Worker } from "node:worker_threads"; +import { resolveStateDir } from "../config/paths.js"; +import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "../state/openclaw-state-db.js"; +import type { AuditEventInput } from "./audit-event-types.js"; + +const MAX_PENDING_AUDIT_EVENTS = 4_096; +// The worker can be synchronously blocked inside SQLite's busy timeout. Keep +// shutdown beyond that window so a queued stop cannot kill an accepted write. +const AUDIT_WRITER_SHUTDOWN_TIMEOUT_MS = OPENCLAW_SQLITE_BUSY_TIMEOUT_MS + 5_000; + +type AuditWriterMessage = + | { type: "ready" } + | { type: "recorded" } + | { type: "record-error"; error: string } + | { type: "maintenance-error"; error: string } + | { type: "stopped" }; + +export type AuditEventWriter = { + ready: Promise; + record: (input: AuditEventInput) => boolean; + stop: () => Promise; +}; + +function resolveAuditEventWriterUrl(currentModuleUrl = import.meta.url): URL { + const currentPath = fileURLToPath(currentModuleUrl); + const normalized = currentPath.replaceAll(path.sep, "/"); + const distMarker = "/dist/"; + const distIndex = normalized.lastIndexOf(distMarker); + if (distIndex >= 0) { + const distRoot = currentPath.slice(0, distIndex + distMarker.length); + return pathToFileURL(path.join(distRoot, "audit", "audit-event-writer.worker.js")); + } + const extension = path.extname(currentPath) || ".js"; + return new URL(`./audit-event-writer.worker${extension}`, currentModuleUrl); +} + +/** Start one bounded worker queue. SQLite contention never blocks the agent-event callback. */ +export function createAuditEventWriter( + options: { + stateDir?: string; + maxPending?: number; + workerUrl?: URL; + onError?: (error: string) => void; + } = {}, +): AuditEventWriter { + const workerUrl = options.workerUrl ?? resolveAuditEventWriterUrl(); + const sourceWorkerExecArgv = workerUrl.pathname.endsWith(".ts") ? ["--import", "tsx"] : undefined; + const maxPending = Math.max(1, Math.floor(options.maxPending ?? MAX_PENDING_AUDIT_EVENTS)); + let worker: Worker; + try { + worker = new Worker(workerUrl, { + workerData: { stateDir: options.stateDir ?? resolveStateDir(process.env) }, + execArgv: sourceWorkerExecArgv, + }); + } catch (error) { + options.onError?.(error instanceof Error ? error.message : String(error)); + return { + ready: Promise.resolve(), + record: () => false, + stop: async () => {}, + }; + } + worker.unref?.(); + + let pending = 0; + let stopped = false; + let unavailable = false; + let readyResolved = false; + let resolveReady!: () => void; + const ready = new Promise((resolve) => { + resolveReady = resolve; + }); + let resolveStop: (() => void) | undefined; + let stopTimer: ReturnType | undefined; + + const markReady = () => { + if (!readyResolved) { + readyResolved = true; + resolveReady(); + } + }; + const finishStop = () => { + if (stopTimer) { + clearTimeout(stopTimer); + stopTimer = undefined; + } + const finish = resolveStop; + resolveStop = undefined; + finish?.(); + }; + const fail = (error: unknown) => { + options.onError?.(error instanceof Error ? error.message : String(error)); + }; + + worker.on("message", (message: AuditWriterMessage) => { + switch (message.type) { + case "ready": + markReady(); + return; + case "recorded": + pending = Math.max(0, pending - 1); + return; + case "record-error": + pending = Math.max(0, pending - 1); + fail(message.error); + return; + case "maintenance-error": + fail(message.error); + return; + case "stopped": + pending = 0; + markReady(); + finishStop(); + } + }); + worker.on("error", (error) => { + unavailable = true; + fail(error); + markReady(); + finishStop(); + }); + worker.on("exit", (code) => { + unavailable = true; + if (!stopped) { + fail(`audit event writer exited with code ${code}`); + } + markReady(); + finishStop(); + }); + + return { + ready, + record: (input) => { + if (stopped || unavailable || pending >= maxPending) { + if (!stopped) { + fail( + unavailable + ? "audit event writer is unavailable; dropping metadata" + : `audit event queue is full (${maxPending}); dropping metadata`, + ); + } + return false; + } + pending += 1; + try { + // Node Worker.postMessage is not the browser Window API and has no targetOrigin. + // oxlint-disable-next-line unicorn/require-post-message-target-origin + worker.postMessage({ type: "record", input }); + return true; + } catch (error) { + pending -= 1; + unavailable = true; + fail(error); + return false; + } + }, + stop: async () => { + if (stopped) { + return; + } + stopped = true; + if (unavailable) { + return; + } + await new Promise((resolve) => { + resolveStop = resolve; + stopTimer = setTimeout(() => { + fail("audit event writer shutdown timed out; pending metadata may be lost"); + void worker.terminate(); + finishStop(); + }, AUDIT_WRITER_SHUTDOWN_TIMEOUT_MS); + try { + // Node Worker.postMessage is not the browser Window API and has no targetOrigin. + // oxlint-disable-next-line unicorn/require-post-message-target-origin + worker.postMessage({ type: "stop" }); + } catch (error) { + fail(error); + finishStop(); + } + }); + }, + }; +} + +export const testApi = { + auditWriterShutdownTimeoutMs: AUDIT_WRITER_SHUTDOWN_TIMEOUT_MS, + maxPendingAuditEvents: MAX_PENDING_AUDIT_EVENTS, + resolveAuditEventWriterUrl, +}; diff --git a/src/audit/audit-event-writer.worker.ts b/src/audit/audit-event-writer.worker.ts new file mode 100644 index 000000000000..9b8ef012ffbf --- /dev/null +++ b/src/audit/audit-event-writer.worker.ts @@ -0,0 +1,52 @@ +/** Worker-thread entrypoint for serialized audit writes and retention maintenance. */ +import { parentPort, workerData } from "node:worker_threads"; +import { closeOpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { pruneExpiredAuditEvents, recordAuditEvent } from "./audit-event-store.js"; +import type { AuditEventInput } from "./audit-event-types.js"; + +const AUDIT_MAINTENANCE_INTERVAL_MS = 60 * 60_000; + +type AuditWriterRequest = { type: "record"; input: AuditEventInput } | { type: "stop" }; + +const stateDir = + workerData && typeof workerData === "object" && typeof workerData.stateDir === "string" + ? workerData.stateDir + : undefined; +if (!parentPort || !stateDir) { + throw new Error("audit event writer requires a parent port and state directory"); +} +const port = parentPort; +const database = { env: { OPENCLAW_STATE_DIR: stateDir } }; + +function reportMaintenance(): void { + try { + pruneExpiredAuditEvents({ database }); + } catch (error) { + port.postMessage({ type: "maintenance-error", error: String(error) }); + } +} + +reportMaintenance(); +const maintenanceTimer = setInterval(reportMaintenance, AUDIT_MAINTENANCE_INTERVAL_MS); +port.postMessage({ type: "ready" }); + +port.on("message", (message: AuditWriterRequest) => { + if (message.type === "record") { + try { + recordAuditEvent(message.input, database); + port.postMessage({ type: "recorded" }); + } catch (error) { + port.postMessage({ type: "record-error", error: String(error) }); + } + return; + } + clearInterval(maintenanceTimer); + reportMaintenance(); + try { + closeOpenClawStateDatabase(); + } catch (error) { + port.postMessage({ type: "maintenance-error", error: String(error) }); + } + port.postMessage({ type: "stopped" }); + port.close(); +}); diff --git a/src/audit/audit-events.test.ts b/src/audit/audit-events.test.ts new file mode 100644 index 000000000000..42fdd2bc35d4 --- /dev/null +++ b/src/audit/audit-events.test.ts @@ -0,0 +1,648 @@ +import { afterAll, afterEach, describe, expect, it } from "vitest"; +import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; +import type { AgentEventPayload } from "../infra/agent-events.js"; +import { + emitTrustedDiagnosticEvent, + onTrustedToolExecutionEvent, + resetDiagnosticEventsForTest, + setDiagnosticsEnabledForProcess, + type TrustedToolExecutionEvent, +} from "../infra/diagnostic-events.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { + createAgentEventAuditRecorder, + projectAgentEventToAudit, + projectToolExecutionEventToAudit, + resetAgentEventAuditForTest, +} from "./agent-event-audit.js"; +import { + auditEventStoreLimits, + listAuditEvents, + pruneExpiredAuditEvents, + recordAuditEvent, +} from "./audit-event-store.js"; +import type { AuditEventInput } from "./audit-event-types.js"; +import type { AuditEventWriter } from "./audit-event-writer.js"; + +const tempDirs: string[] = []; + +function createDatabaseOptions() { + return { env: { OPENCLAW_STATE_DIR: makeTempDir(tempDirs, "openclaw-audit-") } }; +} + +function auditInput(overrides: Partial = {}): AuditEventInput { + return { + sourceSequence: 1, + occurredAt: Date.now(), + kind: "agent_run", + action: "agent.run.started", + status: "started", + actorType: "agent", + actorId: "main", + agentId: "main", + sessionKey: "agent:main:main", + sessionId: "session-1", + runId: "run-1", + ...overrides, + }; +} + +function agentEvent(overrides: Partial): AgentEventPayload { + return { + runId: "run-1", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + data: { phase: "start" }, + sessionKey: "agent:coder:main", + sessionId: "session-1", + agentId: "coder", + ...overrides, + }; +} + +function toolEvent(overrides: Partial = {}): TrustedToolExecutionEvent { + return { + type: "tool.execution.started", + seq: 1, + ts: Date.now(), + runId: "run-1", + sessionKey: "agent:coder:main", + sessionId: "session-1", + toolName: "exec", + toolCallId: "call-1", + ...overrides, + } as TrustedToolExecutionEvent; +} + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); + resetAgentEventAuditForTest(); + resetDiagnosticEventsForTest(); +}); + +afterAll(() => { + cleanupTempDirs(tempDirs); +}); + +describe("audit event persistence", () => { + it("persists stable ordering, filters, and cursor pagination across reopen", () => { + const database = createDatabaseOptions(); + const now = Date.now(); + const oldest = recordAuditEvent(auditInput({ occurredAt: now, sourceSequence: 1 }), database); + recordAuditEvent( + auditInput({ + occurredAt: now + 1, + sourceSequence: 2, + kind: "tool_action", + action: "tool.action.started", + runId: "run-2", + toolCallId: "call-1", + toolName: "read", + }), + database, + ); + recordAuditEvent( + auditInput({ + occurredAt: now + 2, + sourceSequence: 3, + kind: "tool_action", + action: "tool.action.finished", + status: "failed", + errorCode: "tool_failed", + runId: "run-2", + toolCallId: "call-1", + toolName: "read", + }), + database, + ); + + const first = listAuditEvents({ database, limit: 2 }); + expect(first.events.map((event) => event.sourceSequence)).toEqual([3, 2]); + expect(first.nextCursor).toBe(first.events[1]?.sequence); + + closeOpenClawStateDatabaseForTest(); + const second = listAuditEvents({ database, limit: 2, cursor: first.nextCursor }); + expect(second.events.map((event) => event.sourceSequence)).toEqual([1]); + expect(second.events[0]?.eventId).toBe(oldest?.eventId); + expect(second.nextCursor).toBeUndefined(); + + const filtered = listAuditEvents({ + database, + limit: 10, + filters: { + agentId: "main", + sessionKey: "agent:main:main", + runId: "run-2", + kind: "tool_action", + status: "failed", + after: now + 2, + before: now + 2, + }, + }); + expect(filtered.events).toHaveLength(1); + expect(filtered.events[0]).toMatchObject({ + action: "tool.action.finished", + errorCode: "tool_failed", + redaction: "metadata_only", + }); + }); + + it("deduplicates replayed source events", () => { + const database = createDatabaseOptions(); + const input = auditInput(); + expect(recordAuditEvent(input, database)).toBeDefined(); + expect(recordAuditEvent(input, database)).toBeUndefined(); + expect(listAuditEvents({ database, limit: 10 }).events).toHaveLength(1); + }); + + it("caps actual rows without treating dedupe sequence gaps as retained records", () => { + const database = createDatabaseOptions(); + const occurredAt = Date.now(); + recordAuditEvent(auditInput({ occurredAt }), database); + const { db } = openOpenClawStateDatabase(database); + db.prepare("UPDATE sqlite_sequence SET seq = ? WHERE name = 'audit_events'").run( + auditEventStoreLimits.maxRows + 1, + ); + + recordAuditEvent(auditInput({ occurredAt: occurredAt + 1, sourceSequence: 2 }), database); + + expect(listAuditEvents({ database, limit: 10 }).events).toHaveLength(2); + }); + + it("keeps reused run ids distinct across actual event timestamps", () => { + const database = createDatabaseOptions(); + const occurredAt = Date.now(); + expect(recordAuditEvent(auditInput({ occurredAt }), database)).toBeDefined(); + expect(recordAuditEvent(auditInput({ occurredAt: occurredAt + 1 }), database)).toBeDefined(); + expect(listAuditEvents({ database, limit: 10 }).events).toHaveLength(2); + }); + + it("excludes and physically prunes records outside the fixed retention window", () => { + const database = createDatabaseOptions(); + const occurredAt = Date.now(); + recordAuditEvent(auditInput({ occurredAt }), database); + const expiredAt = occurredAt + auditEventStoreLimits.retentionMs + 1; + + expect(listAuditEvents({ database, limit: 10, now: expiredAt }).events).toEqual([]); + pruneExpiredAuditEvents({ database, now: expiredAt }); + expect(listAuditEvents({ database, limit: 10, now: occurredAt }).events).toEqual([]); + }); +}); + +describe("agent activity audit projection", () => { + it("preserves explicit agent identity for unscoped session keys", () => { + const projected = projectAgentEventToAudit( + agentEvent({ sessionKey: "global", agentId: "support" }), + ); + + expect(projected).toMatchObject({ + actorType: "agent", + actorId: "support", + agentId: "support", + sessionKey: "global", + }); + }); + + it("keeps a valid unknown agent id distinct from missing provenance", () => { + const runId = "run-agent-named-unknown"; + const started = projectAgentEventToAudit( + agentEvent({ runId, sessionKey: "global", agentId: "unknown" }), + ); + const finished = projectAgentEventToAudit( + agentEvent({ + runId, + seq: 2, + sessionKey: undefined, + sessionId: undefined, + agentId: undefined, + data: { phase: "end" }, + }), + ); + const tool = projectToolExecutionEventToAudit( + toolEvent({ + runId, + seq: 3, + sessionKey: undefined, + sessionId: undefined, + agentId: undefined, + }), + ); + const missing = projectAgentEventToAudit( + agentEvent({ + runId: "run-missing-provenance", + sessionKey: undefined, + sessionId: undefined, + agentId: undefined, + }), + ); + + expect([started, finished, tool]).toEqual([ + expect.objectContaining({ actorType: "agent", actorId: "unknown", agentId: "unknown" }), + expect.objectContaining({ actorType: "agent", actorId: "unknown", agentId: "unknown" }), + expect.objectContaining({ actorType: "agent", actorId: "unknown", agentId: "unknown" }), + ]); + expect(missing).toMatchObject({ + actorType: "system", + actorId: "unknown", + agentId: "unknown", + }); + }); + + it("keeps tool actions on the canonical lifecycle session", () => { + const runId = "run-channel-routed"; + projectAgentEventToAudit( + agentEvent({ + runId, + sessionKey: "agent:support:channel:customer", + sessionId: "session-canonical", + agentId: "support", + }), + ); + + const projected = projectToolExecutionEventToAudit( + toolEvent({ + runId, + sessionKey: "agent:main:sandbox:temporary", + sessionId: "session-sandbox", + agentId: "main", + }), + ); + + expect(projected).toMatchObject({ + actorId: "support", + agentId: "support", + sessionKey: "agent:support:channel:customer", + sessionId: "session-canonical", + }); + }); + + it("prefers authoritative tool lifecycle time over diagnostic observation time", () => { + const sourceTimestampMs = 1_750_000_000_000; + const projected = projectToolExecutionEventToAudit( + toolEvent({ ts: sourceTimestampMs + 30_000, sourceTimestampMs }), + ); + + expect(projected?.occurredAt).toBe(sourceTimestampMs); + expect(projectToolExecutionEventToAudit(toolEvent({ ts: sourceTimestampMs }))?.occurredAt).toBe( + sourceTimestampMs, + ); + }); + + it("prefers authoritative run lifecycle time over listener observation time", () => { + const startedAt = 1_750_000_000_000; + const endedAt = startedAt + 1_000; + const observedAt = endedAt + 30_000; + + expect( + projectAgentEventToAudit(agentEvent({ ts: observedAt, data: { phase: "start", startedAt } })) + ?.occurredAt, + ).toBe(startedAt); + expect( + projectAgentEventToAudit( + agentEvent({ + seq: 2, + ts: observedAt, + data: { phase: "end", startedAt, endedAt }, + }), + )?.occurredAt, + ).toBe(endedAt); + expect( + projectAgentEventToAudit( + agentEvent({ ts: observedAt, data: { phase: "start", startedAt: "invalid" } }), + )?.occurredAt, + ).toBe(observedAt); + }); + + it("omits prompt, arguments, results, and raw errors from run and tool records", () => { + const secret = "super-secret-payload"; + projectAgentEventToAudit(agentEvent({ data: { phase: "start", prompt: secret }, seq: 1 })); + const started = projectToolExecutionEventToAudit( + toolEvent({ seq: 2, sessionKey: undefined, sessionId: undefined }), + ); + const failed = projectToolExecutionEventToAudit( + toolEvent({ + type: "tool.execution.error", + seq: 3, + sessionKey: undefined, + sessionId: undefined, + durationMs: 10, + errorCategory: secret, + errorCode: secret, + }), + ); + + expect(started).toMatchObject({ + status: "started", + agentId: "coder", + sessionKey: "agent:coder:main", + toolName: "exec", + }); + expect(failed).toMatchObject({ + status: "failed", + errorCode: "tool_failed", + action: "tool.action.finished", + }); + expect(JSON.stringify({ started, failed })).not.toContain(secret); + expect(started).not.toHaveProperty("args"); + expect(failed).not.toHaveProperty("result"); + expect(failed).not.toHaveProperty("error"); + }); + + it("redacts provider-controlled tool identities at the durable boundary", () => { + const secret = `secret-${"x".repeat(600)}`; + const projected = projectToolExecutionEventToAudit( + toolEvent({ toolName: secret, toolCallId: secret }), + ); + const repeated = projectToolExecutionEventToAudit( + toolEvent({ toolName: secret, toolCallId: secret }), + ); + + expect(projected).toMatchObject({ + toolName: "unknown", + toolCallId: expect.stringMatching(/^sha256:[a-f0-9]{64}$/u), + }); + expect(repeated?.toolCallId).toBe(projected?.toolCallId); + expect(JSON.stringify(projected)).not.toContain(secret); + }); + + it.each([ + [{ phase: "error", error: "raw failure" }, "failed", "run_failed"], + [{ phase: "end", aborted: true }, "cancelled", "run_cancelled"], + [ + { phase: "end", aborted: true, stopReason: "aborted", providerStarted: true }, + "cancelled", + "run_cancelled", + ], + [ + { phase: "end", aborted: true, stopReason: "rpc", providerStarted: true }, + "cancelled", + "run_cancelled", + ], + [ + { phase: "end", aborted: true, stopReason: "stop", providerStarted: true }, + "cancelled", + "run_cancelled", + ], + [ + { phase: "end", aborted: true, stopReason: "relay-closed", status: "cancelled" }, + "cancelled", + "run_cancelled", + ], + [{ phase: "end", aborted: true, livenessState: "abandoned" }, "cancelled", "run_cancelled"], + [{ phase: "error", timeoutPhase: "provider" }, "timed_out", "run_timed_out"], + [{ phase: "error", livenessState: "blocked" }, "blocked", "run_blocked"], + [{ phase: "end", livenessState: "abandoned", replayInvalid: true }, "failed", "run_failed"], + ] as const)("classifies terminal run metadata %#", (data, status, errorCode) => { + const projected = projectAgentEventToAudit(agentEvent({ seq: 4, data, stream: "lifecycle" })); + expect(projected).toMatchObject({ + action: "agent.run.finished", + status, + errorCode, + }); + expect(projected).not.toHaveProperty("error"); + }); + + it("preserves timeout precedence when terminal cleanup is also aborted", () => { + const projected = projectAgentEventToAudit( + agentEvent({ + data: { phase: "end", aborted: true, stopReason: "timeout", timeoutPhase: "provider" }, + }), + ); + + expect(projected).toMatchObject({ status: "timed_out", errorCode: "run_timed_out" }); + }); + + it.each([ + ["tool.execution.completed", "succeeded", undefined], + ["tool.execution.error", "failed", "tool_failed"], + ["tool.execution.blocked", "blocked", "tool_blocked"], + ] as const)("classifies trusted tool terminal event %s", (type, status, errorCode) => { + const projected = projectToolExecutionEventToAudit( + toolEvent({ + type, + ...(type === "tool.execution.completed" || type === "tool.execution.error" + ? { durationMs: 10 } + : { deniedReason: "policy", reason: "secret detail" }), + ...(type === "tool.execution.error" ? { errorCategory: "test" } : {}), + }), + ); + + expect(projected).toMatchObject({ status }); + expect(projected?.errorCode).toBe(errorCode); + expect(projected).not.toHaveProperty("reason"); + }); + + it("does not project pre-invocation schema quarantine as a tool action", () => { + const projected = projectToolExecutionEventToAudit( + toolEvent({ + type: "tool.execution.blocked", + toolCallId: undefined, + deniedReason: "unsupported_tool_schema", + reason: "unsupported input schema", + }), + ); + + expect(projected).toBeUndefined(); + }); + + it.each(["aborted", "AbortError", "cancelled"])( + "classifies trusted tool error category %s as cancellation", + (errorCategory) => { + const projected = projectToolExecutionEventToAudit( + toolEvent({ + type: "tool.execution.error", + durationMs: 10, + errorCategory, + }), + ); + + expect(projected).toMatchObject({ status: "cancelled", errorCode: "tool_cancelled" }); + }, + ); + + it.each([ + ["cancelled", "runtime_tool_error", "cancelled", "tool_cancelled"], + ["timed_out", "aborted", "timed_out", "tool_timed_out"], + ["failed", "AbortError", "failed", "tool_failed"], + ] as const)( + "projects trusted tool terminal reason %s ahead of error category %s", + (terminalReason, errorCategory, status, errorCode) => { + const projected = projectToolExecutionEventToAudit( + toolEvent({ + type: "tool.execution.error", + durationMs: 10, + errorCategory, + terminalReason, + }), + ); + + expect(projected).toMatchObject({ status, errorCode }); + }, + ); + + it.each(["failed", "cancelled", "timed_out"] as const)( + "keeps an unavailable tool outcome explicitly unknown despite %s provenance", + (terminalReason) => { + const projected = projectToolExecutionEventToAudit( + toolEvent({ + type: "tool.execution.error", + durationMs: 10, + errorCategory: "codex_native_tool_outcome_unknown", + errorCode: "tool_outcome_unknown", + terminalReason, + }), + ); + + expect(projected).toMatchObject({ + status: "unknown", + errorCode: "tool_outcome_unknown", + }); + }, + ); + + it("keeps the trusted tool lifecycle active when optional diagnostics are disabled", () => { + const seen: TrustedToolExecutionEvent[] = []; + const stop = onTrustedToolExecutionEvent((event) => seen.push(event)); + setDiagnosticsEnabledForProcess(false); + + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId: "run-disabled-diagnostics", + toolName: "message", + toolCallId: "call-1", + }); + + stop(); + expect(seen).toMatchObject([ + { + type: "tool.execution.started", + runId: "run-disabled-diagnostics", + toolName: "message", + }, + ]); + }); + + it("preserves exact identifiers without capturing content", () => { + const runId = `run-${"x".repeat(600)}`; + const projected = projectAgentEventToAudit(agentEvent({ runId })); + + expect(projected?.runId).toBe(runId); + }); + + it("settles an error followed by a cleanup end as one failed outcome", async () => { + const inputs: AuditEventInput[] = []; + const writer: AuditEventWriter = { + ready: Promise.resolve(), + record: (input) => { + inputs.push(input); + return true; + }, + stop: async () => {}, + }; + const recorder = createAgentEventAuditRecorder({ writer }); + const lifecycleGeneration = "gateway-1"; + + recorder.record(agentEvent({ lifecycleGeneration, seq: 1 })); + recorder.record( + agentEvent({ + lifecycleGeneration, + seq: 2, + data: { phase: "error", error: "request failed" }, + }), + ); + recorder.record(agentEvent({ lifecycleGeneration, seq: 3, data: { phase: "end" } })); + await recorder.stop(); + + expect(inputs.map(({ action, status }) => ({ action, status }))).toEqual([ + { action: "agent.run.started", status: "started" }, + { action: "agent.run.finished", status: "failed" }, + ]); + }); + + it("keeps one start when a retry cancels a pending terminal", async () => { + const inputs: AuditEventInput[] = []; + const writer: AuditEventWriter = { + ready: Promise.resolve(), + record: (input) => { + inputs.push(input); + return true; + }, + stop: async () => {}, + }; + const recorder = createAgentEventAuditRecorder({ writer, terminalSettleMs: 60_000 }); + const lifecycleGeneration = "gateway-retry"; + + recorder.record(agentEvent({ lifecycleGeneration, seq: 1 })); + recorder.record(agentEvent({ lifecycleGeneration, seq: 2, data: { phase: "error" } })); + recorder.record(agentEvent({ lifecycleGeneration, seq: 3 })); + recorder.record(agentEvent({ lifecycleGeneration, seq: 4, data: { phase: "end" } })); + recorder.record(agentEvent({ lifecycleGeneration, seq: 5 })); + recorder.record(agentEvent({ lifecycleGeneration, seq: 6, data: { phase: "end" } })); + + expect(inputs.map(({ action, status }) => ({ action, status }))).toEqual([ + { action: "agent.run.started", status: "started" }, + { action: "agent.run.finished", status: "succeeded" }, + { action: "agent.run.started", status: "started" }, + { action: "agent.run.finished", status: "succeeded" }, + ]); + await recorder.stop(); + }); + + it("persists definitive successful terminals immediately in source order", async () => { + const inputs: AuditEventInput[] = []; + const writer: AuditEventWriter = { + ready: Promise.resolve(), + record: (input) => { + inputs.push(input); + return true; + }, + stop: async () => {}, + }; + const recorder = createAgentEventAuditRecorder({ writer, terminalSettleMs: 60_000 }); + + recorder.record(agentEvent({ seq: 1 })); + recorder.record(agentEvent({ seq: 2, data: { phase: "end" } })); + recorder.recordTool( + toolEvent({ + type: "tool.execution.completed", + seq: 3, + durationMs: 1, + }), + ); + + expect(inputs.map(({ action, status }) => ({ action, status }))).toEqual([ + { action: "agent.run.started", status: "started" }, + { action: "agent.run.finished", status: "succeeded" }, + { action: "tool.action.finished", status: "succeeded" }, + ]); + await recorder.stop(); + }); + + it("merges multiple terminal observations through the canonical outcome contract", async () => { + const inputs: AuditEventInput[] = []; + const writer: AuditEventWriter = { + ready: Promise.resolve(), + record: (input) => { + inputs.push(input); + return true; + }, + stop: async () => {}, + }; + const recorder = createAgentEventAuditRecorder({ writer }); + + recorder.record(agentEvent({ seq: 1 })); + recorder.record(agentEvent({ seq: 2, data: { phase: "error" } })); + recorder.record(agentEvent({ seq: 3, data: { phase: "end", aborted: true } })); + await recorder.stop(); + + expect(inputs.at(-1)).toMatchObject({ + action: "agent.run.finished", + status: "cancelled", + errorCode: "run_cancelled", + }); + }); +}); diff --git a/src/auto-reply/reply/acp-projector.ts b/src/auto-reply/reply/acp-projector.ts index 49da6b6baadc..a38d49b61f4b 100644 --- a/src/auto-reply/reply/acp-projector.ts +++ b/src/auto-reply/reply/acp-projector.ts @@ -4,6 +4,7 @@ import { normalizeOptionalLowercaseString, normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; +import { resolveAcpToolTerminalOutcome } from "../../acp/tool-status.js"; import { EmbeddedBlockChunker } from "../../agents/embedded-agent-block-chunker.js"; import { formatToolSummary, resolveToolDisplay } from "../../agents/tool-display.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; @@ -24,7 +25,6 @@ const ACP_LIVE_IDLE_MIN_CHARS = 80; const ACP_LIVE_SOFT_FLUSH_CHARS = 220; const ACP_LIVE_HARD_FLUSH_CHARS = 480; -const TERMINAL_TOOL_STATUSES = new Set(["completed", "failed", "cancelled", "done", "error"]); const HIDDEN_BOUNDARY_TAGS = new Set(["tool_call", "tool_call_update"]); type AcpProjectedDeliveryMeta = { @@ -349,7 +349,7 @@ export function createAcpReplyProjector(params: { return; } const status = normalizeToolStatus(event.status); - const isTerminal = status ? TERMINAL_TOOL_STATUSES.has(status) : false; + const isTerminal = resolveAcpToolTerminalOutcome(status) !== undefined; pendingHiddenBoundary = pendingHiddenBoundary || event.tag === "tool_call" || isTerminal; }; @@ -367,7 +367,7 @@ export function createAcpReplyProjector(params: { const hash = hashText(renderedToolSummary); const toolCallId = normalizeOptionalString(event.toolCallId); const status = normalizeToolStatus(event.status); - const isTerminal = status ? TERMINAL_TOOL_STATUSES.has(status) : false; + const isTerminal = resolveAcpToolTerminalOutcome(status) !== undefined; const isStart = status === "in_progress" || event.tag === "tool_call"; if (settings.repeatSuppression) { diff --git a/src/auto-reply/reply/agent-lifecycle-terminal.ts b/src/auto-reply/reply/agent-lifecycle-terminal.ts index 13b85e3f4444..3180d6f2f189 100644 --- a/src/auto-reply/reply/agent-lifecycle-terminal.ts +++ b/src/auto-reply/reply/agent-lifecycle-terminal.ts @@ -42,9 +42,10 @@ export function createAgentLifecycleTerminalBackstop(params: { sessionKey?: string; startedAt?: number; getLifecycleGeneration: () => string; - resolveAbortLifecycleFields: () => { + resolveTerminationFields: (error?: unknown) => { aborted?: true; stopReason?: string; + timeoutPhase?: string; }; }): AgentLifecycleTerminalBackstop { let terminalEmitted = false; @@ -74,8 +75,10 @@ export function createAgentLifecycleTerminalBackstop(params: { return; } terminalEmitted = true; - const abortLifecycleFields = params.resolveAbortLifecycleFields(); - const restartAbort = abortLifecycleFields.stopReason === AGENT_RUN_RESTART_ABORT_STOP_REASON; + const terminationFields = params.resolveTerminationFields( + phase === "error" ? resultOrError : undefined, + ); + const restartAbort = terminationFields.stopReason === AGENT_RUN_RESTART_ABORT_STOP_REASON; const data: Record = { ...deferredTerminalMetadata, phase: restartAbort ? "end" : phase, @@ -87,18 +90,18 @@ export function createAgentLifecycleTerminalBackstop(params: { data.stopReason = AGENT_RUN_RESTART_ABORT_STOP_REASON; } else if (phase === "error") { data.error = formatErrorMessage(resultOrError); - Object.assign(data, abortLifecycleFields); + Object.assign(data, terminationFields); } else { const meta = resultOrError && typeof resultOrError === "object" && "meta" in resultOrError ? (resultOrError as { meta?: Record }).meta : undefined; Object.assign(data, resolveAgentLifecycleTerminalMetadata(meta)); - if (abortLifecycleFields.aborted === true) { + if (terminationFields.aborted === true) { data.aborted = true; } - if (abortLifecycleFields.stopReason && !readStringValue(data.stopReason)) { - data.stopReason = abortLifecycleFields.stopReason; + if (terminationFields.stopReason && !readStringValue(data.stopReason)) { + data.stopReason = terminationFields.stopReason; } } if (extraData) { diff --git a/src/auto-reply/reply/agent-runner-cli-dispatch.test.ts b/src/auto-reply/reply/agent-runner-cli-dispatch.test.ts index 2f69d68f7531..677745ff78d8 100644 --- a/src/auto-reply/reply/agent-runner-cli-dispatch.test.ts +++ b/src/auto-reply/reply/agent-runner-cli-dispatch.test.ts @@ -1,6 +1,7 @@ // Tests CLI dispatch arguments and runtime selection for agent runner turns. import { afterEach, describe, expect, it, vi } from "vitest"; import type { EmbeddedAgentRunResult } from "../../agents/embedded-agent-runner/types.js"; +import { FailoverError } from "../../agents/failover-error.js"; import { createAgentRunRestartAbortError } from "../../agents/run-termination.js"; import { emitAgentEvent, @@ -199,6 +200,7 @@ describe("runCliAgentWithLifecycle", () => { const events: Array<{ stream?: string; lifecycleGeneration?: string; + agentId?: string; data?: Record; }> = []; const lifecycleGeneration = getAgentEventLifecycleGeneration(); @@ -219,6 +221,7 @@ describe("runCliAgentWithLifecycle", () => { provider: "claude-cli", runParams: { sessionId: "session-1", + agentId: "support", sessionFile: "/tmp/session.jsonl", workspaceDir: "/tmp/workspace", prompt: "hello", @@ -238,6 +241,7 @@ describe("runCliAgentWithLifecycle", () => { expect( lifecycleEvents.every((event) => event.lifecycleGeneration === lifecycleGeneration), ).toBe(true); + expect(lifecycleEvents.every((event) => event.agentId === "support")).toBe(true); }); it("preserves restart ownership when the CLI resolves after cancellation", async () => { @@ -286,6 +290,44 @@ describe("runCliAgentWithLifecycle", () => { expect(events.some((event) => event.stream === "assistant")).toBe(false); }); + it("attributes a structured CLI watchdog timeout on the terminal event", async () => { + const events: Array<{ stream?: string; data?: Record }> = []; + const stop = onAgentEvent((event) => { + if (event.runId === "run-timeout") { + events.push(event); + } + }); + cliDispatchState.runCliAgentMock.mockRejectedValueOnce( + new FailoverError("CLI produced no output", { reason: "timeout" }), + ); + + await expect( + runCliAgentWithLifecycle({ + runId: "run-timeout", + provider: "claude-cli", + runParams: { + sessionId: "session-1", + sessionFile: "/tmp/session.jsonl", + workspaceDir: "/tmp/workspace", + prompt: "hello", + provider: "claude-cli", + model: "claude", + thinkLevel: "off", + timeoutMs: 1_000, + runId: "run-timeout", + }, + }), + ).rejects.toThrow("CLI produced no output"); + stop(); + + expect( + events.find((event) => event.stream === "lifecycle" && event.data?.phase === "error")?.data, + ).toMatchObject({ + stopReason: "timeout", + timeoutPhase: "provider", + }); + }); + it("propagates yielded result metadata on lifecycle end", async () => { const events: Array<{ stream?: string; data?: Record }> = []; const stop = onAgentEvent((event) => { diff --git a/src/auto-reply/reply/agent-runner-cli-dispatch.ts b/src/auto-reply/reply/agent-runner-cli-dispatch.ts index 12b32767e876..a566bba77030 100644 --- a/src/auto-reply/reply/agent-runner-cli-dispatch.ts +++ b/src/auto-reply/reply/agent-runner-cli-dispatch.ts @@ -16,6 +16,7 @@ import { import { isAgentRunRestartAbortReason, resolveAgentRunAbortLifecycleFields, + resolveAgentRunErrorLifecycleFields, } from "../../agents/run-termination.js"; import type { SessionEntry } from "../../config/sessions.js"; import { updateSessionEntry } from "../../config/sessions/session-accessor.js"; @@ -506,6 +507,7 @@ async function runCliAgentWithLifecycleInternal( if (emitLifecycleStart) { emitAgentEvent({ runId: params.runId, + ...(params.runParams.agentId ? { agentId: params.runParams.agentId } : {}), ...(params.runParams.sessionKey ? { sessionKey: params.runParams.sessionKey } : {}), ...(params.runParams.sessionId ? { sessionId: params.runParams.sessionId } : {}), ...(params.lifecycleGeneration ? { lifecycleGeneration: params.lifecycleGeneration } : {}), @@ -590,6 +592,7 @@ async function runCliAgentWithLifecycleInternal( if (emitLifecycleTerminal) { emitAgentEvent({ runId: params.runId, + ...(params.runParams.agentId ? { agentId: params.runParams.agentId } : {}), ...(params.runParams.sessionKey ? { sessionKey: params.runParams.sessionKey } : {}), ...(params.runParams.sessionId ? { sessionId: params.runParams.sessionId } : {}), ...(params.lifecycleGeneration ? { lifecycleGeneration: params.lifecycleGeneration } : {}), @@ -611,6 +614,7 @@ async function runCliAgentWithLifecycleInternal( if (emitLifecycleTerminal) { emitAgentEvent({ runId: params.runId, + ...(params.runParams.agentId ? { agentId: params.runParams.agentId } : {}), ...(params.runParams.sessionKey ? { sessionKey: params.runParams.sessionKey } : {}), ...(params.runParams.sessionId ? { sessionId: params.runParams.sessionId } : {}), ...(params.lifecycleGeneration ? { lifecycleGeneration: params.lifecycleGeneration } : {}), @@ -620,7 +624,7 @@ async function runCliAgentWithLifecycleInternal( startedAt, endedAt: Date.now(), error: String(err), - ...resolveAgentRunAbortLifecycleFields(params.runParams.abortSignal), + ...resolveAgentRunErrorLifecycleFields(err, params.runParams.abortSignal), }, }); lifecycleTerminalEmitted = true; @@ -636,6 +640,7 @@ async function runCliAgentWithLifecycleInternal( if (emitLifecycleTerminal && !lifecycleTerminalEmitted) { emitAgentEvent({ runId: params.runId, + ...(params.runParams.agentId ? { agentId: params.runParams.agentId } : {}), ...(params.runParams.sessionKey ? { sessionKey: params.runParams.sessionKey } : {}), ...(params.runParams.sessionId ? { sessionId: params.runParams.sessionId } : {}), ...(params.lifecycleGeneration ? { lifecycleGeneration: params.lifecycleGeneration } : {}), diff --git a/src/auto-reply/reply/agent-runner-execution.test.ts b/src/auto-reply/reply/agent-runner-execution.test.ts index ff473fa0116c..ca5d686b5974 100644 --- a/src/auto-reply/reply/agent-runner-execution.test.ts +++ b/src/auto-reply/reply/agent-runner-execution.test.ts @@ -2319,6 +2319,47 @@ describe("runAgentTurnWithFallback", () => { expect(lifecycleEvents.some((event) => event.data.phase === "end")).toBe(false); }); + it("preserves a CLI watchdog timeout through the lifecycle backstop", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + try { + return await params.run("codex-cli", "gpt-5.4"); + } catch (cause) { + throw new Error("All model fallback candidates failed", { cause }); + } + }); + state.runCliAgentMock.mockRejectedValueOnce( + new FailoverError("CLI produced no output", { reason: "timeout" }), + ); + const followupRun = createFollowupRun(); + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + opts: { runId: "run-cli-timeout" }, + }), + ); + + expect( + emitAgentEvent.mock.calls + .map((call) => call[0]) + .find( + (event) => + event.runId === "run-cli-timeout" && + event.stream === "lifecycle" && + event.data.phase === "error", + )?.data, + ).toMatchObject({ + stopReason: "timeout", + timeoutPhase: "provider", + fallbackExhaustedFailure: true, + }); + }); + it("keeps fallback auth available for later same-provider fallback models", async () => { const probe = { provider: "anthropic", diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts index 03caa4d0659a..8e3572f1593a 100644 --- a/src/auto-reply/reply/agent-runner-execution.ts +++ b/src/auto-reply/reply/agent-runner-execution.ts @@ -68,7 +68,7 @@ import { AGENT_RUN_RESTART_ABORT_STOP_REASON, createAgentRunRestartAbortError, isAgentRunRestartAbortReason, - resolveAgentRunAbortLifecycleFields, + resolveAgentRunErrorLifecycleFields, } from "../../agents/run-termination.js"; import { buildAgentRuntimeOutcomePlan } from "../../agents/runtime-plan/build.js"; import { resolveGroupSessionKey, type SessionEntry } from "../../config/sessions.js"; @@ -1774,6 +1774,7 @@ async function runAgentTurnWithFallbackInternal( registerAgentRunContext(runId, { sessionKey: params.sessionKey, ...(params.followupRun.run.sessionId ? { sessionId: params.followupRun.run.sessionId } : {}), + agentId: params.followupRun.run.agentId, lifecycleGeneration, verboseLevel: params.resolvedVerboseLevel, isHeartbeat: params.isHeartbeat, @@ -2379,8 +2380,8 @@ async function runAgentTurnWithFallbackInternal( sessionKey: params.sessionKey, startedAt: cliLifecycleStartedAt, getLifecycleGeneration: () => lifecycleGeneration, - resolveAbortLifecycleFields: () => ({ - ...resolveAgentRunAbortLifecycleFields(runAbortSignal), + resolveTerminationFields: (error) => ({ + ...resolveAgentRunErrorLifecycleFields(error, runAbortSignal), ...(isReplyOperationRestartAbort(params.replyOperation) ? { aborted: true as const, @@ -2619,8 +2620,8 @@ async function runAgentTurnWithFallbackInternal( runId, sessionKey: params.sessionKey, getLifecycleGeneration: () => lifecycleGeneration, - resolveAbortLifecycleFields: () => ({ - ...resolveAgentRunAbortLifecycleFields(runAbortSignal), + resolveTerminationFields: (error) => ({ + ...resolveAgentRunErrorLifecycleFields(error, runAbortSignal), ...(isReplyOperationRestartAbort(params.replyOperation) ? { aborted: true as const, @@ -3485,7 +3486,7 @@ async function runAgentTurnWithFallbackInternal( ? params.opts.abortSignal : undefined; const abortLifecycleFields = { - ...resolveAgentRunAbortLifecycleFields(abortedSignal), + ...resolveAgentRunErrorLifecycleFields(err, abortedSignal), ...(isReplyOperationRestartAbort(params.replyOperation) ? { aborted: true as const, diff --git a/src/auto-reply/reply/dispatch-acp.test.ts b/src/auto-reply/reply/dispatch-acp.test.ts index f73d8dcf02c4..202a0a3d7fd5 100644 --- a/src/auto-reply/reply/dispatch-acp.test.ts +++ b/src/auto-reply/reply/dispatch-acp.test.ts @@ -32,6 +32,13 @@ const managerMocks = vi.hoisted(() => ({ })), })); +const auditMocks = vi.hoisted(() => ({ + emitAcpLifecycleStart: vi.fn(), + emitAcpRuntimeEvent: vi.fn(), + emitAcpLifecycleEnd: vi.fn(), + emitAcpLifecycleError: vi.fn(), +})); + const policyMocks = vi.hoisted(() => ({ resolveAcpDispatchPolicyError: vi.fn<(cfg: OpenClawConfig) => AcpRuntimeError | null>(() => null), resolveAcpAgentPolicyError: vi.fn<(cfg: OpenClawConfig, agent: string) => AcpRuntimeError | null>( @@ -116,6 +123,18 @@ vi.mock("./dispatch-acp-manager.runtime.js", () => ({ }), })); +vi.mock("../../agents/command/attempt-execution.runtime.js", () => ({ + createAcpToolLifecycleTracker: () => ({ + active: new Map(), + terminalToolCallIds: new Set(), + saturated: false, + }), + emitAcpLifecycleStart: auditMocks.emitAcpLifecycleStart, + emitAcpRuntimeEvent: auditMocks.emitAcpRuntimeEvent, + emitAcpLifecycleEnd: auditMocks.emitAcpLifecycleEnd, + emitAcpLifecycleError: auditMocks.emitAcpLifecycleError, +})); + vi.mock("../../acp/policy.js", () => ({ resolveAcpDispatchPolicyError: (cfg: OpenClawConfig) => policyMocks.resolveAcpDispatchPolicyError(cfg), @@ -304,6 +323,7 @@ function createAcpConfigWithVisibleToolTags(): OpenClawConfig { async function runDispatch(params: { bodyForAgent: string; + runId?: string; cfg?: OpenClawConfig; dispatcher?: ReplyDispatcher; shouldRouteToOriginating?: boolean; @@ -311,6 +331,7 @@ async function runDispatch(params: { originatingTo?: string; onReplyStart?: () => void; images?: Array<{ data: string; mimeType: string }>; + abortSignal?: AbortSignal; ctxOverrides?: Record; sessionKeyOverride?: string; suppressUserDelivery?: boolean; @@ -329,8 +350,10 @@ async function runDispatch(params: { }), cfg: params.cfg ?? createAcpTestConfig(), dispatcher: params.dispatcher ?? createDispatcher().dispatcher, + ...(params.runId ? { runId: params.runId } : {}), sessionKey: targetSessionKey, images: params.images, + abortSignal: params.abortSignal, inboundAudio: false, suppressUserDelivery: params.suppressUserDelivery, suppressReplyLifecycle: params.suppressReplyLifecycle, @@ -434,6 +457,10 @@ function expectRoutedPayload(callIndex: number, payload: Partial) describe("tryDispatchAcpReply", () => { beforeEach(() => { + auditMocks.emitAcpLifecycleStart.mockReset(); + auditMocks.emitAcpRuntimeEvent.mockReset(); + auditMocks.emitAcpLifecycleEnd.mockReset(); + auditMocks.emitAcpLifecycleError.mockReset(); managerMocks.resolveSession.mockReset(); managerMocks.runTurn.mockReset(); managerMocks.runTurn.mockImplementation( @@ -476,6 +503,74 @@ describe("tryDispatchAcpReply", () => { globalThis.fetch = originalFetch; }); + it("projects normal ACP dispatch lifecycle and tool events into audit diagnostics", async () => { + setReadyAcpResolution(); + mockToolLifecycleTurn("tool-audit"); + + await runDispatch({ bodyForAgent: "audit this turn" }); + + expect(auditMocks.emitAcpLifecycleStart).toHaveBeenCalledWith( + expect.objectContaining({ + runId: expect.any(String), + sessionKey, + startedAt: expect.any(Number), + auditOnly: true, + }), + ); + expect(auditMocks.emitAcpRuntimeEvent).toHaveBeenCalledTimes(3); + expect(auditMocks.emitAcpRuntimeEvent).toHaveBeenCalledWith( + expect.objectContaining({ + runId: expect.any(String), + sessionKey, + auditOnly: true, + event: expect.objectContaining({ type: "tool_call", toolCallId: "tool-audit" }), + }), + ); + expect(auditMocks.emitAcpLifecycleEnd).toHaveBeenCalledWith( + expect.objectContaining({ runId: expect.any(String), sessionKey, auditOnly: true }), + ); + expect(auditMocks.emitAcpLifecycleError).not.toHaveBeenCalled(); + }); + + it("keeps caller-owned run ids on the shared lifecycle path", async () => { + setReadyAcpResolution(); + + await runDispatch({ bodyForAgent: "audit this turn", runId: "caller-run" }); + + expect(auditMocks.emitAcpLifecycleStart).toHaveBeenCalledWith( + expect.objectContaining({ runId: "caller-run", auditOnly: false }), + ); + expect(auditMocks.emitAcpLifecycleEnd).toHaveBeenCalledWith( + expect.objectContaining({ runId: "caller-run", auditOnly: false }), + ); + }); + + it("keeps audit run ids unique when channel message ids repeat", async () => { + setReadyAcpResolution(); + + await runDispatch({ + bodyForAgent: "first turn", + ctxOverrides: { MessageSid: "channel-local-1" }, + }); + await runDispatch({ + bodyForAgent: "second turn", + ctxOverrides: { MessageSid: "channel-local-1" }, + }); + + const auditRunIds = [0, 1].map( + (index) => + requireRecord( + mockArg(auditMocks.emitAcpLifecycleStart, index, 0, `audit start ${index}`), + "audit start", + ).runId, + ); + expect(new Set(auditRunIds).size).toBe(2); + expect([runTurnCall(0).requestId, runTurnCall(1).requestId]).toEqual([ + "channel-local-1", + "channel-local-1", + ]); + }); + it("routes default ACP output to the originating channel as a final reply", async () => { setReadyAcpResolution(); mockRoutedTextTurn("hello"); @@ -682,6 +777,55 @@ describe("tryDispatchAcpReply", () => { expect(managerMocks.runTurn).not.toHaveBeenCalled(); expect(onReplyStart).not.toHaveBeenCalled(); + expect(auditMocks.emitAcpLifecycleStart).not.toHaveBeenCalled(); + expect(auditMocks.emitAcpLifecycleEnd).not.toHaveBeenCalled(); + expect(auditMocks.emitAcpLifecycleError).not.toHaveBeenCalled(); + }); + + it("records cancellation only after ACP output flushing", async () => { + setReadyAcpResolution(); + const abortController = new AbortController(); + managerMocks.runTurn.mockImplementationOnce( + async ({ onEvent }: { onEvent: (event: unknown) => Promise }) => { + await onEvent({ type: "text_delta", text: "partial", tag: "agent_message_chunk" }); + await onEvent({ type: "done", status: "cancelled" }); + abortController.abort(); + }, + ); + + await runDispatch({ + bodyForAgent: "cancel this turn", + abortSignal: abortController.signal, + }); + + expect(auditMocks.emitAcpLifecycleEnd).toHaveBeenCalledWith( + expect.objectContaining({ + abortSignal: abortController.signal, + resultStatus: "cancelled", + }), + ); + expect(auditMocks.emitAcpLifecycleError).not.toHaveBeenCalled(); + }); + + it("records an ACP error when output finalization fails", async () => { + setReadyAcpResolution(); + mockVisibleTextTurn("visible output"); + const { dispatcher } = createDispatcher(); + vi.mocked(dispatcher.waitForIdle) + .mockRejectedValueOnce(new Error("output settlement failed")) + .mockResolvedValue(undefined); + + await runDispatch({ + bodyForAgent: "finalize this turn", + dispatcher, + }); + + expect(auditMocks.emitAcpLifecycleEnd).not.toHaveBeenCalled(); + expect(auditMocks.emitAcpLifecycleError).toHaveBeenCalledWith( + expect.objectContaining({ + error: expect.objectContaining({ message: "output settlement failed" }), + }), + ); }); it("skips media understanding for text-only ACP turns", async () => { @@ -1414,6 +1558,11 @@ describe("tryDispatchAcpReply", () => { "ACP dispatch is disabled by policy.", ); expect(bindingServiceMocks.unbind).not.toHaveBeenCalled(); + expect(auditMocks.emitAcpLifecycleStart).toHaveBeenCalledOnce(); + expect(auditMocks.emitAcpLifecycleError).toHaveBeenCalledWith( + expect.objectContaining({ terminalOutcome: "blocked" }), + ); + expect(auditMocks.emitAcpLifecycleEnd).not.toHaveBeenCalled(); }); it("fails closed when ACP dispatch cannot enforce restrictive runtime toolsAllow", async () => { @@ -1429,6 +1578,25 @@ describe("tryDispatchAcpReply", () => { expect(managerMocks.runTurn).not.toHaveBeenCalled(); expect(dispatcherCall(dispatcher.sendFinalReply).isError).toBe(true); expect(dispatcherCall(dispatcher.sendFinalReply).text).toContain("runtime toolsAllow"); + expect(auditMocks.emitAcpLifecycleError).toHaveBeenCalledWith( + expect.objectContaining({ terminalOutcome: "blocked" }), + ); + }); + + it("audits ACP agent-policy rejections as blocked attempts", async () => { + setReadyAcpResolution(); + policyMocks.resolveAcpAgentPolicyError.mockReturnValue( + new AcpRuntimeError("ACP_SESSION_INIT_FAILED", "ACP agent is not allowed by policy."), + ); + + await runDispatch({ bodyForAgent: "test" }); + + expect(managerMocks.runTurn).not.toHaveBeenCalled(); + expect(auditMocks.emitAcpLifecycleStart).toHaveBeenCalledOnce(); + expect(auditMocks.emitAcpLifecycleError).toHaveBeenCalledWith( + expect.objectContaining({ terminalOutcome: "blocked" }), + ); + expect(auditMocks.emitAcpLifecycleEnd).not.toHaveBeenCalled(); }); it("allows wildcard runtime toolsAllow through ACP dispatch", async () => { diff --git a/src/auto-reply/reply/dispatch-acp.ts b/src/auto-reply/reply/dispatch-acp.ts index 36cff6f3fa20..92d821b3506c 100644 --- a/src/auto-reply/reply/dispatch-acp.ts +++ b/src/auto-reply/reply/dispatch-acp.ts @@ -18,7 +18,6 @@ import type { ChatType } from "../../channels/chat-type.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { TtsAutoMode } from "../../config/types.tts.js"; import { logVerbose } from "../../globals.js"; -import { emitAgentEvent } from "../../infra/agent-events.js"; import { isDiagnosticsEnabled } from "../../infra/diagnostic-events.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { generateSecureUuid } from "../../infra/secure-random.js"; @@ -53,6 +52,9 @@ import type { ReplyDispatchKind, ReplyDispatcher } from "./reply-dispatcher.type const dispatchAcpManagerRuntimeLoader = createLazyImportLoader( () => import("./dispatch-acp-manager.runtime.js"), ); +const dispatchAcpAuditRuntimeLoader = createLazyImportLoader( + () => import("../../agents/command/attempt-execution.runtime.js"), +); type OrderedAcpAttachment = { attachment: AcpTurnAttachment; @@ -101,6 +103,10 @@ function loadDispatchAcpManagerRuntime() { return dispatchAcpManagerRuntimeLoader.load(); } +function loadDispatchAcpAuditRuntime() { + return dispatchAcpAuditRuntimeLoader.load(); +} + function loadDispatchAcpSessionRuntime() { return dispatchAcpSessionRuntimeLoader.load(); } @@ -216,30 +222,14 @@ function finishAcpDispatchAttempt(params: { delivery: AcpDispatchDeliveryCoordinator; getStats: () => AcpDispatchStatsSnapshot; sessionKey: string; - runId?: string; startedAt: number; outcome: AcpDispatchOutcome; - lifecyclePhase?: "end" | "error"; recordProcessed: DispatchProcessedRecorder; markIdle: (reason: string) => void; }): AcpDispatchAttemptResult { const counts = params.dispatcher.getQueuedCounts(); params.delivery.applyRoutedCounts(counts); const acpStats = params.getStats(); - const runId = normalizeOptionalString(params.runId); - if (runId && params.lifecyclePhase) { - emitAgentEvent({ - runId, - sessionKey: params.sessionKey, - stream: "lifecycle", - data: { - phase: params.lifecyclePhase, - startedAt: params.startedAt, - endedAt: Date.now(), - ...(params.outcome.kind === "error" ? { error: params.outcome.error.message } : {}), - }, - }); - } if (params.outcome.kind === "ok") { logVerbose( `acp-dispatch: session=${params.sessionKey} outcome=ok latencyMs=${Date.now() - params.startedAt} queueDepth=${acpStats.turns.queueDepth} activeRuntimes=${acpStats.runtimeCache.activeSessions}`, @@ -535,34 +525,90 @@ export async function tryDispatchAcpReply(params: { }); const acpDispatchStartedAt = Date.now(); - const finishAttempt = (options: { - queuedFinal: boolean; - outcome: AcpDispatchOutcome; - lifecyclePhase?: "end" | "error"; - }) => + const finishAttempt = (options: { queuedFinal: boolean; outcome: AcpDispatchOutcome }) => finishAcpDispatchAttempt({ ...options, dispatcher: params.dispatcher, delivery, getStats: () => acpManager.getObservabilitySnapshot(params.cfg), sessionKey, - runId: params.runId, startedAt: acpDispatchStartedAt, recordProcessed: params.recordProcessed, markIdle: params.markIdle, }); + const requestId = resolveAcpRequestId(params.ctx); + const existingRunId = normalizeOptionalString(params.runId); + const auditOnly = existingRunId === undefined; + const auditRunId = existingRunId ?? generateSecureUuid(); + const auditRuntime = await loadDispatchAcpAuditRuntime(); + const auditToolTracker = auditRuntime.createAcpToolLifecycleTracker(); + let auditStarted = false; + let auditFinished = false; + let auditTerminalOutcome: "blocked" | undefined; + let auditStopReason: string | undefined; + let auditResultStatus: "completed" | "cancelled" | undefined; + const emitAuditStart = () => { + if (auditStarted) { + return; + } + auditStarted = true; + auditRuntime.emitAcpLifecycleStart({ + runId: auditRunId, + sessionKey: canonicalSessionKey, + agentId: acpAgentId, + startedAt: Date.now(), + auditOnly, + }); + }; + const emitAuditEnd = () => { + if (auditFinished) { + return; + } + emitAuditStart(); + auditFinished = true; + auditRuntime.emitAcpLifecycleEnd({ + runId: auditRunId, + toolTracker: auditToolTracker, + sessionKey: canonicalSessionKey, + agentId: acpAgentId, + ...(params.abortSignal ? { abortSignal: params.abortSignal } : {}), + ...(auditStopReason ? { stopReason: auditStopReason } : {}), + ...(auditResultStatus ? { resultStatus: auditResultStatus } : {}), + auditOnly, + }); + }; + const emitAuditError = (error: unknown) => { + if (auditFinished) { + return; + } + emitAuditStart(); + auditFinished = true; + auditRuntime.emitAcpLifecycleError({ + runId: auditRunId, + toolTracker: auditToolTracker, + sessionKey: canonicalSessionKey, + agentId: acpAgentId, + ...(params.abortSignal ? { abortSignal: params.abortSignal } : {}), + ...(auditTerminalOutcome ? { terminalOutcome: auditTerminalOutcome } : {}), + auditOnly, + error, + }); + }; try { const dispatchPolicyError = resolveAcpDispatchPolicyError(params.cfg); if (dispatchPolicyError) { + auditTerminalOutcome = "blocked"; throw dispatchPolicyError; } if (isRestrictiveRuntimeToolsAllow(params.toolsAllow)) { + auditTerminalOutcome = "blocked"; throw new AcpRuntimeError( "ACP_DISPATCH_DISABLED", "ACP dispatch cannot enforce runtime toolsAllow for this session; use an embedded runtime for restricted tool policy.", ); } if (acpResolution.kind === "stale") { + emitAuditError(acpResolution.error); await maybeUnbindStaleBoundConversations({ targetSessionKey: canonicalSessionKey, error: acpResolution.error, @@ -578,6 +624,7 @@ export async function tryDispatchAcpReply(params: { } const agentPolicyError = resolveAcpAgentPolicyError(params.cfg, resolvedAcpAgent); if (agentPolicyError) { + auditTerminalOutcome = "blocked"; throw agentPolicyError; } let extractedFileImages = params.extractedFileImages ?? []; @@ -654,6 +701,7 @@ export async function tryDispatchAcpReply(params: { return { queuedFinal: false, counts }; } + emitAuditStart(); try { await delivery.startReplyLifecycle(); } catch (error) { @@ -669,9 +717,24 @@ export async function tryDispatchAcpReply(params: { }), attachments: attachments.length > 0 ? attachments : undefined, mode: "prompt", - requestId: resolveAcpRequestId(params.ctx), + requestId, ...(params.abortSignal ? { signal: params.abortSignal } : {}), - onEvent: async (event) => await projector.onEvent(event), + onEvent: async (event) => { + auditRuntime.emitAcpRuntimeEvent({ + runId: auditRunId, + toolTracker: auditToolTracker, + sessionKey: canonicalSessionKey, + agentId: acpAgentId, + ...(params.abortSignal ? { abortSignal: params.abortSignal } : {}), + auditOnly, + event, + }); + if (event.type === "done") { + auditStopReason = event.stopReason; + auditResultStatus = event.status; + } + await projector.onEvent(event); + }, }); await projector.flush(true); @@ -680,6 +743,7 @@ export async function tryDispatchAcpReply(params: { delivery.applyRoutedCounts(counts); params.recordProcessed("completed", { reason: "acp_aborted" }); params.markIdle("message_aborted"); + emitAuditEnd(); return { queuedFinal, counts }; } try { @@ -712,18 +776,20 @@ export async function tryDispatchAcpReply(params: { shouldEmitResolvedIdentityNotice, })) || queuedFinal; - return finishAttempt({ + const result = finishAttempt({ queuedFinal, outcome: { kind: "ok" }, - lifecyclePhase: "end", }); + emitAuditEnd(); + return result; } catch (err) { - await projector.flush(true); const acpError = toAcpRuntimeError({ error: err, fallbackCode: "ACP_TURN_FAILED", fallbackMessage: "ACP turn failed before completion.", }); + emitAuditError(acpError); + await projector.flush(true); await maybeUnbindStaleBoundConversations({ targetSessionKey: canonicalSessionKey, error: acpError, @@ -736,7 +802,6 @@ export async function tryDispatchAcpReply(params: { return finishAttempt({ queuedFinal, outcome: { kind: "error", error: acpError }, - lifecyclePhase: "error", }); } } diff --git a/src/auto-reply/reply/dispatch-from-config.acp-abort.test.ts b/src/auto-reply/reply/dispatch-from-config.acp-abort.test.ts index 23edfb843430..319548e7deae 100644 --- a/src/auto-reply/reply/dispatch-from-config.acp-abort.test.ts +++ b/src/auto-reply/reply/dispatch-from-config.acp-abort.test.ts @@ -217,6 +217,7 @@ describe("dispatchReplyFromConfig ACP abort", () => { diagnosticMocks.logSessionStateChange.mockReset(); diagnosticMocks.markDiagnosticSessionProgress.mockReset(); agentEventMocks.emitAgentEvent.mockReset(); + agentEventMocks.emitAgentAuditEvent.mockReset(); agentEventMocks.onAgentEvent.mockReset().mockImplementation(() => () => {}); setNoAbort(); }); diff --git a/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts b/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts index 16ccba09e3d2..3ac2ea326052 100644 --- a/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts +++ b/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts @@ -118,6 +118,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => { runTurn: vi.fn(), })); agentEventMocks.emitAgentEvent.mockReset(); + agentEventMocks.emitAgentAuditEvent.mockReset(); agentEventMocks.onAgentEvent.mockReset().mockImplementation(() => () => {}); diagnosticMocks.logMessageQueued.mockReset(); diagnosticMocks.logMessageProcessed.mockReset(); diff --git a/src/auto-reply/reply/dispatch-from-config.shared.test-harness.ts b/src/auto-reply/reply/dispatch-from-config.shared.test-harness.ts index 3c25ec5f403d..953d8735501b 100644 --- a/src/auto-reply/reply/dispatch-from-config.shared.test-harness.ts +++ b/src/auto-reply/reply/dispatch-from-config.shared.test-harness.ts @@ -129,6 +129,7 @@ const acpManagerRuntimeMocks = vi.hoisted(() => ({ getAcpSessionManager: vi.fn(), })); const agentEventMocks = vi.hoisted(() => ({ + emitAgentAuditEvent: vi.fn(), emitAgentEvent: vi.fn(), onAgentEvent: vi.fn<(listener: unknown) => () => void>(() => () => {}), })); @@ -285,6 +286,7 @@ vi.mock("../../infra/outbound/session-binding-service.js", () => ({ }), })); vi.mock("../../infra/agent-events.js", () => ({ + emitAgentAuditEvent: (params: unknown) => agentEventMocks.emitAgentAuditEvent(params), emitAgentEvent: (params: unknown) => agentEventMocks.emitAgentEvent(params), onAgentEvent: (listener: unknown) => agentEventMocks.onAgentEvent(listener), })); diff --git a/src/auto-reply/reply/dispatch-from-config.test.ts b/src/auto-reply/reply/dispatch-from-config.test.ts index 9d8968dfcd24..cf55b332371d 100644 --- a/src/auto-reply/reply/dispatch-from-config.test.ts +++ b/src/auto-reply/reply/dispatch-from-config.test.ts @@ -191,6 +191,7 @@ const acpManagerRuntimeMocks = vi.hoisted(() => ({ getAcpSessionManager: vi.fn(), })); const agentEventMocks = vi.hoisted(() => ({ + emitAgentAuditEvent: vi.fn(), emitAgentEvent: vi.fn(), onAgentEvent: vi.fn<(listener: unknown) => () => void>(() => () => {}), })); @@ -531,6 +532,7 @@ vi.mock("../../bindings/records.js", () => ({ sessionBindingMocks.touch(...args), })); vi.mock("../../infra/agent-events.js", () => ({ + emitAgentAuditEvent: (params: unknown) => agentEventMocks.emitAgentAuditEvent(params), emitAgentEvent: (params: unknown) => agentEventMocks.emitAgentEvent(params), onAgentEvent: (listener: unknown) => agentEventMocks.onAgentEvent(listener), })); @@ -1127,6 +1129,7 @@ describe("dispatchReplyFromConfig", () => { acpMocks.getAcpRuntimeBackend.mockReset(); acpMocks.requireAcpRuntimeBackend.mockReset(); agentEventMocks.emitAgentEvent.mockReset(); + agentEventMocks.emitAgentAuditEvent.mockReset(); agentEventMocks.onAgentEvent.mockReset(); agentEventMocks.onAgentEvent.mockReturnValue(() => {}); sessionBindingMocks.listBySession.mockReset(); @@ -5869,7 +5872,7 @@ describe("dispatchReplyFromConfig", () => { stream?: unknown; }, ) - .find((event) => event.runId === "run-acp-lifecycle-end"); + .find((event) => event.runId === "run-acp-lifecycle-end" && event.data?.phase === "end"); expect(lifecycleEvent?.sessionKey).toBe("agent:codex-acp:session-1"); expect(lifecycleEvent?.stream).toBe("lifecycle"); expect(lifecycleEvent?.data?.phase).toBe("end"); @@ -5935,7 +5938,7 @@ describe("dispatchReplyFromConfig", () => { stream?: unknown; }, ) - .find((event) => event.runId === "run-acp-lifecycle-error"); + .find((event) => event.runId === "run-acp-lifecycle-error" && event.data?.phase === "error"); expect(lifecycleEvent?.sessionKey).toBe("agent:codex-acp:session-1"); expect(lifecycleEvent?.stream).toBe("lifecycle"); expect(lifecycleEvent?.data?.phase).toBe("error"); diff --git a/src/auto-reply/reply/followup-runner.ts b/src/auto-reply/reply/followup-runner.ts index 414c53f7e1fe..4637db8eebc1 100644 --- a/src/auto-reply/reply/followup-runner.ts +++ b/src/auto-reply/reply/followup-runner.ts @@ -31,7 +31,7 @@ import { resolveCliRuntimeExecutionProvider } from "../../agents/model-runtime-a import { isCliProvider } from "../../agents/model-selection-cli.js"; import { isAgentRunRestartAbortReason, - resolveAgentRunAbortLifecycleFields, + resolveAgentRunErrorLifecycleFields, } from "../../agents/run-termination.js"; import { buildAgentRuntimeDeliveryPlan, @@ -793,6 +793,7 @@ export function createFollowupRunner(params: { registerAgentRunContext(runId, { sessionKey: run.sessionKey, ...(run.sessionId ? { sessionId: run.sessionId } : {}), + agentId: run.agentId, lifecycleGeneration, verboseLevel: run.verboseLevel, isControlUiVisible: shouldSurfaceToControlUi, @@ -852,6 +853,7 @@ export function createFollowupRunner(params: { registerAgentRunContext(runId, { sessionKey: run.sessionKey, ...(owningSessionId ? { sessionId: owningSessionId } : {}), + agentId: run.agentId, lifecycleGeneration, verboseLevel: run.verboseLevel, isControlUiVisible: shouldSurfaceToControlUi, @@ -1082,8 +1084,8 @@ export function createFollowupRunner(params: { sessionKey: replySessionKey, startedAt: cliLifecycleStartedAt, getLifecycleGeneration: () => lifecycleGeneration, - resolveAbortLifecycleFields: () => - resolveAgentRunAbortLifecycleFields(runAbortSignal), + resolveTerminationFields: (error) => + resolveAgentRunErrorLifecycleFields(error, runAbortSignal), }); let droppedCliSessionReplacement = false; pendingLifecycleTerminal = { provider, model, backstop: lifecycleBackstop }; @@ -1255,8 +1257,8 @@ export function createFollowupRunner(params: { runId, sessionKey: replySessionKey, getLifecycleGeneration: () => lifecycleGeneration, - resolveAbortLifecycleFields: () => - resolveAgentRunAbortLifecycleFields(runAbortSignal), + resolveTerminationFields: (error) => + resolveAgentRunErrorLifecycleFields(error, runAbortSignal), }); pendingLifecycleTerminal = { provider, model, backstop: lifecycleBackstop }; const followupCurrentMessageId = resolveFollowupCurrentMessageId(); diff --git a/src/cli/program/command-registry-core.ts b/src/cli/program/command-registry-core.ts index 6696cee34113..04a608b34027 100644 --- a/src/cli/program/command-registry-core.ts +++ b/src/cli/program/command-registry-core.ts @@ -87,6 +87,11 @@ const coreEntrySpecs: readonly CommandGroupDescriptorSpec< loadModule: () => import("./register.migrate.js"), exportName: "registerMigrateCommand", }, + { + commandNames: ["audit"], + loadModule: () => import("./register.audit.js"), + exportName: "registerAuditCommand", + }, { commandNames: ["doctor", "dashboard", "reset", "uninstall"], loadModule: () => import("./register.maintenance.js"), diff --git a/src/cli/program/config-guard.test.ts b/src/cli/program/config-guard.test.ts index 640a19266cda..a983c958d3ad 100644 --- a/src/cli/program/config-guard.test.ts +++ b/src/cli/program/config-guard.test.ts @@ -353,7 +353,7 @@ describe("ensureConfigReady", () => { "", `Fix: ${formatCliCommand("openclaw doctor --fix")}`, `Inspect: ${formatCliCommand("openclaw config validate")}`, - "Status, health, logs, tasks list/audit, and doctor commands still run with invalid config.", + "Audit, status, health, logs, tasks list/audit, and doctor commands still run with invalid config.", ]); expect(runtime.exit).toHaveBeenCalledWith(1); }); @@ -389,6 +389,9 @@ describe("ensureConfigReady", () => { const statusRuntime = await runEnsureConfigReady(["status"]); expect(statusRuntime.exit).not.toHaveBeenCalled(); + const auditRuntime = await runEnsureConfigReady(["audit"]); + expect(auditRuntime.exit).not.toHaveBeenCalled(); + const bareGatewayRuntime = await runEnsureConfigReady(["gateway"]); expect(bareGatewayRuntime.exit).not.toHaveBeenCalled(); diff --git a/src/cli/program/config-guard.ts b/src/cli/program/config-guard.ts index ba582589dd63..eaac25a6b0d9 100644 --- a/src/cli/program/config-guard.ts +++ b/src/cli/program/config-guard.ts @@ -10,7 +10,14 @@ import { resolveRequiredHomeDir } from "../../infra/home-dir.js"; import type { RuntimeEnv } from "../../runtime.js"; import { shouldMigrateStateFromPath } from "../argv.js"; -const ALLOWED_INVALID_COMMANDS = new Set(["doctor", "logs", "health", "help", "status"]); +const ALLOWED_INVALID_COMMANDS = new Set([ + "audit", + "doctor", + "logs", + "health", + "help", + "status", +]); const ALLOWED_INVALID_GATEWAY_SUBCOMMANDS = new Set([ "run", "status", @@ -304,7 +311,7 @@ export async function ensureConfigReady(params: { ); params.runtime.error( muted( - "Status, health, logs, tasks list/audit, and doctor commands still run with invalid config.", + "Audit, status, health, logs, tasks list/audit, and doctor commands still run with invalid config.", ), ); if (!allowInvalid) { diff --git a/src/cli/program/core-command-descriptors.ts b/src/cli/program/core-command-descriptors.ts index 728471f939ac..38e44ec1d202 100644 --- a/src/cli/program/core-command-descriptors.ts +++ b/src/cli/program/core-command-descriptors.ts @@ -98,6 +98,11 @@ const coreCliCommandCatalog = defineCommandDescriptorCatalog([ description: "Fetch health from the running gateway", hasSubcommands: false, }, + { + name: "audit", + description: "Inspect metadata-only agent run and tool action records", + hasSubcommands: false, + }, { name: "sessions", description: "List stored conversation sessions", diff --git a/src/cli/program/register.audit.ts b/src/cli/program/register.audit.ts new file mode 100644 index 000000000000..1f8601904fa4 --- /dev/null +++ b/src/cli/program/register.audit.ts @@ -0,0 +1,51 @@ +// Audit command registration for privacy-preserving run/tool history. +import type { Command } from "commander"; +import { formatDocsLink } from "../../../packages/terminal-core/src/links.js"; +import { theme } from "../../../packages/terminal-core/src/theme.js"; +import { auditListCommand, type AuditListCommandOptions } from "../../commands/audit.js"; +import { defaultRuntime } from "../../runtime.js"; +import { runCommandWithRuntime } from "../cli-utils.js"; + +/** Register the bounded operator audit query command. */ +export function registerAuditCommand(program: Command): void { + program + .command("audit") + .description("Inspect metadata-only agent run and tool action records") + .option("--agent ", "Filter by agent id") + .option("--session ", "Filter by exact session key") + .option("--run ", "Filter by run id") + .option("--kind ", "Filter by kind (agent_run or tool_action)") + .option( + "--status ", + "Filter by status (started, succeeded, failed, cancelled, timed_out, blocked, unknown)", + ) + .option("--after ", "Include records at/after ISO time or Unix milliseconds") + .option("--before ", "Include records at/before ISO time or Unix milliseconds") + .option("--cursor ", "Continue from a previous result cursor") + .option("--limit ", "Maximum records (1-500)", "100") + .option("--json", "Output a bounded JSON page", false) + .addHelpText( + "after", + () => + `\n${theme.muted("Docs:")} ${formatDocsLink("/cli/audit", "docs.openclaw.ai/cli/audit")}\n`, + ) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await auditListCommand( + { + agentId: opts.agent as string | undefined, + sessionKey: opts.session as string | undefined, + runId: opts.run as string | undefined, + kind: opts.kind as AuditListCommandOptions["kind"], + status: opts.status as AuditListCommandOptions["status"], + after: opts.after as string | undefined, + before: opts.before as string | undefined, + cursor: opts.cursor as string | undefined, + limit: opts.limit as string | undefined, + json: Boolean(opts.json), + }, + defaultRuntime, + ); + }); + }); +} diff --git a/src/commands/agent.acp.test.ts b/src/commands/agent.acp.test.ts index 1aa022e471fc..793ac9f2a363 100644 --- a/src/commands/agent.acp.test.ts +++ b/src/commands/agent.acp.test.ts @@ -78,6 +78,11 @@ vi.mock("../agents/command/attempt-execution.runtime.js", () => { }; return { + createAcpToolLifecycleTracker: () => ({ + active: new Map(), + terminalToolCallIds: new Set(), + saturated: false, + }), createAcpVisibleTextAccumulator, emitAcpLifecycleStart: attemptExecutionMocks.emitAcpLifecycleStart, emitAcpLifecycleEnd: attemptExecutionMocks.emitAcpLifecycleEnd, diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 4ba397234df2..be0387f31d31 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -93,6 +93,11 @@ vi.mock("../agents/command/cli-compaction.js", () => { vi.mock("../agents/command/attempt-execution.runtime.js", () => { return { buildAcpResult: vi.fn(), + createAcpToolLifecycleTracker: () => ({ + active: new Map(), + terminalToolCallIds: new Set(), + saturated: false, + }), createAcpVisibleTextAccumulator: vi.fn(), emitAcpAssistantDelta: vi.fn(), emitAcpLifecycleEnd: vi.fn(), diff --git a/src/commands/audit.test.ts b/src/commands/audit.test.ts new file mode 100644 index 000000000000..538267b046f3 --- /dev/null +++ b/src/commands/audit.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { testApi } from "./audit.js"; + +describe("audit command parsing", () => { + it("parses ISO and millisecond timestamps", () => { + expect(testApi.parseAuditTimestamp("2026-07-01T00:00:00Z", "--after")).toBe( + Date.parse("2026-07-01T00:00:00Z"), + ); + expect(testApi.parseAuditTimestamp("1234", "--after")).toBe(1234); + expect(() => testApi.parseAuditTimestamp("not-a-date", "--after")).toThrow("--after"); + }); + + it("keeps exports bounded", () => { + expect(testApi.parseAuditLimit(undefined)).toBe(100); + expect(testApi.parseAuditLimit("500")).toBe(500); + expect(() => testApi.parseAuditLimit("501")).toThrow("1 and 500"); + }); + + it("renders untrusted metadata as one terminal-safe row", () => { + const [header, row] = testApi.formatAuditRows([ + { + eventId: "event-1", + sequence: 1, + sourceSequence: 1, + occurredAt: 0, + kind: "tool_action", + action: "tool.action.finished", + status: "failed", + actor: { type: "agent", id: "main" }, + agentId: "main\nforged", + runId: "run\tcolumn", + toolName: "\u001b]8;;https://example.invalid\u0007unsafe", + redaction: "metadata_only", + }, + ]); + + expect(header).toContain("TIME"); + expect(row).not.toContain("\n"); + expect(row).not.toContain("\u001b"); + expect(row).toContain("main\\nforged"); + expect(row).toContain("run\\tcolumn"); + }); +}); diff --git a/src/commands/audit.ts b/src/commands/audit.ts new file mode 100644 index 000000000000..bbb849d05bc1 --- /dev/null +++ b/src/commands/audit.ts @@ -0,0 +1,120 @@ +/** Operator CLI for bounded metadata-only run/tool audit pages. */ +import { timestampMsToIsoString } from "@openclaw/normalization-core/number-coercion"; +import type { + AuditEvent, + AuditListParams, + AuditListResult, +} from "../../packages/gateway-protocol/src/index.js"; +import { sanitizeTerminalText } from "../../packages/terminal-core/src/safe-text.js"; +import { callGateway } from "../gateway/call.js"; +import { parseStrictPositiveInteger } from "../infra/parse-finite-number.js"; +import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; + +const DEFAULT_AUDIT_LIMIT = 100; +const MAX_AUDIT_LIMIT = 500; + +export type AuditListCommandOptions = { + agentId?: string; + sessionKey?: string; + runId?: string; + kind?: AuditListParams["kind"]; + status?: AuditListParams["status"]; + after?: string; + before?: string; + cursor?: string; + limit?: string; + json?: boolean; +}; + +function parseAuditTimestamp(value: string | undefined, flag: string): number | undefined { + const trimmed = value?.trim(); + if (!trimmed) { + return undefined; + } + if (/^\d+$/.test(trimmed)) { + const parsed = Number(trimmed); + if (Number.isSafeInteger(parsed)) { + return parsed; + } + } + const parsed = Date.parse(trimmed); + if (!Number.isNaN(parsed)) { + return parsed; + } + throw new Error(`${flag} must be an ISO timestamp or Unix milliseconds.`); +} + +function parseAuditLimit(value: string | undefined): number { + if (!value) { + return DEFAULT_AUDIT_LIMIT; + } + const parsed = parseStrictPositiveInteger(value); + if (parsed === undefined || parsed > MAX_AUDIT_LIMIT) { + throw new Error(`--limit must be between 1 and ${MAX_AUDIT_LIMIT}.`); + } + return parsed; +} + +function short(value: string | undefined, maxChars: number): string { + if (!value) { + return "-"; + } + const sanitized = sanitizeTerminalText(value); + if (!sanitized) { + return "-"; + } + return sanitized.length <= maxChars ? sanitized : `${sanitized.slice(0, maxChars - 1)}…`; +} + +function formatAuditRows(events: AuditEvent[]): string[] { + const rows = ["TIME\tKIND\tSTATUS\tAGENT\tRUN\tACTION"]; + for (const event of events) { + rows.push( + [ + timestampMsToIsoString(event.occurredAt) ?? String(event.occurredAt), + event.kind, + event.status, + short(event.agentId, 18), + short(event.runId, 18), + event.toolName ? `${event.action}:${short(event.toolName, 28)}` : event.action, + ].join("\t"), + ); + } + return rows; +} + +/** Query one stable page. JSON output is a bounded export with its next cursor. */ +export async function auditListCommand( + options: AuditListCommandOptions, + runtime: RuntimeEnv, +): Promise { + const after = parseAuditTimestamp(options.after, "--after"); + const before = parseAuditTimestamp(options.before, "--before"); + if (after !== undefined && before !== undefined && after > before) { + throw new Error("--after must not be later than --before."); + } + const params: AuditListParams = { + limit: parseAuditLimit(options.limit), + ...(options.agentId ? { agentId: options.agentId } : {}), + ...(options.sessionKey ? { sessionKey: options.sessionKey } : {}), + ...(options.runId ? { runId: options.runId } : {}), + ...(options.kind ? { kind: options.kind } : {}), + ...(options.status ? { status: options.status } : {}), + ...(after !== undefined ? { after } : {}), + ...(before !== undefined ? { before } : {}), + ...(options.cursor ? { cursor: options.cursor } : {}), + }; + const result = await callGateway({ method: "audit.list", params }); + if (options.json) { + writeRuntimeJson(runtime, result); + return; + } + for (const row of formatAuditRows(result.events)) { + runtime.log(row); + } + if (result.nextCursor) { + runtime.log(`More records: --cursor ${result.nextCursor}`); + } +} + +export const testApi = { formatAuditRows, parseAuditLimit, parseAuditTimestamp }; diff --git a/src/config/types.base.ts b/src/config/types.base.ts index 5280f254b51e..f010b89f1a0b 100644 --- a/src/config/types.base.ts +++ b/src/config/types.base.ts @@ -340,6 +340,15 @@ export type DiagnosticsCacheTraceConfig = { includeSystem?: boolean; }; +export type AuditConfig = { + /** + * Record metadata-only audit events (agent runs and tool actions) into the + * shared state database. Content is never stored. Default: true. Disabling + * stops new writes; existing records stay readable until they expire. + */ + enabled?: boolean; +}; + export type DiagnosticsConfig = { enabled?: boolean; /** Optional ad-hoc diagnostics flags (e.g. "telegram.http"). */ diff --git a/src/config/types.openclaw.ts b/src/config/types.openclaw.ts index 019797623b63..e3fab5b0a7df 100644 --- a/src/config/types.openclaw.ts +++ b/src/config/types.openclaw.ts @@ -6,7 +6,13 @@ import type { AcpConfig } from "./types.acp.js"; import type { AgentBinding, AgentsConfig } from "./types.agents.js"; import type { ApprovalsConfig } from "./types.approvals.js"; import type { AuthConfig } from "./types.auth.js"; -import type { DiagnosticsConfig, LoggingConfig, SessionConfig, WebConfig } from "./types.base.js"; +import type { + AuditConfig, + DiagnosticsConfig, + LoggingConfig, + SessionConfig, + WebConfig, +} from "./types.base.js"; import type { BrowserConfig } from "./types.browser.js"; import type { ChannelsConfig } from "./types.channels.js"; import type { CliConfig } from "./types.cli.js"; @@ -134,6 +140,8 @@ export type OpenClawConfig = { diagnostics?: DiagnosticsConfig; /** Log sink, level, rotation, and redaction settings. */ logging?: LoggingConfig; + /** Metadata-only agent activity audit ledger settings. */ + audit?: AuditConfig; /** Security audit suppressions and security policy settings. */ security?: SecurityConfig; /** CLI defaults and command-specific settings. */ diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index bbc4cd88b2ae..71c552da9625 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -642,6 +642,12 @@ export const OpenClawSchema = z }) .strict() .optional(), + audit: z + .object({ + enabled: z.boolean().optional(), + }) + .strict() + .optional(), logging: z .object({ level: LoggingLevelSchema.optional(), diff --git a/src/gateway/mcp-http.handlers.ts b/src/gateway/mcp-http.handlers.ts index bf7e989fbed1..1b5334f84dd4 100644 --- a/src/gateway/mcp-http.handlers.ts +++ b/src/gateway/mcp-http.handlers.ts @@ -3,7 +3,12 @@ import crypto from "node:crypto"; import { ContentBlockSchema, type ContentBlock } from "@modelcontextprotocol/sdk/types.js"; import { runBeforeToolCallHook, type HookContext } from "../agents/agent-tools.before-tool-call.js"; -import { formatErrorMessage } from "../infra/errors.js"; +import { + formatToolExecutionErrorMessage, + resolveToolExecutionErrorKind, + resolveToolResultFailureKind, +} from "../agents/tool-result-error.js"; +import type { McpLoopbackToolCallOutcome } from "./mcp-http.loopback-runtime.js"; import { MCP_LOOPBACK_SERVER_NAME, MCP_LOOPBACK_SERVER_VERSION, @@ -60,12 +65,12 @@ export async function handleMcpJsonRpc(params: { toolSchema: McpToolSchemaEntry[]; hookContext?: HookContext; signal?: AbortSignal; - onToolCallResult?: (call: { - toolName: string; - args: Record; - result?: unknown; - isError: boolean; - }) => void; + onToolCallResult?: ( + call: { + toolName: string; + args: Record; + } & McpLoopbackToolCallOutcome, + ) => void; onToolCallPrepared?: (call: { toolName: string; args: Record }) => void; }): Promise { const { id, method, params: methodParams } = params.message; @@ -118,13 +123,12 @@ export async function handleMcpJsonRpc(params: { } const toolCallId = `mcp-${crypto.randomUUID()}`; let executedToolArgs = toolArgs; - const reportToolCallResult = (result: unknown, isError: boolean) => { + const reportToolCallResult = (outcome: McpLoopbackToolCallOutcome) => { try { params.onToolCallResult?.({ toolName, args: executedToolArgs, - result, - isError, + ...outcome, }); } catch { // Observability callbacks must never alter the tool result returned to the MCP client. @@ -141,6 +145,15 @@ export async function handleMcpJsonRpc(params: { signal: params.signal, }); if (hookResult.blocked) { + const disposition = hookResult.kind === "failure" ? hookResult.disposition : "blocked"; + reportToolCallResult( + disposition === "blocked" + ? { + outcome: disposition, + deniedReason: hookResult.deniedReason ?? "plugin-before-tool-call", + } + : { outcome: disposition }, + ); return jsonRpcResult(id, { content: [{ type: "text", text: hookResult.reason }], isError: true, @@ -153,14 +166,24 @@ export async function handleMcpJsonRpc(params: { // Observability callbacks must never alter the tool result returned to the MCP client. } const result = await tool.execute(toolCallId, hookResult.params, params.signal); - reportToolCallResult(result, false); + const failureKind = resolveToolResultFailureKind(result); + reportToolCallResult( + failureKind === "blocked" + ? { outcome: "blocked", deniedReason: "tool_result_blocked" } + : { outcome: failureKind ?? "completed", result }, + ); return jsonRpcResult(id, { content: normalizeToolCallContent(result), - isError: false, + isError: failureKind !== undefined, }); } catch (error) { - reportToolCallResult(error, true); - const message = formatErrorMessage(error); + // A disconnected request does not identify the enclosing run outcome, + // but its payload may prove partial delivery and prevent a duplicate send. + reportToolCallResult({ + outcome: params.signal?.aborted ? "unknown" : resolveToolExecutionErrorKind(error), + result: error, + }); + const message = formatToolExecutionErrorMessage(error, "tool execution failed"); return jsonRpcResult(id, { content: [{ type: "text", text: message || "tool execution failed" }], isError: true, diff --git a/src/gateway/mcp-http.loopback-runtime.ts b/src/gateway/mcp-http.loopback-runtime.ts index 11205b7b8b14..1b7b5bb526c1 100644 --- a/src/gateway/mcp-http.loopback-runtime.ts +++ b/src/gateway/mcp-http.loopback-runtime.ts @@ -5,12 +5,19 @@ type McpLoopbackRuntime = { nonOwnerToken: string; }; +export type McpLoopbackToolCallTerminalOutcome = + | { outcome: "blocked"; deniedReason: string } + | { outcome: "cancelled" | "failed" | "timed_out" | "unknown"; result?: unknown }; + +export type McpLoopbackToolCallOutcome = + | { outcome: "completed"; result?: unknown } + | McpLoopbackToolCallTerminalOutcome; + export type McpLoopbackToolCallResult = { toolName: string; args: Record; - result?: unknown; - isError: boolean; -}; + correlationId?: string; +} & McpLoopbackToolCallOutcome; export type McpLoopbackToolCallStart = Pick; @@ -20,7 +27,7 @@ type McpLoopbackToolCallCapture = { onRequestStart?: () => void; onRequestClassified?: () => void; onRequestFinish?: () => void; - onToolCallStart?: (call: McpLoopbackToolCallStart) => void; + onToolCallStart?: (call: McpLoopbackToolCallStart) => string | void; onToolCallUpdate?: (calls: { previous: McpLoopbackToolCallStart; current: McpLoopbackToolCallStart; @@ -41,6 +48,7 @@ export type McpLoopbackRequestCaptureHandle = { export type McpLoopbackToolCallCaptureHandle = { capture: McpLoopbackToolCallCapture; call: McpLoopbackToolCallStart; + correlationId?: string; prepared: boolean; finished: boolean; }; @@ -76,7 +84,7 @@ export function beginMcpLoopbackToolCallCapture(params: { onRequestStart?: () => void; onRequestClassified?: () => void; onRequestFinish?: () => void; - onToolCallStart?: (call: McpLoopbackToolCallStart) => void; + onToolCallStart?: (call: McpLoopbackToolCallStart) => string | void; onToolCallUpdate?: (calls: { previous: McpLoopbackToolCallStart; current: McpLoopbackToolCallStart; @@ -196,12 +204,14 @@ export function markMcpLoopbackToolCallStarted(params: { const call = { toolName, args: params.args }; capture.inFlight += 1; notifyMcpLoopbackToolCallCaptureActivity(capture); + let correlationId: string | undefined; try { - capture.onToolCallStart?.(call); + const observedCorrelationId = capture.onToolCallStart?.(call); + correlationId = typeof observedCorrelationId === "string" ? observedCorrelationId : undefined; } catch { // Delivery observation is diagnostic state; it must not alter tool execution. } - return { capture, call, prepared: false, finished: false }; + return { capture, call, correlationId, prepared: false, finished: false }; } /** Update an admitted call with the final arguments produced by gateway hooks. */ @@ -223,23 +233,29 @@ export function updateMcpLoopbackToolCallCapture( } /** Report a completed call without letting observer failures alter tool execution. */ -export function recordMcpLoopbackToolCallResult(params: { - captureHandle: McpLoopbackToolCallCaptureHandle; - toolName: string; - args: Record; - result?: unknown; - isError: boolean; -}): void { +export function recordMcpLoopbackToolCallResult( + params: { + captureHandle: McpLoopbackToolCallCaptureHandle; + toolName: string; + args: Record; + } & McpLoopbackToolCallOutcome, +): void { const toolName = params.toolName.trim(); if (!toolName) { return; } try { + const outcome: McpLoopbackToolCallOutcome = + params.outcome === "blocked" + ? { outcome: "blocked", deniedReason: params.deniedReason } + : { outcome: params.outcome, result: params.result }; params.captureHandle.capture.onToolCallResult({ toolName, args: params.args, - result: params.result, - isError: params.isError, + ...outcome, + ...(params.captureHandle.correlationId + ? { correlationId: params.captureHandle.correlationId } + : {}), }); } catch { // Delivery observation is diagnostic state; it must not turn a successful tool call into error. diff --git a/src/gateway/mcp-http.test.ts b/src/gateway/mcp-http.test.ts index d92db8c4a197..daef7b977f12 100644 --- a/src/gateway/mcp-http.test.ts +++ b/src/gateway/mcp-http.test.ts @@ -2,14 +2,19 @@ // JSON-RPC surface, including hook filtering and context propagation. import { request } from "node:http"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AnyAgentTool } from "../agents/tools/common.js"; import { getFreePortBlockWithPermissionFallback } from "../test-utils/ports.js"; import { buildMcpToolSchema } from "./mcp-http.schema.js"; type MockGatewayTool = { name: string; + label: string; description: string; parameters: Record; - execute: (...args: unknown[]) => Promise<{ content: unknown[] }>; + execute: (...args: unknown[]) => Promise<{ + content: unknown[]; + details?: Record; + }>; }; type MockGatewayScopedTools = { @@ -18,7 +23,19 @@ type MockGatewayScopedTools = { }; type MockBeforeToolCallHookResult = - | { blocked: true; reason: string } + | { + blocked: true; + kind: "veto"; + deniedReason?: "plugin-before-tool-call" | "plugin-approval" | "tool-loop"; + reason: string; + } + | { + blocked: true; + kind: "failure"; + disposition: "blocked" | "cancelled" | "failed" | "timed_out"; + deniedReason?: "plugin-before-tool-call" | "plugin-approval" | "tool-loop"; + reason: string; + } | { blocked: false; params: unknown }; type ScopedToolsCall = { @@ -74,6 +91,7 @@ const resolveGatewayScopedToolsMock = vi.hoisted(() => tools: [ { name: "message", + label: "Message", description: "send a message", parameters: { type: "object", properties: {} }, execute: async () => ({ @@ -103,6 +121,7 @@ vi.mock("./tool-resolution.js", () => ({ })); import { resetAttachGrantsForTest, mintAttachGrant } from "./mcp-grant-store.js"; +import { handleMcpJsonRpc } from "./mcp-http.handlers.js"; import { createMcpAttachGrantServerConfig, createMcpLoopbackServerConfig, @@ -511,6 +530,7 @@ function getBeforeToolCallHookInput(index: number): BeforeToolCallHookInput { function makeMockTool(overrides: Partial = {}): MockGatewayTool { return { name: "mockplugin_tool", + label: "Mock tool", description: "mock tool", parameters: { type: "object", properties: {} }, execute: async () => ({ @@ -942,6 +962,7 @@ describe("mcp loopback server", () => { tools: [ { name: "schema_probe", + label: "Schema probe", description: "exercise no-argument MCP schemas", parameters: { type: "object" }, execute: async () => ({ @@ -1110,15 +1131,24 @@ describe("mcp loopback server", () => { it("captures only successful calls with an explicit CLI capture key", async () => { const captureKey = "google-gemini-cli"; const captured: Array<{ toolName: string; args: Record }> = []; + const blockedResults: unknown[] = []; const startedTargets: unknown[] = []; const finishedTargets: unknown[] = []; beginMcpLoopbackToolCallCapture({ captureKey, - onToolCallStart: ({ args }) => startedTargets.push(args.target), + onToolCallStart: ({ args }) => { + startedTargets.push(args.target); + return typeof args.target === "string" ? args.target : undefined; + }, onToolCallFinish: ({ args }) => finishedTargets.push(args.target), - onToolCallResult: ({ toolName, args }) => { - if (toolName === "message" && args.action === "send") { - captured.push({ toolName, args }); + onToolCallResult: (result) => { + if (result.outcome === "blocked") { + blockedResults.push({ + correlationId: result.correlationId, + deniedReason: result.deniedReason, + }); + } else if (result.toolName === "message" && result.args.action === "send") { + captured.push({ toolName: result.toolName, args: result.args }); } }, }); @@ -1137,6 +1167,7 @@ describe("mcp loopback server", () => { runBeforeToolCallHookMock.mockResolvedValueOnce({ blocked: true, + kind: "veto", reason: "blocked for test", }); expect( @@ -1168,6 +1199,120 @@ describe("mcp loopback server", () => { ]); expect(startedTargets).toEqual(["chat123", "blocked"]); expect(finishedTargets).toEqual(["chat123", "blocked"]); + expect(blockedResults).toEqual([ + { correlationId: "blocked", deniedReason: "plugin-before-tool-call" }, + ]); + }); + + it("preserves hook failure dispositions in CLI capture", async () => { + const captureKey = "hook-failure-dispositions"; + const captured: unknown[] = []; + beginMcpLoopbackToolCallCapture({ + captureKey, + onToolCallStart: ({ args }) => String(args.target), + onToolCallResult: (result) => { + captured.push({ + outcome: result.outcome, + correlationId: result.correlationId, + ...(result.outcome === "blocked" ? { deniedReason: result.deniedReason } : {}), + }); + }, + }); + const { runtime } = await startLoopbackServerForTest(); + const cases = [ + { disposition: "failed" }, + { disposition: "cancelled" }, + { disposition: "timed_out" }, + { disposition: "blocked", deniedReason: "plugin-approval" }, + ] as const; + + for (const testCase of cases) { + runBeforeToolCallHookMock.mockResolvedValueOnce({ + blocked: true, + kind: "failure", + disposition: testCase.disposition, + ...(testCase.disposition === "blocked" ? { deniedReason: testCase.deniedReason } : {}), + reason: "hook prevented execution", + }); + const response = await sendLoopbackToolCall({ + token: runtime.ownerToken, + name: "message", + args: { action: "send", target: testCase.disposition, message: "not sent" }, + headers: { "x-openclaw-cli-capture-key": captureKey }, + }); + expect(response.status).toBe(200); + expect((await readMcpPayload(response)).result?.isError).toBe(true); + } + + expect(captured).toEqual([ + { outcome: "failed", correlationId: "failed" }, + { outcome: "cancelled", correlationId: "cancelled" }, + { outcome: "timed_out", correlationId: "timed_out" }, + { + outcome: "blocked", + correlationId: "blocked", + deniedReason: "plugin-approval", + }, + ]); + }); + + it("classifies resolved structured results for CLI capture", async () => { + const captureKey = "structured-results"; + const captured: unknown[] = []; + beginMcpLoopbackToolCallCapture({ + captureKey, + onToolCallStart: ({ args }) => String(args.status), + onToolCallResult: (result) => { + captured.push({ + outcome: result.outcome, + correlationId: result.correlationId, + ...(result.outcome === "blocked" ? { deniedReason: result.deniedReason } : {}), + }); + }, + }); + mockScopedTools([ + makeMessageTool({ + execute: async (_toolCallId, args) => ({ + content: [{ type: "text", text: "result" }], + details: args as Record, + }), + }), + ]); + const { runtime } = await startLoopbackServerForTest(); + const cases = [ + { status: "completed", expected: "completed" }, + { status: "failed", expected: "failed" }, + { status: "blocked", expected: "blocked" }, + { status: "cancelled", expected: "cancelled" }, + { status: "completed-timeout", expected: "timed_out", timedOut: true }, + ] as const; + + for (const testCase of cases) { + const response = await sendLoopbackToolCall({ + token: runtime.ownerToken, + name: "message", + args: { + status: testCase.status === "completed-timeout" ? "completed" : testCase.status, + ...("timedOut" in testCase && testCase.timedOut ? { timedOut: true } : {}), + }, + headers: { "x-openclaw-cli-capture-key": captureKey }, + }); + expect(response.status).toBe(200); + const payload = await readMcpPayload(response); + expect(payload.result?.isError).toBe(testCase.expected !== "completed"); + } + + expect(captured).toEqual([ + { outcome: "completed", correlationId: "completed" }, + { outcome: "failed", correlationId: "failed" }, + { + outcome: "blocked", + correlationId: "blocked", + deniedReason: "tool_result_blocked", + }, + { outcome: "cancelled", correlationId: "cancelled" }, + { outcome: "timed_out", correlationId: "completed" }, + ]); }); it("updates capture accounting with hook-rewritten tool arguments", async () => { @@ -1245,7 +1390,7 @@ describe("mcp loopback server", () => { toolName: "message", args: { action: "send", target: "chat123" }, result: { content: "x".repeat(20 * 1024) }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(captureHandle); @@ -1283,7 +1428,7 @@ describe("mcp loopback server", () => { toolName: "message", args: { action: "send", target: "first-turn" }, result: { status: "sent" }, - isError: false, + outcome: "completed", }); markMcpLoopbackToolCallFinished(firstHandle); @@ -1354,7 +1499,7 @@ describe("mcp loopback server", () => { expect.objectContaining({ toolName: "message", args: { action: "send", target: "late-body" }, - isError: false, + outcome: "completed", }), ); }); @@ -1435,12 +1580,48 @@ describe("mcp loopback server", () => { expect(captured).toHaveBeenCalledWith( expect.objectContaining({ toolName: "message", - isError: true, + outcome: "failed", result: expect.objectContaining({ sentBeforeError: true }), }), ); }); + it("preserves thrown timeout outcomes in CLI capture", async () => { + const captureKey = "thrown-timeout"; + const captured = vi.fn(); + const timeoutError = Object.assign(new Error("tool deadline elapsed"), { + name: "TimeoutError", + }); + beginMcpLoopbackToolCallCapture({ + captureKey, + onToolCallResult: captured, + }); + mockScopedTools([ + makeMessageTool({ + execute: async () => { + throw timeoutError; + }, + }), + ]); + const { runtime } = await startLoopbackServerForTest(); + + const response = await sendLoopbackToolCall({ + token: runtime.ownerToken, + name: "message", + args: { action: "send", target: "chat123", message: "late" }, + headers: { "x-openclaw-cli-capture-key": captureKey }, + }); + + expect((await readMcpPayload(response)).result?.isError).toBe(true); + expect(captured).toHaveBeenCalledWith( + expect.objectContaining({ + toolName: "message", + outcome: "timed_out", + result: timeoutError, + }), + ); + }); + it("ignores calls after a capture is cleared", () => { const captureKey = "cleared-capture"; const captured = vi.fn(); @@ -1519,6 +1700,7 @@ describe("mcp loopback server", () => { })); runBeforeToolCallHookMock.mockResolvedValueOnce({ blocked: true, + kind: "veto", reason: "blocked by hook", }); const payload = await callMessageToolWithExecute(execute); @@ -1548,6 +1730,41 @@ describe("mcp loopback server", () => { expect(signal).toBeInstanceOf(AbortSignal); }); + it("preserves request-disconnect evidence without classifying a tool failure", async () => { + const controller = new AbortController(); + controller.abort(); + const onToolCallResult = vi.fn(); + const partialDelivery = Object.assign(new Error("request disconnected"), { + name: "AbortError", + sentBeforeError: true, + }); + const tool = makeMessageTool({ + execute: async () => { + throw partialDelivery; + }, + }); + + await handleMcpJsonRpc({ + message: { + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { name: "message", arguments: { body: "hello" } }, + }, + tools: [tool as unknown as AnyAgentTool], + toolSchema: buildMockMcpToolSchema([tool]), + signal: controller.signal, + onToolCallResult, + }); + + expect(onToolCallResult).toHaveBeenCalledWith({ + toolName: "message", + args: { body: "hello" }, + outcome: "unknown", + result: partialDelivery, + }); + }); + it("tracks the active runtime only while the server is running", async () => { server = await startMcpLoopbackServer(0); const active = getActiveMcpLoopbackRuntime(); diff --git a/src/gateway/mcp-http.ts b/src/gateway/mcp-http.ts index c41282628cb1..29629a172ae5 100644 --- a/src/gateway/mcp-http.ts +++ b/src/gateway/mcp-http.ts @@ -285,13 +285,10 @@ export async function startMcpLoopbackServer(port = 0): Promise<{ } : undefined, onToolCallResult: cliCaptureHandle - ? ({ toolName: resultToolName, args, result, isError }) => { + ? (result) => { recordMcpLoopbackToolCallResult({ captureHandle: cliCaptureHandle, - toolName: resultToolName, - args, - result, - isError, + ...result, }); } : undefined, diff --git a/src/gateway/method-scopes.test.ts b/src/gateway/method-scopes.test.ts index 0e1cadbdda8e..6484e98684a7 100644 --- a/src/gateway/method-scopes.test.ts +++ b/src/gateway/method-scopes.test.ts @@ -42,6 +42,7 @@ describe("method scope resolution", () => { it.each([ ["sessions.resolve", ["operator.read"]], ["tasks.list", ["operator.read"]], + ["audit.list", ["operator.read"]], ["tasks.get", ["operator.read"]], ["config.schema.lookup", ["operator.read"]], ["sessions.create", ["operator.write"]], diff --git a/src/gateway/methods/core-descriptors.ts b/src/gateway/methods/core-descriptors.ts index 7e7f08b75e66..407d2b759a97 100644 --- a/src/gateway/methods/core-descriptors.ts +++ b/src/gateway/methods/core-descriptors.ts @@ -97,6 +97,7 @@ export const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [ { name: "tools.catalog", scope: "operator.read" }, { name: "tools.effective", scope: "operator.read", startup: true }, { name: "tools.invoke", scope: "operator.write" }, + { name: "audit.list", scope: "operator.read" }, { name: "tasks.list", scope: "operator.read" }, { name: "tasks.get", scope: "operator.read" }, { name: "tasks.cancel", scope: "operator.write" }, diff --git a/src/gateway/server-close.test.ts b/src/gateway/server-close.test.ts index 02b2cae82c5f..5fc866c08a09 100644 --- a/src/gateway/server-close.test.ts +++ b/src/gateway/server-close.test.ts @@ -7,7 +7,7 @@ import type { InternalHookEvent } from "../hooks/internal-hooks.js"; type TriggerInternalHookMock = (event: InternalHookEvent) => Promise; -const mocks = { +const mocks = vi.hoisted(() => ({ logInfo: vi.fn(), logWarn: vi.fn(), listChannelPlugins: vi.fn((): Array<{ id: "telegram" | "discord" }> => []), @@ -15,7 +15,7 @@ const mocks = { disposeAllSessionMcpRuntimes: vi.fn(async () => undefined), triggerInternalHook: vi.fn(async (_eventValue) => undefined), disposeAllBundleLspRuntimes: vi.fn(async () => undefined), -}; +})); const WEBSOCKET_CLOSE_GRACE_MS = 1_000; const WEBSOCKET_CLOSE_FORCE_CONTINUE_MS = 250; const HTTP_CLOSE_GRACE_MS = 1_000; diff --git a/src/gateway/server-close.ts b/src/gateway/server-close.ts index 67b9e35cc581..60c7efeb470b 100644 --- a/src/gateway/server-close.ts +++ b/src/gateway/server-close.ts @@ -688,7 +688,7 @@ export function createGatewayCloseHandler( dedupeCleanup: ReturnType; mediaCleanup: ReturnType | null; worktreeCleanup: ReturnType | null; - agentUnsub: (() => void) | null; + agentUnsub: (() => Promise | void) | null; heartbeatUnsub: (() => void) | null; transcriptUnsub: (() => void) | null; lifecycleUnsub: (() => void) | null; diff --git a/src/gateway/server-methods.ts b/src/gateway/server-methods.ts index 5ae4526df344..77652530acb3 100644 --- a/src/gateway/server-methods.ts +++ b/src/gateway/server-methods.ts @@ -74,6 +74,10 @@ const loadArtifactsHandlers = lazyHandlerModule( () => import("./server-methods/artifacts.js"), (module) => module.artifactsHandlers, ); +const loadAuditHandlers = lazyHandlerModule( + () => import("./server-methods/audit.js"), + (module) => module.auditHandlers, +); const loadAttachHandlers = lazyHandlerModule( () => import("./server-methods/attach.js"), (module) => module.attachHandlers, @@ -482,6 +486,10 @@ export const coreGatewayHandlers: GatewayRequestHandlers = { ], loadHandlers: loadTalkHandlers, }), + ...createLazyCoreHandlers({ + methods: ["audit.list"], + loadHandlers: loadAuditHandlers, + }), ...createLazyCoreHandlers({ methods: ["tasks.list", "tasks.get", "tasks.cancel"], loadHandlers: loadTasksHandlers, diff --git a/src/gateway/server-methods/agent-job.ts b/src/gateway/server-methods/agent-job.ts index 69ee39fe4a4e..09e1412bde71 100644 --- a/src/gateway/server-methods/agent-job.ts +++ b/src/gateway/server-methods/agent-job.ts @@ -1,6 +1,7 @@ // Agent job tracking caches terminal run snapshots so `agent.wait` can observe // recent run outcomes even after the live event stream has moved on. import { + AGENT_RUN_TERMINAL_RETRY_GRACE_MS, buildAgentRunTerminalOutcome, mergeAgentRunTerminalOutcome, type AgentRunTerminalOutcome, @@ -11,20 +12,6 @@ import type { AgentWaitTerminalSnapshot } from "./agent-wait-dedupe.js"; const AGENT_RUN_CACHE_TTL_MS = 10 * 60_000; const AGENT_RUN_CACHE_MAX_ENTRIES = 5_000; -/** - * Embedded runs can emit transient lifecycle `error` events while auth/model - * failover is still in progress. Give errors a short grace window so a - * subsequent `start` event can cancel premature terminal snapshots. - */ -const AGENT_RUN_ERROR_RETRY_GRACE_MS = 15_000; -/** - * Some embedded runtimes emit an intermediate lifecycle `end` with - * `aborted=true` immediately before retrying the same run. Hold timeout - * snapshots briefly so `agent.wait` does not resolve to a stale timeout when a - * final success is about to arrive. - */ -const AGENT_RUN_TIMEOUT_RETRY_GRACE_MS = 15_000; - const agentRunCache = new Map(); const agentRunStarts = new Map(); const pendingAgentRunErrors = new Map(); @@ -140,7 +127,7 @@ function schedulePendingAgentRunError(snapshot: AgentRunSnapshot) { } clearPendingAgentRunTimeout(snapshot.runId); clearPendingAgentRunError(snapshot.runId); - const dueAt = Date.now() + AGENT_RUN_ERROR_RETRY_GRACE_MS; + const dueAt = Date.now() + AGENT_RUN_TERMINAL_RETRY_GRACE_MS; const timer = setTimeout(() => { const pending = pendingAgentRunErrors.get(snapshot.runId); if (!pending) { @@ -148,7 +135,7 @@ function schedulePendingAgentRunError(snapshot: AgentRunSnapshot) { } pendingAgentRunErrors.delete(snapshot.runId); recordAgentRunSnapshot(pending.snapshot); - }, AGENT_RUN_ERROR_RETRY_GRACE_MS); + }, AGENT_RUN_TERMINAL_RETRY_GRACE_MS); timer.unref?.(); pendingAgentRunErrors.set(snapshot.runId, { snapshot, dueAt, timer }); } @@ -162,7 +149,7 @@ function schedulePendingAgentRunTimeout(snapshot: AgentRunSnapshot) { } clearPendingAgentRunError(snapshot.runId); clearPendingAgentRunTimeout(snapshot.runId); - const dueAt = Date.now() + AGENT_RUN_TIMEOUT_RETRY_GRACE_MS; + const dueAt = Date.now() + AGENT_RUN_TERMINAL_RETRY_GRACE_MS; const timer = setTimeout(() => { const pending = pendingAgentRunTimeouts.get(snapshot.runId); if (!pending) { @@ -170,7 +157,7 @@ function schedulePendingAgentRunTimeout(snapshot: AgentRunSnapshot) { } pendingAgentRunTimeouts.delete(snapshot.runId); recordAgentRunSnapshot(pending.snapshot); - }, AGENT_RUN_TIMEOUT_RETRY_GRACE_MS); + }, AGENT_RUN_TERMINAL_RETRY_GRACE_MS); timer.unref?.(); pendingAgentRunTimeouts.set(snapshot.runId, { snapshot, dueAt, timer }); } @@ -416,14 +403,14 @@ export async function waitForAgentJob(params: { const scheduleErrorFinish = ( snapshot: AgentRunSnapshot, - delayMs = AGENT_RUN_ERROR_RETRY_GRACE_MS, + delayMs = AGENT_RUN_TERMINAL_RETRY_GRACE_MS, ) => { scheduleTerminalFinish("error", snapshot, delayMs); }; const scheduleTimeoutFinish = ( snapshot: AgentRunSnapshot, - delayMs = AGENT_RUN_TIMEOUT_RETRY_GRACE_MS, + delayMs = AGENT_RUN_TERMINAL_RETRY_GRACE_MS, ) => { scheduleTerminalFinish("timeout", snapshot, delayMs); }; diff --git a/src/gateway/server-methods/agent-wait-dedupe.test.ts b/src/gateway/server-methods/agent-wait-dedupe.test.ts index 699628324ea4..039793959fab 100644 --- a/src/gateway/server-methods/agent-wait-dedupe.test.ts +++ b/src/gateway/server-methods/agent-wait-dedupe.test.ts @@ -138,11 +138,13 @@ describe("agent wait dedupe helper", () => { return waitForTerminalGatewayDedupe({ dedupe, runId, timeoutMs: 1_000, ...options }); } - const RPC_QUEUE_TIMEOUT_SNAPSHOT = { - status: "timeout", + const RPC_QUEUE_CANCEL_SNAPSHOT = { + status: "error", + startedAt: undefined, endedAt: 100, error: undefined, stopReason: "rpc", + livenessState: undefined, timeoutPhase: "queue", providerStarted: false, } as const; @@ -475,7 +477,7 @@ describe("agent wait dedupe helper", () => { payload: okPayload(runId, { endedAt: 200 }), }); - expectTerminalSnapshot(dedupe, runId, RPC_QUEUE_TIMEOUT_SNAPSHOT); + expectTerminalSnapshot(dedupe, runId, RPC_QUEUE_CANCEL_SNAPSHOT); }); it("preserves an RPC cancel snapshot when a later accepted write reuses the key", () => { @@ -494,7 +496,7 @@ describe("agent wait dedupe helper", () => { payload: { runId, status: "accepted" }, }); - expectTerminalSnapshot(dedupe, runId, RPC_QUEUE_TIMEOUT_SNAPSHOT); + expectTerminalSnapshot(dedupe, runId, RPC_QUEUE_CANCEL_SNAPSHOT); }); it("lets an earlier terminal completion correct a provisional timeout snapshot", () => { @@ -560,7 +562,7 @@ describe("agent wait dedupe helper", () => { payload: { runId, status: "error", summary: "late failure", endedAt: 200 }, }); - expectTerminalSnapshot(dedupe, runId, RPC_QUEUE_TIMEOUT_SNAPSHOT); + expectTerminalSnapshot(dedupe, runId, RPC_QUEUE_CANCEL_SNAPSHOT); }); it("resolves multiple waiters for the same run id", async () => { diff --git a/src/gateway/server-methods/audit.test.ts b/src/gateway/server-methods/audit.test.ts new file mode 100644 index 000000000000..5e75816da183 --- /dev/null +++ b/src/gateway/server-methods/audit.test.ts @@ -0,0 +1,74 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { auditHandlers } from "./audit.js"; + +const listAuditEvents = vi.hoisted(() => vi.fn()); + +vi.mock("../../audit/audit-event-store.js", () => ({ listAuditEvents })); + +async function runAuditHandler(params: Record) { + const respond = vi.fn(); + await auditHandlers["audit.list"]({ params, respond } as never); + return respond; +} + +describe("audit.list", () => { + beforeEach(() => { + listAuditEvents.mockReset(); + listAuditEvents.mockReturnValue({ + events: [ + { + eventId: "event-1", + sequence: 10, + sourceSequence: 2, + occurredAt: 100, + kind: "agent_run", + action: "agent.run.finished", + status: "succeeded", + actorType: "agent", + actorId: "main", + agentId: "main", + runId: "run-1", + redaction: "metadata_only", + }, + ], + nextCursor: 10, + }); + }); + + it("passes bounded filters and maps the public actor shape", async () => { + const respond = await runAuditHandler({ + agentId: "main", + kind: "agent_run", + after: 50, + before: 150, + limit: 25, + cursor: "11", + }); + expect(listAuditEvents).toHaveBeenCalledWith({ + limit: 25, + cursor: 11, + filters: { agentId: "main", kind: "agent_run", after: 50, before: 150 }, + }); + expect(respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ + events: [expect.objectContaining({ actor: { type: "agent", id: "main" } })], + nextCursor: "10", + }), + ); + }); + + it("rejects malformed cursors and inverted ranges", async () => { + expect(await runAuditHandler({ cursor: "bad" })).toHaveBeenCalledWith( + false, + undefined, + expect.any(Object), + ); + expect(await runAuditHandler({ after: 2, before: 1 })).toHaveBeenCalledWith( + false, + undefined, + expect.any(Object), + ); + expect(listAuditEvents).not.toHaveBeenCalled(); + }); +}); diff --git a/src/gateway/server-methods/audit.ts b/src/gateway/server-methods/audit.ts new file mode 100644 index 000000000000..5fa04d28e329 --- /dev/null +++ b/src/gateway/server-methods/audit.ts @@ -0,0 +1,93 @@ +// Metadata-only operator audit queries over the canonical shared SQLite ledger. +import { + ErrorCodes, + errorShape, + formatValidationErrors, + type AuditEvent, + validateAuditListParams, +} from "../../../packages/gateway-protocol/src/index.js"; +import { listAuditEvents } from "../../audit/audit-event-store.js"; +import type { AuditEventRecord } from "../../audit/audit-event-types.js"; +import type { GatewayRequestHandlers } from "./types.js"; + +const DEFAULT_AUDIT_LIST_LIMIT = 100; +const MAX_AUDIT_LIST_LIMIT = 500; + +function parseAuditCursor(cursor: string | undefined): number | undefined | null { + if (cursor === undefined) { + return undefined; + } + if (!/^\d+$/.test(cursor)) { + return null; + } + const parsed = Number(cursor); + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null; +} + +function mapAuditEvent(event: AuditEventRecord): AuditEvent { + return { + eventId: event.eventId, + sequence: event.sequence, + sourceSequence: event.sourceSequence, + occurredAt: event.occurredAt, + kind: event.kind, + action: event.action, + status: event.status, + ...(event.errorCode ? { errorCode: event.errorCode } : {}), + actor: { type: event.actorType, id: event.actorId }, + agentId: event.agentId, + ...(event.sessionKey ? { sessionKey: event.sessionKey } : {}), + ...(event.sessionId ? { sessionId: event.sessionId } : {}), + runId: event.runId, + ...(event.toolCallId ? { toolCallId: event.toolCallId } : {}), + ...(event.toolName ? { toolName: event.toolName } : {}), + redaction: "metadata_only", + }; +} + +export const auditHandlers: GatewayRequestHandlers = { + "audit.list": ({ params, respond }) => { + if (!validateAuditListParams(params)) { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + `invalid audit.list params: ${formatValidationErrors(validateAuditListParams.errors)}`, + ), + ); + return; + } + const cursor = parseAuditCursor(params.cursor); + if ( + cursor === null || + (params.after !== undefined && params.before !== undefined && params.after > params.before) + ) { + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, "invalid audit.list range or cursor"), + ); + return; + } + const page = listAuditEvents({ + limit: Math.min(params.limit ?? DEFAULT_AUDIT_LIST_LIMIT, MAX_AUDIT_LIST_LIMIT), + ...(cursor !== undefined ? { cursor } : {}), + filters: { + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + ...(params.runId ? { runId: params.runId } : {}), + ...(params.kind ? { kind: params.kind } : {}), + ...(params.status ? { status: params.status } : {}), + ...(params.after !== undefined ? { after: params.after } : {}), + ...(params.before !== undefined ? { before: params.before } : {}), + }, + }); + respond(true, { + events: page.events.map(mapAuditEvent), + ...(page.nextCursor !== undefined ? { nextCursor: String(page.nextCursor) } : {}), + }); + }, +}; + +export const testApi = { mapAuditEvent, parseAuditCursor }; diff --git a/src/gateway/server-runtime-handles.ts b/src/gateway/server-runtime-handles.ts index 73e0c344c5ff..e605600b418d 100644 --- a/src/gateway/server-runtime-handles.ts +++ b/src/gateway/server-runtime-handles.ts @@ -36,7 +36,7 @@ export type GatewayServerMutableState = { stopModelPricingRefresh: () => void; mcpServer: { port: number; close: () => Promise } | undefined; configReloader: GatewayConfigReloaderHandle; - agentUnsub: (() => void) | null; + agentUnsub: (() => Promise | void) | null; heartbeatUnsub: (() => void) | null; transcriptUnsub: (() => void) | null; lifecycleUnsub: (() => void) | null; @@ -73,7 +73,7 @@ export function createGatewayServerMutableState(): GatewayServerMutableState { stopModelPricingRefresh: () => {}, mcpServer: undefined as { port: number; close: () => Promise } | undefined, configReloader: { stop: async () => {} } satisfies GatewayConfigReloaderHandle, - agentUnsub: null as (() => void) | null, + agentUnsub: null as (() => Promise | void) | null, heartbeatUnsub: null as (() => void) | null, transcriptUnsub: null as (() => void) | null, lifecycleUnsub: null as (() => void) | null, diff --git a/src/gateway/server-runtime-subscriptions.test.ts b/src/gateway/server-runtime-subscriptions.test.ts index 025d539938da..0dd2033601a2 100644 --- a/src/gateway/server-runtime-subscriptions.test.ts +++ b/src/gateway/server-runtime-subscriptions.test.ts @@ -28,6 +28,29 @@ const mockLog: SubsystemLogger = { child: () => mockLog, }; +const auditTestState = vi.hoisted(() => ({ + enabled: true, + created: 0, + stopped: 0, +})); + +vi.mock("../audit/audit-config.js", () => ({ + isAuditLedgerEnabled: () => auditTestState.enabled, +})); + +vi.mock("../audit/agent-event-audit.js", () => ({ + createAgentEventAuditRecorder: () => { + auditTestState.created += 1; + return { + record: vi.fn(), + recordTool: vi.fn(), + stop: vi.fn(async () => { + auditTestState.stopped += 1; + }), + }; + }, +})); + vi.mock("./server-chat.js", () => { throw new Error("server-chat lazy load failure"); }); @@ -69,16 +92,37 @@ describe("startGatewayEventSubscriptions", () => { beforeEach(() => { vi.clearAllMocks(); + auditTestState.enabled = true; + auditTestState.created = 0; + auditTestState.stopped = 0; }); - afterEach(() => { - unsubs?.agentUnsub(); + afterEach(async () => { + await unsubs?.agentUnsub(); unsubs?.heartbeatUnsub(); unsubs?.transcriptUnsub(); unsubs?.lifecycleUnsub(); resetAgentEventsForTest(); }); + it("records audit events by default and stops the recorder on unsubscribe", async () => { + unsubs = startGatewayEventSubscriptions(createParams()); + + expect(auditTestState.created).toBe(1); + await unsubs.agentUnsub(); + expect(auditTestState.stopped).toBe(1); + }); + + it("creates no audit recorder when audit.enabled is false", async () => { + auditTestState.enabled = false; + unsubs = startGatewayEventSubscriptions(createParams()); + + expect(auditTestState.created).toBe(0); + // Disabled wiring must still unsubscribe cleanly. + await unsubs.agentUnsub(); + expect(auditTestState.stopped).toBe(0); + }); + it("logs lazy agent event module failures", async () => { unsubs = startGatewayEventSubscriptions(createParams()); diff --git a/src/gateway/server-runtime-subscriptions.ts b/src/gateway/server-runtime-subscriptions.ts index 7c8c106202f3..5a275522e838 100644 --- a/src/gateway/server-runtime-subscriptions.ts +++ b/src/gateway/server-runtime-subscriptions.ts @@ -1,7 +1,10 @@ // Gateway event subscription wiring for agent, heartbeat, transcript, and lifecycle broadcasts. import { resolveDefaultAgentId } from "../agents/agent-scope.js"; +import { createAgentEventAuditRecorder } from "../audit/agent-event-audit.js"; +import { isAuditLedgerEnabled } from "../audit/audit-config.js"; import { getRuntimeConfig } from "../config/io.js"; -import { clearAgentRunContext, onAgentEvent } from "../infra/agent-events.js"; +import { clearAgentRunContext, onAgentAuditEvent, onAgentEvent } from "../infra/agent-events.js"; +import { onTrustedToolExecutionEvent } from "../infra/diagnostic-events.js"; import { onHeartbeatEvent } from "../infra/heartbeat-events.js"; import type { SubsystemLogger } from "../logging/subsystem.js"; import { onSessionLifecycleEvent } from "../sessions/session-lifecycle-events.js"; @@ -54,6 +57,18 @@ export function startGatewayEventSubscriptions(params: { chatAbortControllers: Map; restartRecoveryCandidates: Map; }) { + // audit.enabled=false stops ledger writes entirely; reads over existing + // records keep working. Resolved once at gateway startup like the other + // runtime subscriptions. + const auditRecorder = isAuditLedgerEnabled(getRuntimeConfig()) + ? createAgentEventAuditRecorder() + : undefined; + const unsubscribePrivateAuditEvents = auditRecorder + ? onAgentAuditEvent(auditRecorder.record) + : undefined; + const unsubscribeToolAuditEvents = auditRecorder + ? onTrustedToolExecutionEvent(auditRecorder.recordTool) + : undefined; const getAgentEventHandler = createLazyPromise( () => { // Lazy-load heavy chat modules only after the first agent event reaches the gateway. @@ -218,7 +233,8 @@ export function startGatewayEventSubscriptions(params: { return lifecycleEventHandlerPromise; }; - const agentUnsub = onAgentEvent((evt) => { + const unsubscribeAgentEvents = onAgentEvent((evt) => { + auditRecorder?.record(evt); const lifecyclePhase = evt.stream === "lifecycle" && typeof evt.data?.phase === "string" ? evt.data.phase @@ -269,6 +285,12 @@ export function startGatewayEventSubscriptions(params: { context: { runId: evt.runId, stream: evt.stream }, }); }); + const agentUnsub = async () => { + unsubscribeAgentEvents(); + unsubscribePrivateAuditEvents?.(); + unsubscribeToolAuditEvents?.(); + await auditRecorder?.stop(); + }; const heartbeatUnsub = onHeartbeatEvent((evt) => { params.broadcast("heartbeat", evt, { dropIfSlow: true }); diff --git a/src/gateway/server.chat.gateway-server-chat.test.ts b/src/gateway/server.chat.gateway-server-chat.test.ts index 365d7208b579..7fd093b2bc62 100644 --- a/src/gateway/server.chat.gateway-server-chat.test.ts +++ b/src/gateway/server.chat.gateway-server-chat.test.ts @@ -396,7 +396,7 @@ describe("gateway server chat", () => { expect(waitRes.ok).toBe(true); expectRecordFields(waitRes.payload, { runId: "idem-sessions-abort-1", - status: "timeout", + status: "error", stopReason: "rpc", }); } else { diff --git a/src/gateway/session-lifecycle-state.test.ts b/src/gateway/session-lifecycle-state.test.ts index f68a3a6db7b0..b6becf8c33dd 100644 --- a/src/gateway/session-lifecycle-state.test.ts +++ b/src/gateway/session-lifecycle-state.test.ts @@ -405,6 +405,16 @@ describe("session lifecycle state", () => { status: "timeout", abortedLastRun: false, }, + { + name: "maps abandoned lifecycle ends to failed sessions", + data: { + phase: "end", + livenessState: "abandoned", + endedAt: 1_550, + }, + status: "failed", + abortedLastRun: false, + }, ])("$name", ({ data, status, abortedLastRun }) => { expectPersistedLifecyclePatch({ data, diff --git a/src/gateway/session-lifecycle-state.ts b/src/gateway/session-lifecycle-state.ts index 6af16265c2c6..1003504681ba 100644 --- a/src/gateway/session-lifecycle-state.ts +++ b/src/gateway/session-lifecycle-state.ts @@ -67,6 +67,7 @@ function mapAgentRunTerminalOutcomeToSessionStatus( case "aborted": return "killed"; case "blocked": + case "abandoned": case "failed": return "failed"; default: diff --git a/src/gateway/tools-invoke-http.test.ts b/src/gateway/tools-invoke-http.test.ts index 8b3508f7d2cf..e5999b7364b8 100644 --- a/src/gateway/tools-invoke-http.test.ts +++ b/src/gateway/tools-invoke-http.test.ts @@ -532,6 +532,7 @@ describe("POST /tools/invoke", () => { setMainAllowedTools({ allow: ["tools_invoke_test"] }); hookMocks.runBeforeToolCallHook.mockResolvedValueOnce({ blocked: true, + kind: "veto", reason: "blocked by test hook", }); @@ -1115,6 +1116,8 @@ describe("tools.invoke Gateway RPC", () => { setMainAllowedTools({ allow: ["tools_invoke_test"] }); hookMocks.runBeforeToolCallHook.mockResolvedValueOnce({ blocked: true, + kind: "failure", + disposition: "blocked", deniedReason: "plugin-approval", reason: "Plugin approval required", params: { mode: "ok" }, diff --git a/src/infra/agent-events.test.ts b/src/infra/agent-events.test.ts index 9e665315f022..fe37c9cfbad8 100644 --- a/src/infra/agent-events.test.ts +++ b/src/infra/agent-events.test.ts @@ -5,10 +5,12 @@ import { captureAgentRunLifecycleGeneration, claimAgentRunContext, clearAgentRunContext, + emitAgentAuditEvent, emitAgentEvent, getAgentEventLifecycleGeneration, getAgentRunContext, listAgentRunsForSession, + onAgentAuditEvent, onAgentEvent, registerAgentRunContext, releaseAgentRunContext, @@ -68,6 +70,46 @@ describe("agent-events sequencing", () => { expect(seen).toEqual([1, 1]); }); + test("keeps audit-only events off the shared agent event bus", () => { + const shared: AgentEventPayload[] = []; + const audit: AgentEventPayload[] = []; + const stopShared = onAgentEvent((event) => shared.push(event)); + const stopAudit = onAgentAuditEvent((event) => audit.push(event)); + + emitAgentAuditEvent({ + runId: "audit-only-run", + sessionKey: "agent:main:acp:session", + stream: "lifecycle", + data: { phase: "start" }, + }); + emitAgentAuditEvent({ + runId: "audit-only-run", + sessionKey: "agent:main:acp:session", + stream: "lifecycle", + data: { phase: "end" }, + }); + emitAgentAuditEvent({ + runId: "audit-only-run", + sessionKey: "agent:main:acp:session", + stream: "lifecycle", + data: { phase: "start" }, + }); + + stopShared(); + stopAudit(); + expect(shared).toEqual([]); + expect(audit.map((event) => [event.data.phase, event.seq])).toEqual([ + ["start", 1], + ["end", 2], + ["start", 1], + ]); + expect(audit[0]).toMatchObject({ + runId: "audit-only-run", + sessionKey: "agent:main:acp:session", + stream: "lifecycle", + }); + }); + test("preserves sequence state when same-generation ownership is reclaimed", () => { const lifecycleGeneration = getAgentEventLifecycleGeneration(); claimAgentRunContext("retry-run", { @@ -492,6 +534,26 @@ describe("agent-events sequencing", () => { expect(receivedSessionKey).toBe("session-quietchat-context"); }); + test("stamps the resolved agent owner for unscoped session keys", () => { + registerAgentRunContext("run-unscoped", { + sessionKey: "global", + agentId: "support", + }); + + let received: AgentEventPayload | undefined; + const stop = onAgentEvent((event) => { + received = event; + }); + emitAgentEvent({ + runId: "run-unscoped", + stream: "lifecycle", + data: { phase: "start" }, + }); + stop(); + + expect(received).toMatchObject({ sessionKey: "global", agentId: "support" }); + }); + test("merges later run context updates into existing runs", () => { resetAgentRunContextForTest(); registerAgentRunContext("run-ctx", { diff --git a/src/infra/agent-events.ts b/src/infra/agent-events.ts index 5d84428e0158..fa4555121218 100644 --- a/src/infra/agent-events.ts +++ b/src/infra/agent-events.ts @@ -130,6 +130,8 @@ export type AgentEventPayload = { /** Per-run metadata used to stamp events and gate Control UI visibility. */ export type AgentRunContext = { sessionKey?: string; + /** Resolved agent owner, including for unscoped session keys. */ + agentId?: string; /** Owning run's sessionId; stamped onto lifecycle events (see AgentEventPayload.sessionId). */ sessionId?: string; /** Gateway lifecycle generation captured when the run was registered. */ @@ -147,6 +149,7 @@ export type AgentRunContext = { type AgentEventState = { seqByRun: Map; listeners: Set<(evt: AgentEventPayload) => void>; + auditListeners: Set<(evt: AgentEventPayload) => void>; runContextById: Map; runContextOwnersById?: Map< string, @@ -171,6 +174,7 @@ function getAgentEventState(): AgentEventState { return resolveGlobalSingleton(AGENT_EVENT_STATE_KEY, () => ({ seqByRun: new Map(), listeners: new Set<(evt: AgentEventPayload) => void>(), + auditListeners: new Set<(evt: AgentEventPayload) => void>(), runContextById: new Map(), lifecycleGeneration: randomUUID(), })); @@ -244,6 +248,9 @@ export function registerAgentRunContext(runId: string, context: AgentRunContext) if (context.sessionId && existing.sessionId !== context.sessionId) { existing.sessionId = context.sessionId; } + if (context.agentId && existing.agentId !== context.agentId) { + existing.agentId = context.agentId; + } if (context.verboseLevel && existing.verboseLevel !== context.verboseLevel) { existing.verboseLevel = context.verboseLevel; } @@ -414,8 +421,9 @@ export function resetAgentRunContextForTest() { getAgentRunContextOwners(state).clear(); } -/** Emits an agent event after assigning per-run sequence, timestamp, and context metadata. */ -export function emitAgentEvent(event: Omit) { +function enrichAgentEvent( + event: Omit, +): AgentEventPayload | undefined { const state = getAgentEventState(); const context = state.runContextById.get(event.runId); const executionLifecycleGeneration = @@ -426,10 +434,10 @@ export function emitAgentEvent(event: Omit) { context?.lifecycleGeneration && executionLifecycleGeneration !== context.lifecycleGeneration ) { - return; + return undefined; } if (ownedLifecycleGeneration && ownedLifecycleGeneration !== state.lifecycleGeneration) { - return; + return undefined; } const nextSeq = (state.seqByRun.get(event.runId) ?? 0) + 1; state.seqByRun.set(event.runId, nextSeq); @@ -455,10 +463,12 @@ export function emitAgentEvent(event: Omit) { event.stream === "lifecycle" ? (ownedLifecycleGeneration ?? state.lifecycleGeneration) : ownedLifecycleGeneration; + const agentId = event.agentId ?? context?.agentId; const enriched: AgentEventPayload = { ...event, sessionKey, ...(sessionId ? { sessionId } : {}), + ...(agentId ? { agentId } : {}), seq: nextSeq, ts: Date.now(), }; @@ -470,7 +480,30 @@ export function emitAgentEvent(event: Omit) { enumerable: false, }); } - notifyListeners(state.listeners, enriched); + return enriched; +} + +/** Emits an agent event after assigning per-run sequence, timestamp, and context metadata. */ +export function emitAgentEvent(event: Omit) { + const enriched = enrichAgentEvent(event); + if (enriched) { + notifyListeners(getAgentEventState().listeners, enriched); + } +} + +/** Emits run metadata only to the Gateway-owned durable audit projection. */ +export function emitAgentAuditEvent(event: Omit) { + const state = getAgentEventState(); + const enriched = enrichAgentEvent(event); + if (enriched) { + notifyListeners(state.auditListeners, enriched); + const phase = event.stream === "lifecycle" ? event.data.phase : undefined; + if ((phase === "end" || phase === "error") && !state.runContextById.has(event.runId)) { + // Private synthetic runs bypass public terminal cleanup. Release sequence state only + // after synchronous audit listeners consume the terminal event and its final ordering. + state.seqByRun.delete(event.runId); + } + } } /** Emits an item activity event on the shared agent event bus. */ @@ -535,11 +568,17 @@ export function onAgentEvent(listener: (evt: AgentEventPayload) => void) { return registerListener(state.listeners, listener); } +/** Subscribes to private audit-only agent events; returns an unsubscribe callback. */ +export function onAgentAuditEvent(listener: (evt: AgentEventPayload) => void) { + return registerListener(getAgentEventState().auditListeners, listener); +} + /** Clears all agent event state, including listeners; test-only helper. */ export function resetAgentEventsForTest() { const state = getAgentEventState(); state.seqByRun.clear(); state.listeners.clear(); + state.auditListeners.clear(); state.runContextById.clear(); getAgentRunContextOwners(state).clear(); } diff --git a/src/infra/diagnostic-events.ts b/src/infra/diagnostic-events.ts index 651ccd22e336..09d780e9e006 100644 --- a/src/infra/diagnostic-events.ts +++ b/src/infra/diagnostic-events.ts @@ -443,11 +443,15 @@ export type DiagnosticToolParamsSummary = | { kind: "number" | "boolean" | "null" | "undefined" | "other" }; export type DiagnosticToolSource = "channel" | "core" | "mcp" | "plugin"; +export type DiagnosticToolTerminalReason = "failed" | "cancelled" | "timed_out"; type DiagnosticToolExecutionBaseEvent = DiagnosticBaseEvent & { runId?: string; sessionKey?: string; sessionId?: string; + agentId?: string; + /** Authoritative lifecycle time from the tool runtime, when it exposes one. */ + sourceTimestampMs?: number; toolName: string; toolSource?: DiagnosticToolSource; toolOwner?: string; @@ -469,6 +473,7 @@ export type DiagnosticToolExecutionErrorEvent = DiagnosticToolExecutionBaseEvent durationMs: number; errorCategory: string; errorCode?: string; + terminalReason?: DiagnosticToolTerminalReason; }; export type DiagnosticToolExecutionBlockedEvent = DiagnosticToolExecutionBaseEvent & { @@ -803,6 +808,11 @@ export type DiagnosticEventInput = DiagnosticNonSecurityEventPayload extends inf : never : never; +type TrustedToolExecutionEventInput = Extract< + DiagnosticEventInput, + { type: TrustedToolExecutionEvent["type"] } +>; + type DiagnosticDispatchInput = DiagnosticEventInput | Omit; export type DiagnosticEventMetadata = Readonly<{ @@ -839,6 +849,19 @@ type TrustedDiagnosticEventListener = ( privateData: DiagnosticEventPrivateData, ) => void; +export type TrustedToolExecutionEvent = Extract< + DiagnosticEventPayload, + { + type: + | "tool.execution.started" + | "tool.execution.completed" + | "tool.execution.error" + | "tool.execution.blocked"; + } +>; + +type TrustedToolExecutionEventListener = (event: TrustedToolExecutionEvent) => void; + type QueuedDiagnosticEvent = { event: DiagnosticEventPayload; metadata: DiagnosticEventMetadata; @@ -851,6 +874,8 @@ type DiagnosticEventsGlobalState = { seq: number; listeners: Set; trustedListeners: Set; + toolExecutionListeners: Set; + toolExecutionSeq: number; dispatchDepth: number; asyncQueue: QueuedDiagnosticEvent[]; asyncDrainScheduled: boolean; @@ -898,6 +923,8 @@ function createDiagnosticEventsState(): DiagnosticEventsGlobalState { seq: 0, listeners: new Set(), trustedListeners: new Set(), + toolExecutionListeners: new Set(), + toolExecutionSeq: 0, dispatchDepth: 0, asyncQueue: [], asyncDrainScheduled: false, @@ -919,6 +946,8 @@ function isDiagnosticEventsState(value: unknown): value is DiagnosticEventsGloba typeof candidate.seq === "number" && candidate.listeners instanceof Set && (candidate.trustedListeners === undefined || candidate.trustedListeners instanceof Set) && + (candidate.toolExecutionListeners === undefined || + candidate.toolExecutionListeners instanceof Set) && typeof candidate.dispatchDepth === "number" && Array.isArray(candidate.asyncQueue) && typeof candidate.asyncDrainScheduled === "boolean" @@ -934,6 +963,8 @@ function getDiagnosticEventsState(): DiagnosticEventsGlobalState { existing.asyncDroppedUntrustedEvents ??= 0; existing.asyncDroppedPriorityEvents ??= 0; existing.trustedListeners ??= new Set(); + existing.toolExecutionListeners ??= new Set(); + existing.toolExecutionSeq ??= 0; return existing; } const state = createDiagnosticEventsState(); @@ -1183,6 +1214,9 @@ function emitDiagnosticEventWithTrust( options: EmitDiagnosticEventOptions = {}, ) { const state = getDiagnosticEventsState(); + if (trusted && isToolExecutionEventInput(event)) { + dispatchTrustedToolExecutionEvent(state, event); + } if (!state.enabled) { return; } @@ -1217,6 +1251,44 @@ function emitDiagnosticEventWithTrust( dispatchDiagnosticEvent(state, enriched, metadata, privateData); } +function isToolExecutionEventInput( + event: DiagnosticDispatchInput, +): event is TrustedToolExecutionEventInput { + return ( + event.type === "tool.execution.started" || + event.type === "tool.execution.completed" || + event.type === "tool.execution.error" || + event.type === "tool.execution.blocked" + ); +} + +function dispatchTrustedToolExecutionEvent( + state: DiagnosticEventsGlobalState, + event: TrustedToolExecutionEventInput, +): void { + state.toolExecutionSeq += 1; + let enriched: TrustedToolExecutionEvent; + try { + enriched = deepFreezeDiagnosticValue( + structuredClone({ ...event, seq: state.toolExecutionSeq, ts: Date.now() }), + ) as TrustedToolExecutionEvent; + } catch (error) { + console.error( + `[diagnostic-events] tool execution clone error type=${event.type}: ${String(error)}`, + ); + return; + } + for (const listener of state.toolExecutionListeners) { + try { + listener(enriched); + } catch (error) { + console.error( + `[diagnostic-events] tool execution listener error type=${enriched.type} seq=${enriched.seq}: ${String(error)}`, + ); + } + } +} + /** Emits an untrusted diagnostic event from external/plugin-facing code. */ export function emitDiagnosticEvent(event: DiagnosticEventInput) { emitDiagnosticEventWithTrust(event, false); @@ -1291,6 +1363,17 @@ export function onTrustedInternalDiagnosticEvent( }; } +/** Subscribes to trusted metadata-only tool execution events, even when diagnostics are disabled. */ +export function onTrustedToolExecutionEvent( + listener: TrustedToolExecutionEventListener, +): () => void { + const state = getDiagnosticEventsState(); + state.toolExecutionListeners.add(listener); + return () => { + state.toolExecutionListeners.delete(listener); + }; +} + /** Checks currently queued async diagnostic events without draining the queue. */ export function hasPendingInternalDiagnosticEvent( predicate: (event: DiagnosticEventPayload, metadata: DiagnosticEventMetadata) => boolean, @@ -1343,6 +1426,8 @@ export function resetDiagnosticEventsForTest(): void { state.seq = 0; state.listeners.clear(); state.trustedListeners.clear(); + state.toolExecutionListeners.clear(); + state.toolExecutionSeq = 0; state.dispatchDepth = 0; state.asyncQueue = []; state.asyncDrainScheduled = false; diff --git a/src/logging/diagnostic-stability.ts b/src/logging/diagnostic-stability.ts index 069990b2619c..ebaa9e28addb 100644 --- a/src/logging/diagnostic-stability.ts +++ b/src/logging/diagnostic-stability.ts @@ -406,6 +406,9 @@ function sanitizeDiagnosticEvent(event: DiagnosticEventPayload): DiagnosticStabi record.source = event.toolSource; record.pluginId = event.toolOwner; record.durationMs = event.durationMs; + if (event.terminalReason) { + record.outcome = event.terminalReason; + } assignReasonCode(record, event.errorCategory); break; case "tool.execution.blocked": diff --git a/src/plugin-sdk/agent-harness-runtime.ts b/src/plugin-sdk/agent-harness-runtime.ts index e0a76d2404bb..b2f540d867f5 100644 --- a/src/plugin-sdk/agent-harness-runtime.ts +++ b/src/plugin-sdk/agent-harness-runtime.ts @@ -119,6 +119,7 @@ export { formatApprovalDisplayPath } from "../infra/approval-display-paths.js"; export { buildAgentHookContextChannelFields } from "../plugins/hook-agent-context.js"; export { emitAgentEvent, onAgentEvent, resetAgentEventsForTest } from "../infra/agent-events.js"; export { runAgentCleanupStep } from "../agents/run-cleanup-timeout.js"; +export { resolveAgentRunAbortLifecycleFields } from "../agents/run-termination.js"; export { log as embeddedAgentLog } from "../agents/embedded-agent-runner/logger.js"; export { buildAgentRuntimePlan } from "../agents/runtime-plan/build.js"; export { @@ -150,6 +151,12 @@ export { isToolResultError, sanitizeToolResult, } from "../agents/embedded-agent-subscribe.tools.js"; +export { + formatToolExecutionErrorMessage, + resolveToolExecutionErrorKind, + resolveToolResultFailureKind, + type ToolResultFailureKind, +} from "../agents/tool-result-error.js"; export { normalizeUsage } from "../agents/usage.js"; export { resolveOpenClawAgentDir } from "./agent-dir-compat.js"; export { @@ -306,6 +313,7 @@ export { consumeAdjustedParamsForToolCall, consumePreExecutionBlockedToolCall, finalizeToolTerminalPresentation, + getBeforeToolCallFailureDisposition, getBeforeToolCallPolicyDiagnosticState, hasBeforeToolCallPolicy, isToolWrappedWithBeforeToolCallHook, @@ -314,6 +322,7 @@ export { setBeforeToolCallDiagnosticsEnabled, wrapToolWithBeforeToolCallHook, type BeforeToolCallPolicyDiagnosticState, + type BeforeToolCallFailureDisposition, type DeferredPluginToolApproval, } from "../agents/agent-tools.before-tool-call.js"; export { isReplaySafeToolCall } from "../agents/tool-mutation.js"; diff --git a/src/shared/agent-liveness.ts b/src/shared/agent-liveness.ts index a98b23f3ae60..e9950d4f4684 100644 --- a/src/shared/agent-liveness.ts +++ b/src/shared/agent-liveness.ts @@ -3,12 +3,23 @@ export function isBlockedLivenessState(livenessState: unknown): boolean { return typeof livenessState === "string" && livenessState.trim().toLowerCase() === "blocked"; } +/** Return true for the normalized liveness state that means a run ended incomplete. */ +export function isAbandonedLivenessState(livenessState: unknown): boolean { + return typeof livenessState === "string" && livenessState.trim().toLowerCase() === "abandoned"; +} + /** Convert a blocked-run error payload into a user-facing wait/status message. */ export function formatBlockedLivenessError(error: unknown): string { const message = typeof error === "string" ? error.trim() : ""; return message || "Agent run blocked before producing a usable result."; } +/** Convert an abandoned-run error payload into a user-facing wait/status message. */ +export function formatAbandonedLivenessError(error: unknown): string { + const message = typeof error === "string" ? error.trim() : ""; + return message || "Agent run ended before producing a complete result."; +} + /** Coerce any blocked liveness state into an error status while preserving other statuses. */ export function normalizeBlockedLivenessWaitStatus< TStatus extends "ok" | "error" | "timeout" | "pending", diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index 1d885db9f7cd..8210bdfbf3e4 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -80,6 +80,26 @@ export interface ApnsRegistrations { updated_at_ms: number; } +export interface AuditEvents { + action: string; + actor_id: string; + actor_type: string; + agent_id: string; + error_code: string | null; + event_id: string; + kind: string; + occurred_at: number; + run_id: string; + sequence: Generated; + session_id: string | null; + session_key: string | null; + source_id: string; + source_sequence: number; + status: string; + tool_call_id: string | null; + tool_name: string | null; +} + export interface AuthProfileState { state_json: string; store_key: string; @@ -991,6 +1011,7 @@ export interface DB { agent_model_catalogs: AgentModelCatalogs; android_notification_recent_packages: AndroidNotificationRecentPackages; apns_registrations: ApnsRegistrations; + audit_events: AuditEvents; auth_profile_state: AuthProfileState; auth_profile_stores: AuthProfileStores; backup_runs: BackupRuns; diff --git a/src/state/openclaw-state-schema.generated.ts b/src/state/openclaw-state-schema.generated.ts index 7c80a3409460..b7cdf05d2852 100644 --- a/src/state/openclaw-state-schema.generated.ts +++ b/src/state/openclaw-state-schema.generated.ts @@ -26,6 +26,44 @@ CREATE TABLE IF NOT EXISTS diagnostic_events ( CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created ON diagnostic_events(scope, created_at, event_key); +CREATE TABLE IF NOT EXISTS audit_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + source_id TEXT NOT NULL UNIQUE, + source_sequence INTEGER NOT NULL, + occurred_at INTEGER NOT NULL, + kind TEXT NOT NULL, + action TEXT NOT NULL, + status TEXT NOT NULL, + error_code TEXT, + actor_type TEXT NOT NULL, + actor_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_key TEXT, + session_id TEXT, + run_id TEXT NOT NULL, + tool_call_id TEXT, + tool_name TEXT +); + +CREATE INDEX IF NOT EXISTS idx_audit_events_time + ON audit_events(occurred_at DESC, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_agent_sequence + ON audit_events(agent_id, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_session_sequence + ON audit_events(session_key, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_run_sequence + ON audit_events(run_id, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_kind_sequence + ON audit_events(kind, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_status_sequence + ON audit_events(status, sequence DESC); + CREATE TABLE IF NOT EXISTS diagnostic_stability_bundles ( bundle_key TEXT NOT NULL PRIMARY KEY, reason TEXT NOT NULL, diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index 224f6e0ae491..6f33bec2018e 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -21,6 +21,44 @@ CREATE TABLE IF NOT EXISTS diagnostic_events ( CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created ON diagnostic_events(scope, created_at, event_key); +CREATE TABLE IF NOT EXISTS audit_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + source_id TEXT NOT NULL UNIQUE, + source_sequence INTEGER NOT NULL, + occurred_at INTEGER NOT NULL, + kind TEXT NOT NULL, + action TEXT NOT NULL, + status TEXT NOT NULL, + error_code TEXT, + actor_type TEXT NOT NULL, + actor_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_key TEXT, + session_id TEXT, + run_id TEXT NOT NULL, + tool_call_id TEXT, + tool_name TEXT +); + +CREATE INDEX IF NOT EXISTS idx_audit_events_time + ON audit_events(occurred_at DESC, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_agent_sequence + ON audit_events(agent_id, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_session_sequence + ON audit_events(session_key, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_run_sequence + ON audit_events(run_id, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_kind_sequence + ON audit_events(kind, sequence DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_status_sequence + ON audit_events(status, sequence DESC); + CREATE TABLE IF NOT EXISTS diagnostic_stability_bundles ( bundle_key TEXT NOT NULL PRIMARY KEY, reason TEXT NOT NULL, diff --git a/src/tasks/task-registry.ts b/src/tasks/task-registry.ts index f44b5d18d5c2..b0e0445fca60 100644 --- a/src/tasks/task-registry.ts +++ b/src/tasks/task-registry.ts @@ -573,6 +573,7 @@ function mapAgentRunTerminalOutcomeToTaskStatus( case "aborted": return "cancelled"; case "blocked": + case "abandoned": case "failed": return "failed"; default: diff --git a/test/release-check.test.ts b/test/release-check.test.ts index c7ecf431fc36..d0668ea6b8bc 100644 --- a/test/release-check.test.ts +++ b/test/release-check.test.ts @@ -678,6 +678,7 @@ describe("collectMissingPackPaths", () => { "scripts/postinstall-bundled-plugins.mjs", "dist/agents/compaction-planning.worker.js", "dist/agents/model-provider-auth.worker.js", + "dist/audit/audit-event-writer.worker.js", "dist/task-registry-control.runtime.js", "dist/telegram-ingress-worker.runtime.js", bundledDistPluginFile("telegram", "runtime-api.js"), @@ -712,6 +713,7 @@ describe("collectMissingPackPaths", () => { "dist/plugin-sdk/root-alias.cjs", "dist/agents/compaction-planning.worker.js", "dist/agents/model-provider-auth.worker.js", + "dist/audit/audit-event-writer.worker.js", "dist/task-registry-control.runtime.js", "dist/telegram-ingress-worker.runtime.js", "dist/build-info.json", diff --git a/test/scripts/lint-suppressions.test.ts b/test/scripts/lint-suppressions.test.ts index 53cecdd37dd0..e94bf8e24eb2 100644 --- a/test/scripts/lint-suppressions.test.ts +++ b/test/scripts/lint-suppressions.test.ts @@ -195,6 +195,7 @@ describe("production lint suppressions", () => { "extensions/matrix/src/onboarding.test-harness.ts|typescript/no-unnecessary-type-parameters|1", "extensions/slack/src/monitor/provider-support.ts|typescript/no-unnecessary-type-parameters|1", "src/agents/agent-bundle-mcp-runtime.ts|unicorn/prefer-add-event-listener|1", + "src/audit/audit-event-writer.ts|unicorn/require-post-message-target-origin|2", "src/channels/plugins/channel-runtime-surface.types.ts|typescript/no-unnecessary-type-parameters|1", "src/channels/plugins/contracts/test-helpers.ts|typescript/no-unnecessary-type-parameters|1", "src/channels/plugins/types.plugin.ts|typescript/no-explicit-any|1", diff --git a/tsdown.config.ts b/tsdown.config.ts index d8c894a25be4..88abe4e5e4f5 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -266,6 +266,7 @@ function buildCoreDistEntries(): Record { "agents/code-mode.worker": "src/agents/code-mode.worker.ts", "agents/compaction-planning.worker": "src/agents/compaction-planning.worker.ts", "agents/model-provider-auth.worker": "src/agents/model-provider-auth.worker.ts", + "audit/audit-event-writer.worker": "src/audit/audit-event-writer.worker.ts", "acp/control-plane/manager": "src/acp/control-plane/manager.ts", "cli/gateway-lifecycle.runtime": "src/cli/gateway-cli/lifecycle.runtime.ts", "provider-dispatcher.runtime": "src/auto-reply/reply/provider-dispatcher.runtime.ts",