From e8fcc93cd3b6724c3eebaadd81effe43abbc117a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 9 Jul 2026 20:11:20 -0700 Subject: [PATCH] fix(talk): await realtime tool result delivery (#103268) * fix(talk): await realtime tool result delivery * fix(talk): terminally cancel queued results * chore: keep release changelog out of PR --------- Co-authored-by: Peter Steinberger --- docs/gateway/protocol.md | 2 +- docs/plugins/sdk-migration.md | 20 +- docs/plugins/sdk-provider-plugins.md | 12 + .../discord/src/voice/manager.e2e.test.ts | 291 +++- extensions/discord/src/voice/realtime.ts | 159 +- .../google-meet/src/agent-consult.test.ts | 145 ++ extensions/google-meet/src/agent-consult.ts | 92 +- extensions/google-meet/src/realtime-node.ts | 2 +- extensions/google-meet/src/realtime.ts | 2 +- extensions/google/index.test.ts | 1 + extensions/google/index.ts | 1 + .../google/realtime-voice-provider.test.ts | 17 +- extensions/google/realtime-voice-provider.ts | 1 + .../openai/realtime-voice-provider.test.ts | 27 +- extensions/openai/realtime-voice-provider.ts | 1 + .../src/webhook/realtime-handler.test.ts | 222 ++- .../src/webhook/realtime-handler.ts | 78 +- src/gateway/server-methods/talk-session.ts | 2 +- src/gateway/server-methods/talk.test.ts | 33 +- src/gateway/talk-realtime-relay.test.ts | 1440 ++++++++++++++++- src/gateway/talk-realtime-relay.ts | 633 ++++++-- src/talk/forced-consult-coordinator.test.ts | 23 +- src/talk/forced-consult-coordinator.ts | 7 +- src/talk/provider-types.ts | 12 +- src/talk/session-runtime.test.ts | 157 +- src/talk/session-runtime.ts | 55 +- .../chat/realtime-talk-gateway-relay.test.ts | 150 ++ .../pages/chat/realtime-talk-gateway-relay.ts | 56 +- .../chat/realtime-talk-google-live.test.ts | 68 +- .../pages/chat/realtime-talk-google-live.ts | 25 +- ui/src/pages/chat/realtime-talk-shared.ts | 46 +- .../pages/chat/realtime-talk-webrtc.test.ts | 46 + ui/src/pages/chat/realtime-talk-webrtc.ts | 12 +- 33 files changed, 3447 insertions(+), 391 deletions(-) create mode 100644 extensions/google-meet/src/agent-consult.test.ts diff --git a/docs/gateway/protocol.md b/docs/gateway/protocol.md index 55421ccb0828..192cf0abe82e 100644 --- a/docs/gateway/protocol.md +++ b/docs/gateway/protocol.md @@ -400,7 +400,7 @@ methods. Treat this as feature discovery, not a full enumeration of - `talk.session.appendAudio` appends base64 PCM input audio to gateway-owned realtime relay and transcription sessions. - `talk.session.startTurn`, `talk.session.endTurn`, and `talk.session.cancelTurn` drive managed-room turn lifecycle with stale-turn rejection before state clears. - `talk.session.cancelOutput` stops assistant audio output, primarily for VAD-gated barge-in in gateway relay sessions. - - `talk.session.submitToolResult` completes a provider tool call emitted by a gateway-owned realtime relay session. Pass `options: { willContinue: true }` for interim tool output when a final result follows, or `options: { suppressResponse: true }` when the tool result should satisfy the provider call without starting another realtime response. + - `talk.session.submitToolResult` completes a provider tool call emitted by a gateway-owned realtime relay session. The request waits for any asynchronous completion signal exposed by the provider bridge; failed submissions keep the linked run active and do not emit a successful tool-result event. Pass `options: { willContinue: true }` for interim tool output or `options: { suppressResponse: true }` when the provider bridge advertises suppression support and the result should not start another response. - `talk.session.steer` sends active-run voice control into a gateway-owned agent-backed Talk session: `{ sessionId, text, mode? }`, where `mode` is `status`, `steer`, `cancel`, or `followup`; omitted mode is classified from the spoken text. - `talk.session.close` closes a gateway-owned relay, transcription, or managed-room session and emits terminal Talk events. - `talk.mode` sets/broadcasts the current Talk mode state for WebChat/Control UI clients. diff --git a/docs/plugins/sdk-migration.md b/docs/plugins/sdk-migration.md index 0b02766a51e8..8c397f9f00b2 100644 --- a/docs/plugins/sdk-migration.md +++ b/docs/plugins/sdk-migration.md @@ -952,16 +952,16 @@ Method map for readers migrating from the older `talk.realtime.*` / The unified control vocabulary is also deliberately narrow: -| Method | Applies to | Contract | -| ------------------------------- | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `talk.session.appendAudio` | `realtime/gateway-relay`, `transcription/gateway-relay` | Append a base64 PCM audio chunk to the provider session owned by the same Gateway connection. | -| `talk.session.startTurn` | `stt-tts/managed-room` | Start a managed-room user turn. | -| `talk.session.endTurn` | `stt-tts/managed-room` | End the active turn after stale-turn validation. | -| `talk.session.cancelTurn` | all Gateway-owned sessions | Cancel active capture/provider/agent/TTS work for a turn. | -| `talk.session.cancelOutput` | `realtime/gateway-relay` | Stop assistant audio output without necessarily ending the user turn. | -| `talk.session.submitToolResult` | `realtime/gateway-relay` | Complete a provider tool call emitted by the relay; pass `options.willContinue` for interim output or `options.suppressResponse` to satisfy the call without another assistant response. | -| `talk.session.steer` | agent-backed Talk sessions | Send spoken `status`, `steer`, `cancel`, or `followup` control to the active embedded run resolved from the Talk session. | -| `talk.session.close` | all unified sessions | Stop relay sessions or revoke managed-room state, then forget the unified session id. | +| Method | Applies to | Contract | +| ------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `talk.session.appendAudio` | `realtime/gateway-relay`, `transcription/gateway-relay` | Append a base64 PCM audio chunk to the provider session owned by the same Gateway connection. | +| `talk.session.startTurn` | `stt-tts/managed-room` | Start a managed-room user turn. | +| `talk.session.endTurn` | `stt-tts/managed-room` | End the active turn after stale-turn validation. | +| `talk.session.cancelTurn` | all Gateway-owned sessions | Cancel active capture/provider/agent/TTS work for a turn. | +| `talk.session.cancelOutput` | `realtime/gateway-relay` | Stop assistant audio output without necessarily ending the user turn. | +| `talk.session.submitToolResult` | `realtime/gateway-relay` | Complete a provider tool call after any asynchronous completion exposed by its bridge; pass `options.willContinue` for interim output or, when supported, `options.suppressResponse` to avoid another assistant response. | +| `talk.session.steer` | agent-backed Talk sessions | Send spoken `status`, `steer`, `cancel`, or `followup` control to the active embedded run resolved from the Talk session. | +| `talk.session.close` | all unified sessions | Stop relay sessions or revoke managed-room state, then forget the unified session id. | Do not introduce provider or platform special cases in core to make this work. Core owns Talk session semantics. Provider plugins own vendor session setup. diff --git a/docs/plugins/sdk-provider-plugins.md b/docs/plugins/sdk-provider-plugins.md index f4e40d761f3d..99a784bdf300 100644 --- a/docs/plugins/sdk-provider-plugins.md +++ b/docs/plugins/sdk-provider-plugins.md @@ -834,6 +834,18 @@ catalog, API-key auth, and dynamic model resolution. clients. Implement `handleBargeIn` when a transport can detect that a human is interrupting assistant playback and the provider supports truncating or clearing the active audio response. + `submitToolResult` may return `void` for synchronous submission, or a + `Promise` for an asynchronous completion boundary the provider + bridge can expose. Gateway relay sessions wait for that promise before + confirming a final result or clearing the linked run; reject it when + submission fails. + Set `supportsToolResultSuppression: false` when the provider cannot + honor `options.suppressResponse`. OpenClaw then avoids suppression for + internal forced-consult and cancellation results, and rejects direct + suppressed-result requests instead of silently starting a response. + Consumers of `createRealtimeVoiceBridgeSession` may likewise return a + promise from `onToolCall`; synchronous throws and rejections are routed + to the session's `onError` callback. Set `handlesInputAudioBargeIn` only when provider VAD confirms an interruption by calling `onClearAudio("barge-in")`. Providers that omit the flag use OpenClaw's local input-audio fallback detection. diff --git a/extensions/discord/src/voice/manager.e2e.test.ts b/extensions/discord/src/voice/manager.e2e.test.ts index bb749f8f3d35..97e879645aa7 100644 --- a/extensions/discord/src/voice/manager.e2e.test.ts +++ b/extensions/discord/src/voice/manager.e2e.test.ts @@ -1,6 +1,9 @@ // Discord tests cover manager plugin behavior. import { PassThrough, type Readable } from "node:stream"; -import type { RealtimeVoiceAgentControlResult } from "openclaw/plugin-sdk/realtime-voice"; +import type { + RealtimeVoiceAgentControlResult, + RealtimeVoiceForcedConsultCoordinator, +} from "openclaw/plugin-sdk/realtime-voice"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { ChannelType } from "../internal/discord.js"; import { createVoiceCaptureState } from "./capture-state.js"; @@ -102,7 +105,10 @@ const { const getVoiceConnectionMockLocal = vi.fn((): MockConnection | undefined => undefined); const realtimeSessionMockLocal = { - bridge: { supportsToolResultContinuation: true }, + bridge: { + supportsToolResultContinuation: true, + supportsToolResultSuppression: true as boolean | undefined, + }, acknowledgeMark: vi.fn(), close: vi.fn(), connect: vi.fn(async () => undefined), @@ -371,6 +377,7 @@ describe("DiscordVoiceManager", () => { realtimeSessionMock.handleBargeIn.mockClear(); realtimeSessionMock.setMediaTimestamp.mockClear(); realtimeSessionMock.submitToolResult.mockClear(); + realtimeSessionMock.bridge.supportsToolResultSuppression = true; createRealtimeVoiceBridgeSessionMock.mockClear(); createRealtimeVoiceBridgeSessionMock.mockReturnValue(realtimeSessionMock); controlRealtimeVoiceAgentRunMock.mockReset(); @@ -2417,6 +2424,104 @@ describe("DiscordVoiceManager", () => { ); }); + it("keeps the realtime tool callback pending until result delivery completes", async () => { + let acceptResult = () => {}; + const accepted = new Promise((resolve) => { + acceptResult = resolve; + }); + realtimeSessionMock.submitToolResult.mockImplementationOnce(() => accepted); + const manager = createManager({ + groupPolicy: "open", + voice: { + enabled: true, + mode: "agent-proxy", + realtime: { provider: "openai" }, + }, + }); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const bridgeParams = lastRealtimeBridgeParams() as + | { + onToolCall?: ( + event: { + itemId: string; + callId: string; + name: string; + args: unknown; + }, + session: typeof realtimeSessionMock, + ) => Promise; + } + | undefined; + + const handled = bridgeParams?.onToolCall?.( + { + itemId: "item-unknown", + callId: "call-unknown", + name: "unknown_tool", + args: {}, + }, + realtimeSessionMock, + ); + if (!handled) { + throw new Error("expected realtime tool callback promise"); + } + let settled = false; + void handled.then(() => { + settled = true; + }); + await Promise.resolve(); + + expect(realtimeSessionMock.submitToolResult).toHaveBeenCalledTimes(1); + expect(settled).toBe(false); + acceptResult(); + await handled; + expect(settled).toBe(true); + }); + + it("does not retry a rejected control result submission as a tool error", async () => { + realtimeSessionMock.submitToolResult.mockRejectedValueOnce(new Error("result delivery failed")); + const manager = createManager({ + groupPolicy: "open", + voice: { + enabled: true, + mode: "agent-proxy", + realtime: { provider: "openai" }, + }, + }); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const bridgeParams = lastRealtimeBridgeParams() as + | { + onToolCall?: ( + event: { + itemId: string; + callId: string; + name: string; + args: unknown; + }, + session: typeof realtimeSessionMock, + ) => Promise; + } + | undefined; + + const handled = bridgeParams?.onToolCall?.( + { + itemId: "item-control", + callId: "call-control", + name: "openclaw_agent_control", + args: { text: "check this", mode: "steer" }, + }, + realtimeSessionMock, + ); + if (!handled) { + throw new Error("expected realtime tool callback promise"); + } + + await expect(handled).rejects.toThrow("result delivery failed"); + expect(realtimeSessionMock.submitToolResult).toHaveBeenCalledTimes(1); + }); + it("rejects malformed realtime consult tool calls without crashing Discord voice", async () => { const manager = createManager({ groupPolicy: "open", @@ -4246,10 +4351,186 @@ describe("DiscordVoiceManager", () => { "call-late", { status: "already_delivered", - message: "OpenClaw already delivered this answer to Discord voice.", + message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it.", }, { suppressResponse: true }, ); + + realtimeSessionMock.bridge.supportsToolResultSuppression = false; + bridgeParams?.onToolCall?.( + { + itemId: "item-late-unsuppressed", + callId: "call-late-unsuppressed", + name: "openclaw_agent_consult", + args: { question: "late question" }, + }, + realtimeSessionMock, + ); + await vi.waitFor(() => { + const call = realtimeSessionMock.submitToolResult.mock.calls.find( + ([callId]) => callId === "call-late-unsuppressed", + ); + expect(call).toEqual([ + "call-late-unsuppressed", + { + status: "already_delivered", + message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it.", + }, + ]); + }); + }); + + it("terminally satisfies a late native call for a cancelled forced consult", async () => { + const manager = createManager({ + groupPolicy: "open", + voice: { + enabled: true, + mode: "agent-proxy", + realtime: { provider: "openai" }, + }, + }); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const entry = getSessionEntry(manager) as { + realtime?: unknown; + }; + const realtime = entry.realtime as { + forcedConsults: RealtimeVoiceForcedConsultCoordinator; + }; + const cancelled = realtime.forcedConsults.prepare("cancelled question"); + if (!cancelled) { + throw new Error("expected forced consult handle"); + } + realtime.forcedConsults.markStarted(cancelled); + realtime.forcedConsults.markCancelled(cancelled); + const bridgeParams = lastRealtimeBridgeParams() as + | { + onToolCall?: ( + event: { + itemId: string; + callId: string; + name: string; + args: unknown; + }, + session: typeof realtimeSessionMock, + ) => Promise; + } + | undefined; + + await bridgeParams?.onToolCall?.( + { + itemId: "item-cancelled", + callId: "call-cancelled", + name: "openclaw_agent_consult", + args: { question: "cancelled question" }, + }, + realtimeSessionMock, + ); + + expect(agentCommandMock).not.toHaveBeenCalled(); + expect(realtimeSessionMock.submitToolResult).toHaveBeenCalledWith( + "call-cancelled", + { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + { suppressResponse: true }, + ); + }); + + it("lets an unsuppressed in-flight native result own forced consult delivery", async () => { + let resolveAgentTurn: ((result: { payloads: Array<{ text: string }> }) => void) | undefined; + agentCommandMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveAgentTurn = resolve; + }), + ); + const manager = createManager({ + groupPolicy: "open", + voice: { + enabled: true, + mode: "agent-proxy", + realtime: { provider: "openai" }, + }, + }); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const entry = getSessionEntry(manager) as { + realtime?: { + beginSpeakerTurn: ( + context: { extraSystemPrompt?: string; senderIsOwner: boolean; speakerLabel: string }, + userId: string, + ) => { sendInputAudio: (audio: Buffer) => void }; + }; + }; + const bridgeParams = lastRealtimeBridgeParams() as + | { + onToolCall?: ( + event: { + itemId: string; + callId: string; + name: string; + args: unknown; + }, + session: typeof realtimeSessionMock, + ) => Promise; + onTranscript?: (role: "user" | "assistant", text: string, isFinal: boolean) => void; + } + | undefined; + const ownerTurn = entry.realtime?.beginSpeakerTurn( + { extraSystemPrompt: undefined, senderIsOwner: true, speakerLabel: "Owner" }, + "u-owner", + ); + ownerTurn?.sendInputAudio(Buffer.alloc(8)); + await emitFinalRealtimeUserTranscript(bridgeParams, "late question"); + realtimeSessionMock.bridge.supportsToolResultSuppression = false; + + const submission = bridgeParams?.onToolCall?.( + { + itemId: "item-late", + callId: "call-late", + name: "openclaw_agent_consult", + args: { question: "late question" }, + }, + realtimeSessionMock, + ); + resolveAgentTurn?.({ payloads: [{ text: "forced answer" }] }); + await submission; + + expect(realtimeSessionMock.submitToolResult).toHaveBeenCalledWith("call-late", { + text: "forced answer", + }); + expectUserMessageNotIncludes("forced answer"); + expectUserMessageNotIncludes("I hit an error while checking that. Please try again."); + + let resolveRetryTurn: ((result: { payloads: Array<{ text: string }> }) => void) | undefined; + agentCommandMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveRetryTurn = resolve; + }), + ); + const retryTurn = entry.realtime?.beginSpeakerTurn( + { extraSystemPrompt: undefined, senderIsOwner: true, speakerLabel: "Owner" }, + "u-owner", + ); + retryTurn?.sendInputAudio(Buffer.alloc(8)); + await emitFinalRealtimeUserTranscript(bridgeParams, "retry question"); + realtimeSessionMock.submitToolResult.mockRejectedValueOnce( + new Error("native delivery rejected"), + ); + const rejectedSubmission = bridgeParams?.onToolCall?.( + { + itemId: "item-retry", + callId: "call-retry", + name: "openclaw_agent_consult", + args: { question: "retry question" }, + }, + realtimeSessionMock, + ); + resolveRetryTurn?.({ payloads: [{ text: "local retry answer" }] }); + + await expect(rejectedSubmission).rejects.toThrow("native delivery rejected"); + await vi.waitFor(() => expectUserMessageIncludes("local retry answer")); }); it("suppresses late forced agent-proxy tool calls when the forced consult rejects", async () => { @@ -4315,7 +4596,7 @@ describe("DiscordVoiceManager", () => { "call-late", { status: "already_delivered", - message: "OpenClaw already delivered this answer to Discord voice.", + message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it.", }, { suppressResponse: true }, ), @@ -4485,7 +4766,7 @@ describe("DiscordVoiceManager", () => { "call-new", { status: "already_delivered", - message: "OpenClaw already delivered this answer to Discord voice.", + message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it.", }, { suppressResponse: true }, ); diff --git a/extensions/discord/src/voice/realtime.ts b/extensions/discord/src/voice/realtime.ts index 5366590210ff..8986038b6ac0 100644 --- a/extensions/discord/src/voice/realtime.ts +++ b/extensions/discord/src/voice/realtime.ts @@ -133,6 +133,7 @@ type RecentAgentProxyConsultResult = type AgentProxyConsultState = { speaker: DiscordRealtimeSpeakerContext; handledByForcedPlayback?: boolean; + providerDelivery?: Promise; promise?: Promise; result?: RecentAgentProxyConsultResult; }; @@ -1107,21 +1108,21 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { ); } - private handleToolCall( + private async handleToolCall( event: RealtimeVoiceToolCallEvent, session: RealtimeVoiceBridgeSession, - ): void { + ): Promise { const callId = event.callId || event.itemId || "unknown"; if (event.name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) { - void this.handleAgentControlToolCall(event, session, callId); + await this.handleAgentControlToolCall(event, session, callId); return; } if (event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { - session.submitToolResult(callId, { error: `Tool "${event.name}" not available` }); + await session.submitToolResult(callId, { error: `Tool "${event.name}" not available` }); return; } if (this.consultToolPolicy === "none") { - session.submitToolResult(callId, { error: `Tool "${event.name}" not available` }); + await session.submitToolResult(callId, { error: `Tool "${event.name}" not available` }); return; } const exactSpeechText = extractDiscordExactSpeechConsultText(event.args); @@ -1129,7 +1130,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { logger.info( `discord voice: realtime exact speech consult bypassed call=${callId || "unknown"} answerChars=${exactSpeechText.length}`, ); - session.submitToolResult(callId, { text: exactSpeechText }); + await session.submitToolResult(callId, { text: exactSpeechText }); return; } let consultMessage: string; @@ -1140,13 +1141,23 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { logger.warn( `discord voice: realtime consult rejected malformed args call=${callId || "unknown"}: ${message}`, ); - session.submitToolResult(callId, { error: message }); + await session.submitToolResult(callId, { error: message }); return; } logger.info( `discord voice: realtime consult requested call=${callId || "unknown"} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} question=${formatVoiceLogPreview(consultMessage)}`, ); const nativeConsult = this.forcedConsults.recordNativeConsult(event.args, callId); + if ( + nativeConsult.kind === "already_delivered" && + this.forcedConsults.isCancelled(nativeConsult.handle) + ) { + await this.submitTerminalRealtimeToolResult(callId, session, { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }); + return; + } const pendingConsult = nativeConsult.kind === "pending" ? nativeConsult.handle : undefined; if (pendingConsult) { this.forcedConsults.rememberQuestion(pendingConsult, consultMessage); @@ -1164,12 +1175,12 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { logger.info( `discord voice: realtime consult matched recent agent result but newer speaker audio is pending call=${callId} speaker=${recentSpeaker?.speakerLabel ?? "unknown"} owner=${recentSpeaker?.senderIsOwner ?? false}`, ); - session.submitToolResult(callId, { + await session.submitToolResult(callId, { error: "Discord speaker context changed before this realtime consult completed", }); return; } - if (this.submitRecentAgentProxyConsultResult(callId, recentConsult, session)) { + if (await this.submitRecentAgentProxyConsultResult(callId, recentConsult, session)) { return; } } @@ -1187,7 +1198,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { logger.warn( `discord voice: realtime consult has no speaker context call=${callId || "unknown"}`, ); - session.submitToolResult(callId, { error: "No Discord speaker context available" }); + await session.submitToolResult(callId, { error: "No Discord speaker context available" }); return; } const promise = this.runAgentTurn({ @@ -1197,19 +1208,19 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { if (recent) { this.setRecentAgentProxyConsultPromise(recent, promise); } - void promise - .then((text) => { - logger.info( - `discord voice: realtime consult answer (${text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(text)}`, - ); - session.submitToolResult(callId, { text }); - }) - .catch((error: unknown) => { - logger.warn( - `discord voice: realtime consult failed call=${callId || "unknown"}: ${formatErrorMessage(error)}`, - ); - session.submitToolResult(callId, { error: formatErrorMessage(error) }); - }); + let text: string; + try { + text = await promise; + } catch (error) { + const message = formatErrorMessage(error); + logger.warn(`discord voice: realtime consult failed call=${callId || "unknown"}: ${message}`); + await session.submitToolResult(callId, { error: message }); + return; + } + logger.info( + `discord voice: realtime consult answer (${text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(text)}`, + ); + await session.submitToolResult(callId, { text }); } private async handleAgentControlToolCall( @@ -1217,18 +1228,20 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { session: RealtimeVoiceBridgeSession, callId: string, ): Promise { + let result: RealtimeVoiceAgentControlResult; try { const parsed = parseRealtimeVoiceAgentControlToolArgs(event.args); - const result = await controlRealtimeVoiceAgentRun({ + result = await controlRealtimeVoiceAgentRun({ sessionKey: this.params.entry.route.sessionKey, text: parsed.text, mode: parsed.mode, }); - this.logAgentControlResult(result); - session.submitToolResult(callId, result); } catch (error) { - session.submitToolResult(callId, { error: formatErrorMessage(error) }); + await session.submitToolResult(callId, { error: formatErrorMessage(error) }); + return; } + this.logAgentControlResult(result); + await session.submitToolResult(callId, result); } private async runAgentTurn(params: { @@ -1493,17 +1506,21 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { }); this.setRecentAgentProxyConsultPromise(pending, promise); const text = await promise; + await state.providerDelivery; logger.info( `discord voice: realtime forced agent consult answer (${text.length} chars) elapsedMs=${Date.now() - startedAt} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}: ${formatVoiceLogPreview(text)}`, ); - if (text.trim()) { + if (text.trim() && state.handledByForcedPlayback) { this.enqueueExactSpeechMessage(text); } } catch (error) { + await state.providerDelivery; logger.warn( `discord voice: realtime forced agent consult failed elapsedMs=${Date.now() - startedAt}: ${formatErrorMessage(error)}`, ); - this.enqueueExactSpeechMessage(DISCORD_REALTIME_FALLBACK_TEXT); + if (state.handledByForcedPlayback) { + this.enqueueExactSpeechMessage(DISCORD_REALTIME_FALLBACK_TEXT); + } } } @@ -1618,41 +1635,65 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { return this.forcedConsults.findRecent(consultMessage); } - private submitRecentAgentProxyConsultResult( + private async submitTerminalRealtimeToolResult( + callId: string, + session: RealtimeVoiceBridgeSession, + result: Record, + ): Promise { + // Providers without suppressed results still need a terminal result; the payload tells the + // model not to repeat audio that Discord already played or restart cancelled work. + if (session.bridge.supportsToolResultSuppression === false) { + await session.submitToolResult(callId, result); + return; + } + await session.submitToolResult(callId, result, { suppressResponse: true }); + } + + private async submitRecentAgentProxyConsultResult( callId: string, recent: AgentProxyConsultHandle, session: RealtimeVoiceBridgeSession, - ): boolean { + ): Promise { const state = recent.context; if (!state) { return false; } - const submitAlreadyDelivered = () => { - session.submitToolResult( - callId, - { - status: "already_delivered", - message: "OpenClaw already delivered this answer to Discord voice.", - }, - { suppressResponse: true }, - ); + const providerOwnsDelivery = Boolean( + state.handledByForcedPlayback && + state.promise && + !state.result && + session.bridge.supportsToolResultSuppression === false, + ); + let resolveProviderDelivery: ((accepted: boolean) => void) | undefined; + if (providerOwnsDelivery) { + // Forced playback waits for native acceptance so a failed delivery can restore + // the local success/fallback path instead of losing the answer entirely. + state.providerDelivery = new Promise((resolve) => { + resolveProviderDelivery = resolve; + }); + } + const submitAlreadyDelivered = async (): Promise => { + await this.submitTerminalRealtimeToolResult(callId, session, { + status: "already_delivered", + message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it.", + }); }; - const submitResult = (result: RecentAgentProxyConsultResult) => { - if (state.handledByForcedPlayback) { - submitAlreadyDelivered(); + const submitResult = async (result: RecentAgentProxyConsultResult): Promise => { + if (state.handledByForcedPlayback && !providerOwnsDelivery) { + await submitAlreadyDelivered(); return; } if (result.status === "fulfilled") { - session.submitToolResult(callId, { text: result.text }); + await session.submitToolResult(callId, { text: result.text }); return; } - session.submitToolResult(callId, { error: result.error }); + await session.submitToolResult(callId, { error: result.error }); }; if (state.result) { logger.info( `discord voice: realtime consult reused recent agent result call=${callId || "unknown"} speaker=${state.speaker.speakerLabel} owner=${state.speaker.senderIsOwner}`, ); - submitResult(state.result); + await submitResult(state.result); return true; } if (!state.promise) { @@ -1661,15 +1702,27 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { logger.info( `discord voice: realtime consult joined in-flight agent result call=${callId || "unknown"} speaker=${state.speaker.speakerLabel} owner=${state.speaker.senderIsOwner}`, ); - if (state.handledByForcedPlayback) { - void state.promise.then(submitAlreadyDelivered, submitAlreadyDelivered); + if (state.handledByForcedPlayback && !providerOwnsDelivery) { + await state.promise.catch(() => undefined); + await submitAlreadyDelivered(); return true; } - void state.promise - .then((text) => session.submitToolResult(callId, { text })) - .catch((error: unknown) => - session.submitToolResult(callId, { error: formatErrorMessage(error) }), - ); + let result: RecentAgentProxyConsultResult; + try { + result = { status: "fulfilled", text: await state.promise }; + } catch (error) { + result = { status: "rejected", error: formatErrorMessage(error) }; + } + try { + await submitResult(result); + if (providerOwnsDelivery) { + state.handledByForcedPlayback = false; + resolveProviderDelivery?.(true); + } + } catch (error) { + resolveProviderDelivery?.(false); + throw error; + } return true; } } diff --git a/extensions/google-meet/src/agent-consult.test.ts b/extensions/google-meet/src/agent-consult.test.ts new file mode 100644 index 000000000000..6aa5f4cac442 --- /dev/null +++ b/extensions/google-meet/src/agent-consult.test.ts @@ -0,0 +1,145 @@ +import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime"; +import type { + RealtimeVoiceBridgeSession, + TalkEventInput, +} from "openclaw/plugin-sdk/realtime-voice"; +import { describe, expect, it, vi } from "vitest"; +import { handleGoogleMeetRealtimeConsultToolCall } from "./agent-consult.js"; +import { resolveGoogleMeetConfig } from "./config.js"; + +function makeSession( + submitToolResult: RealtimeVoiceBridgeSession["submitToolResult"], + supportsToolResultContinuation = false, +): RealtimeVoiceBridgeSession { + return { + bridge: { supportsToolResultContinuation }, + submitToolResult, + } as unknown as RealtimeVoiceBridgeSession; +} + +function makeRuntime() { + const sessions: Record> = {}; + const runEmbeddedAgent = vi.fn(async () => ({ + payloads: [{ text: "Use the launch summary." }], + meta: {}, + })); + const runtime = { + agent: { + resolveAgentDir: vi.fn(() => "/tmp/agent"), + resolveAgentWorkspaceDir: vi.fn(() => "/tmp/workspace"), + ensureAgentWorkspace: vi.fn(async () => {}), + resolveAgentTimeoutMs: vi.fn(() => 1_000), + session: { + resolveStorePath: vi.fn(() => "/tmp/sessions.json"), + loadSessionStore: vi.fn(() => sessions), + saveSessionStore: vi.fn(async () => {}), + updateSessionStore: vi.fn( + async ( + _storePath: string, + update: (store: Record>) => unknown, + ) => await update(sessions), + ), + getSessionEntry: vi.fn(({ sessionKey }: { sessionKey: string }) => sessions[sessionKey]), + patchSessionEntry: vi.fn( + async ({ + sessionKey, + fallbackEntry, + update, + }: { + sessionKey: string; + fallbackEntry: Record; + update: ( + entry: Record, + ) => Promise> | Record; + }) => { + const current = sessions[sessionKey] ?? fallbackEntry; + const patch = await update(current); + const next = { ...current, ...patch }; + sessions[sessionKey] = next; + return next; + }, + ), + resolveSessionFilePath: vi.fn(() => "/tmp/session.json"), + }, + runEmbeddedAgent, + }, + } as unknown as PluginRuntime; + return { runEmbeddedAgent, runtime }; +} + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +} as unknown as RuntimeLogger; + +describe("handleGoogleMeetRealtimeConsultToolCall", () => { + it("emits a final tool event only after the bridge accepts the result", async () => { + let acceptResult = () => {}; + const accepted = new Promise((resolve) => { + acceptResult = resolve; + }); + const submitToolResult = vi.fn(() => accepted); + const events: TalkEventInput[] = []; + + const handled = handleGoogleMeetRealtimeConsultToolCall({ + strategy: "agent", + session: makeSession(submitToolResult), + event: { + itemId: "item-1", + callId: "call-1", + name: "openclaw_agent_consult", + args: { question: "What should I say?" }, + }, + config: resolveGoogleMeetConfig({}), + fullConfig: {}, + runtime: {} as PluginRuntime, + logger, + meetingSessionId: "meet-1", + transcript: [], + onTalkEvent: (event) => events.push(event), + }); + + expect(submitToolResult).toHaveBeenCalledTimes(1); + expect(events).toEqual([]); + acceptResult(); + await handled; + expect(events).toEqual([ + expect.objectContaining({ type: "tool.error", callId: "call-1", final: true }), + ]); + }); + + it("does not retry a rejected result submission as a second tool error", async () => { + const deliveryError = new Error("result delivery failed"); + const submitToolResult = vi.fn(async () => { + throw deliveryError; + }); + const events: TalkEventInput[] = []; + const { runEmbeddedAgent, runtime } = makeRuntime(); + + await expect( + handleGoogleMeetRealtimeConsultToolCall({ + strategy: "bidi", + session: makeSession(submitToolResult), + event: { + itemId: "item-1", + callId: "call-1", + name: "openclaw_agent_consult", + args: { question: "What should I say?" }, + }, + config: resolveGoogleMeetConfig({ realtime: { agentId: "jay" } }), + fullConfig: {}, + runtime, + logger, + meetingSessionId: "meet-1", + transcript: [], + onTalkEvent: (event) => events.push(event), + }), + ).rejects.toThrow("result delivery failed"); + + expect(runEmbeddedAgent).toHaveBeenCalledTimes(1); + expect(submitToolResult).toHaveBeenCalledTimes(1); + expect(events.map((event) => event.type)).toEqual(["tool.progress"]); + }); +}); diff --git a/extensions/google-meet/src/agent-consult.ts b/extensions/google-meet/src/agent-consult.ts index 2873280f2f84..187f8644ec83 100644 --- a/extensions/google-meet/src/agent-consult.ts +++ b/extensions/google-meet/src/agent-consult.ts @@ -29,16 +29,20 @@ export function resolveGoogleMeetRealtimeTools(policy: GoogleMeetToolPolicy): Re return resolveRealtimeVoiceAgentConsultTools(policy); } -function submitGoogleMeetConsultWorkingResponse( +async function submitGoogleMeetConsultWorkingResponse( session: RealtimeVoiceBridgeSession, callId: string, -): void { +): Promise { if (!session.bridge.supportsToolResultContinuation) { return; } - session.submitToolResult(callId, buildRealtimeVoiceAgentConsultWorkingResponse("participant"), { - willContinue: true, - }); + await session.submitToolResult( + callId, + buildRealtimeVoiceAgentConsultWorkingResponse("participant"), + { + willContinue: true, + }, + ); } export async function consultOpenClawAgentForGoogleMeet(params: { @@ -77,7 +81,7 @@ export async function consultOpenClawAgentForGoogleMeet(params: { }); } -export function handleGoogleMeetRealtimeConsultToolCall(params: { +export async function handleGoogleMeetRealtimeConsultToolCall(params: { strategy: string; session: RealtimeVoiceBridgeSession; event: RealtimeVoiceToolCallEvent; @@ -89,69 +93,67 @@ export function handleGoogleMeetRealtimeConsultToolCall(params: { requesterSessionKey?: string; transcript: Array<{ role: "user" | "assistant"; text: string }>; onTalkEvent?: (event: TalkEventInput) => void; -}): void { +}): Promise { const callId = params.event.callId || params.event.itemId; if (params.strategy !== "bidi") { + const error = `Tool "${params.event.name}" is only available in bidi realtime strategy`; + await params.session.submitToolResult(callId, { error }); params.onTalkEvent?.({ type: "tool.error", callId, payload: { name: params.event.name, - error: `Tool "${params.event.name}" is only available in bidi realtime strategy`, + error, }, final: true, }); - params.session.submitToolResult(callId, { - error: `Tool "${params.event.name}" is only available in bidi realtime strategy`, - }); return; } if (params.event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { + const error = `Tool "${params.event.name}" not available`; + await params.session.submitToolResult(callId, { error }); params.onTalkEvent?.({ type: "tool.error", callId, - payload: { name: params.event.name, error: `Tool "${params.event.name}" not available` }, + payload: { name: params.event.name, error }, final: true, }); - params.session.submitToolResult(callId, { - error: `Tool "${params.event.name}" not available`, - }); return; } + await submitGoogleMeetConsultWorkingResponse(params.session, callId); params.onTalkEvent?.({ type: "tool.progress", callId, payload: { name: params.event.name, status: "working" }, }); - submitGoogleMeetConsultWorkingResponse(params.session, callId); - void consultOpenClawAgentForGoogleMeet({ - config: params.config, - fullConfig: params.fullConfig, - runtime: params.runtime, - logger: params.logger, - meetingSessionId: params.meetingSessionId, - requesterSessionKey: params.requesterSessionKey, - args: params.event.args, - transcript: params.transcript, - }) - .then((result) => { - params.onTalkEvent?.({ - type: "tool.result", - callId, - payload: { name: params.event.name, result }, - final: true, - }); - params.session.submitToolResult(callId, result); - }) - .catch((error: unknown) => { - params.onTalkEvent?.({ - type: "tool.error", - callId, - payload: { name: params.event.name, error: formatErrorMessage(error) }, - final: true, - }); - params.session.submitToolResult(callId, { - error: formatErrorMessage(error), - }); + let result: { text: string }; + try { + result = await consultOpenClawAgentForGoogleMeet({ + config: params.config, + fullConfig: params.fullConfig, + runtime: params.runtime, + logger: params.logger, + meetingSessionId: params.meetingSessionId, + requesterSessionKey: params.requesterSessionKey, + args: params.event.args, + transcript: params.transcript, }); + } catch (error) { + const message = formatErrorMessage(error); + await params.session.submitToolResult(callId, { error: message }); + params.onTalkEvent?.({ + type: "tool.error", + callId, + payload: { name: params.event.name, error: message }, + final: true, + }); + return; + } + await params.session.submitToolResult(callId, result); + params.onTalkEvent?.({ + type: "tool.result", + callId, + payload: { name: params.event.name, result }, + final: true, + }); } diff --git a/extensions/google-meet/src/realtime-node.ts b/extensions/google-meet/src/realtime-node.ts index 9c62ce45579b..e8af2161dd2c 100644 --- a/extensions/google-meet/src/realtime-node.ts +++ b/extensions/google-meet/src/realtime-node.ts @@ -677,7 +677,7 @@ export async function startNodeRealtimeAudioBridge(params: { payload: { name: event.name, args: event.args }, }); const turnId = ensureTalkTurn(); - handleGoogleMeetRealtimeConsultToolCall({ + return handleGoogleMeetRealtimeConsultToolCall({ strategy, session, event, diff --git a/extensions/google-meet/src/realtime.ts b/extensions/google-meet/src/realtime.ts index 1af94fa7b4e2..a93965f3de8c 100644 --- a/extensions/google-meet/src/realtime.ts +++ b/extensions/google-meet/src/realtime.ts @@ -1260,7 +1260,7 @@ export async function startCommandRealtimeAudioBridge(params: { payload: { name: event.name, args: event.args }, }); const turnId = ensureTalkTurn(); - handleGoogleMeetRealtimeConsultToolCall({ + return handleGoogleMeetRealtimeConsultToolCall({ strategy, session, event, diff --git a/extensions/google/index.test.ts b/extensions/google/index.test.ts index 8d550c1c44a8..42127b672bf2 100644 --- a/extensions/google/index.test.ts +++ b/extensions/google/index.test.ts @@ -405,6 +405,7 @@ describe("google provider plugin hooks", () => { throw new Error("expected Google realtime bridge"); } expect(bridge.supportsToolResultContinuation).toBe(false); + expect(bridge.supportsToolResultSuppression).toBe(false); expect(bridge.sendAudio(Buffer.alloc(160))).toBeUndefined(); expect(bridge.setMediaTimestamp(20)).toBeUndefined(); expect(bridge.sendUserMessage?.("hello")).toBeUndefined(); diff --git a/extensions/google/index.ts b/extensions/google/index.ts index dbccdf83b31e..2434f83622d9 100644 --- a/extensions/google/index.ts +++ b/extensions/google/index.ts @@ -248,6 +248,7 @@ function createLazyGoogleRealtimeVoiceBridge( get supportsToolResultContinuation() { return bridge?.supportsToolResultContinuation ?? false; }, + supportsToolResultSuppression: false, connect: async () => { const loadedBridge = await loadBridge(); if (closed) { diff --git a/extensions/google/realtime-voice-provider.test.ts b/extensions/google/realtime-voice-provider.test.ts index af55a0e1eee8..6593eae03b97 100644 --- a/extensions/google/realtime-voice-provider.test.ts +++ b/extensions/google/realtime-voice-provider.test.ts @@ -188,6 +188,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { }); expect(bridge.supportsToolResultContinuation).toBe(false); + expect(bridge.supportsToolResultSuppression).toBe(false); await bridge.connect(); const params = lastConnectParams(); @@ -1295,7 +1296,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { args: { query: "hi" }, }); - bridge.submitToolResult("call-1", { result: "ok" }); + void bridge.submitToolResult("call-1", { result: "ok" }); expect(session.sendToolResponse).toHaveBeenCalledWith({ functionResponses: [ @@ -1330,12 +1331,12 @@ describe("buildGoogleRealtimeVoiceProvider", () => { }, }); - bridge.submitToolResult( + void bridge.submitToolResult( "consult-call", { status: "working", message: "Tell the participant you are checking." }, { willContinue: true }, ); - bridge.submitToolResult("consult-call", { text: "The meeting starts at 3." }); + void bridge.submitToolResult("consult-call", { text: "The meeting starts at 3." }); expect(session.sendToolResponse).toHaveBeenNthCalledWith(1, { functionResponses: [ @@ -1381,13 +1382,13 @@ describe("buildGoogleRealtimeVoiceProvider", () => { }, }); - bridge.submitToolResult("consult-call", { status: "working" }, { willContinue: true }); + void bridge.submitToolResult("consult-call", { status: "working" }, { willContinue: true }); expect(session.sendToolResponse).not.toHaveBeenCalled(); expect(requireFirstError(onError).message).toContain( "does not support continuing tool responses", ); - bridge.submitToolResult("consult-call", { text: "The meeting starts at 3." }); + void bridge.submitToolResult("consult-call", { text: "The meeting starts at 3." }); expect(session.sendToolResponse).toHaveBeenCalledWith({ functionResponses: [ @@ -1412,7 +1413,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { await bridge.connect(); - bridge.submitToolResult("missing-call", { result: "ok" }); + void bridge.submitToolResult("missing-call", { result: "ok" }); expect(session.sendToolResponse).not.toHaveBeenCalled(); const error = requireFirstError(onError); @@ -1444,11 +1445,11 @@ describe("buildGoogleRealtimeVoiceProvider", () => { throw sendError; }); - bridge.submitToolResult("call-1", ["retryable"]); + void bridge.submitToolResult("call-1", ["retryable"]); expect(onError).toHaveBeenCalledWith(sendError); - bridge.submitToolResult("call-1", { result: "ok" }); + void bridge.submitToolResult("call-1", { result: "ok" }); expect(session.sendToolResponse).toHaveBeenLastCalledWith({ functionResponses: [ diff --git a/extensions/google/realtime-voice-provider.ts b/extensions/google/realtime-voice-provider.ts index a0957653d00b..a077ff632482 100644 --- a/extensions/google/realtime-voice-provider.ts +++ b/extensions/google/realtime-voice-provider.ts @@ -494,6 +494,7 @@ function formatGoogleLiveCloseEvent( class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { readonly supportsToolResultContinuation: boolean; + readonly supportsToolResultSuppression = false; private session: GoogleLiveSession | null = null; private connected = false; diff --git a/extensions/openai/realtime-voice-provider.test.ts b/extensions/openai/realtime-voice-provider.test.ts index 210b15b92ece..7f1406862e6a 100644 --- a/extensions/openai/realtime-voice-provider.test.ts +++ b/extensions/openai/realtime-voice-provider.test.ts @@ -316,6 +316,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { }); expect(bridge.supportsToolResultContinuation).toBe(true); + expect(bridge.supportsToolResultSuppression).toBe(true); }); it("adds OpenClaw attribution headers to native realtime websocket requests", () => { @@ -1938,7 +1939,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), ); - bridge.submitToolResult("call_1", { text: "done" }); + void bridge.submitToolResult("call_1", { text: "done" }); expect(parseSent(socket).slice(-1)).toEqual([ { @@ -2154,7 +2155,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" }))); await connecting; - bridge.submitToolResult("call_1", { status: "working" }, { willContinue: true }); + void bridge.submitToolResult("call_1", { status: "working" }, { willContinue: true }); expect(parseSent(socket).slice(-1)).toEqual([ { @@ -2168,7 +2169,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { ]); expect(hasSentEventType(socket, "response.create")).toBe(false); - bridge.submitToolResult("call_1", { text: "done" }); + void bridge.submitToolResult("call_1", { text: "done" }); expect(parseSent(socket).slice(-3)).toEqual([ { @@ -2209,7 +2210,11 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" }))); await connecting; - bridge.submitToolResult("call_1", { status: "already_delivered" }, { suppressResponse: true }); + void bridge.submitToolResult( + "call_1", + { status: "already_delivered" }, + { suppressResponse: true }, + ); expect(parseSent(socket).slice(-1)).toEqual([ { @@ -2248,13 +2253,13 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { "message", Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), ); - bridge.submitToolResult("call_1", { status: "working" }, { willContinue: true }); + void bridge.submitToolResult("call_1", { status: "working" }, { willContinue: true }); socket.emit("message", Buffer.from(JSON.stringify({ type: "response.done" }))); expect(onError).not.toHaveBeenCalled(); expect(parseSent(socket).filter((event) => event.type === "response.create")).toEqual([]); - bridge.submitToolResult("call_1", { text: "done" }); + void bridge.submitToolResult("call_1", { text: "done" }); expect(parseSent(socket).slice(-3)).toEqual([ { @@ -2292,7 +2297,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), ); - bridge.submitToolResult("call_1", { text: "done" }); + void bridge.submitToolResult("call_1", { text: "done" }); socket.emit("message", Buffer.from(JSON.stringify({ type: "response.cancelled" }))); expect(parseSent(socket).slice(-1)).toEqual([expectedResponseCreateEvent()]); @@ -2530,7 +2535,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), ); - bridge.submitToolResult("call_1", { text: "done" }); + void bridge.submitToolResult("call_1", { text: "done" }); bridge.handleBargeIn?.({ audioPlaybackActive: true }); const responseCancelEvent = parseSent(socket).findLast( (event) => event.type === "response.cancel", @@ -2598,7 +2603,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { if (!responseCancelEvent?.event_id) { throw new Error("expected response.cancel event id"); } - bridge.submitToolResult("call_1", { text: "done" }); + void bridge.submitToolResult("call_1", { text: "done" }); socket.emit("message", Buffer.from(JSON.stringify({ type: "response.done" }))); const sessionUpdateCount = parseSent(socket).filter( (event) => event.type === "session.update", @@ -2656,7 +2661,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { "message", Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), ); - bridge.submitToolResult("call_1", { text: "done" }); + void bridge.submitToolResult("call_1", { text: "done" }); expect(parseSent(socket).slice(-1)[0]?.type).toBe("conversation.item.create"); @@ -2706,7 +2711,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" }))); await connecting; - bridge.submitToolResult("call_1", { text: "done" }); + void bridge.submitToolResult("call_1", { text: "done" }); const responseCreateEvent = parseSent(socket).findLast( (event) => event.type === "response.create", ); diff --git a/extensions/openai/realtime-voice-provider.ts b/extensions/openai/realtime-voice-provider.ts index 949f4746164b..9d567d441037 100644 --- a/extensions/openai/realtime-voice-provider.ts +++ b/extensions/openai/realtime-voice-provider.ts @@ -471,6 +471,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { private static readonly BASE_RECONNECT_DELAY_MS = 1000; private static readonly CONNECT_TIMEOUT_MS = 10_000; readonly supportsToolResultContinuation = true; + readonly supportsToolResultSuppression = true; private ws: WebSocket | null = null; private connected = false; diff --git a/extensions/voice-call/src/webhook/realtime-handler.test.ts b/extensions/voice-call/src/webhook/realtime-handler.test.ts index a50004638e25..f70c0466b0b9 100644 --- a/extensions/voice-call/src/webhook/realtime-handler.test.ts +++ b/extensions/voice-call/src/webhook/realtime-handler.test.ts @@ -2,6 +2,7 @@ import http from "node:http"; import type { RealtimeVoiceBridge, + RealtimeVoiceForcedConsultCoordinator, RealtimeVoiceProviderPlugin, RealtimeVoiceToolCallEvent, } from "openclaw/plugin-sdk/realtime-voice"; @@ -1036,8 +1037,45 @@ describe("RealtimeCallHandler path routing", () => { } | undefined; let resolveConsult: ((value: unknown) => void) | undefined; + let resolveWorkingSubmission: (() => void) | undefined; + let rejectWorkingSubmission = false; + const resolveFinalSubmissions: Array<() => void> = []; let receivedPartialTranscript: string | undefined; - const submitToolResult = vi.fn(); + const submitToolResult = vi.fn( + (_callId: string, result: unknown, _options?: unknown): void | Promise => { + if ( + rejectWorkingSubmission && + result && + typeof result === "object" && + "status" in result && + result.status === "working" + ) { + return Promise.reject(new Error("working result rejected")); + } + if ( + _callId === "consult-call" && + result && + typeof result === "object" && + "status" in result && + result.status === "working" + ) { + return new Promise((resolve) => { + resolveWorkingSubmission = resolve; + }); + } + if ( + result && + typeof result === "object" && + "text" in result && + result.text === "The basement lights are on." + ) { + return new Promise((resolve) => { + resolveFinalSubmissions.push(resolve); + }); + } + return undefined; + }, + ); const bridge = makeBridge({ supportsToolResultContinuation: true, submitToolResult, @@ -1048,33 +1086,35 @@ describe("RealtimeCallHandler path routing", () => { return bridge; }, ); - const getCallByProviderCallId = vi.fn( - (): CallRecord => ({ - callId: "call-1", - providerCallId: "CA-tool", - provider: "twilio", - direction: "inbound", - state: "ringing", - from: "+15550001234", - to: "+15550009999", - startedAt: Date.now(), - transcript: [], - processedEventIds: [], - metadata: {}, - }), - ); + const call: CallRecord = { + callId: "call-1", + providerCallId: "CA-tool", + provider: "twilio", + direction: "inbound", + state: "ringing", + from: "+15550001234", + to: "+15550009999", + startedAt: Date.now(), + transcript: [], + processedEventIds: [], + metadata: {}, + }; + const getCallByProviderCallId = vi.fn((): CallRecord => call); const handler = makeHandler(undefined, { manager: { getCallByProviderCallId, }, realtimeProvider: makeRealtimeProvider(createBridge), }); - handler.registerToolHandler("openclaw_agent_consult", (_args, _callId, context) => { - receivedPartialTranscript = context.partialUserTranscript; - return new Promise((resolve) => { - resolveConsult = resolve; - }); - }); + const consultHandler = vi.fn( + (_args: unknown, _callId: string, context: { partialUserTranscript?: string }) => { + receivedPartialTranscript = context.partialUserTranscript; + return new Promise((resolve) => { + resolveConsult = resolve; + }); + }, + ); + handler.registerToolHandler("openclaw_agent_consult", consultHandler); handler.registerToolHandler("custom_lookup", async () => ({ ok: true })); const server = await startRealtimeServer(handler); @@ -1099,6 +1139,14 @@ describe("RealtimeCallHandler path routing", () => { name: "openclaw_agent_consult", args: { question: "Are the basement lights on?" }, }); + callbacks?.onToolCall?.({ + itemId: "item-2", + callId: "consult-call-2", + name: "openclaw_agent_consult", + args: { question: "Are the basement lights on?" }, + }); + expect(receivedPartialTranscript).toBeUndefined(); + resolveWorkingSubmission?.(); await vi.advanceTimersByTimeAsync(350); await waitForRealtimeTest(() => { expect(receivedPartialTranscript).toBe("Are the basement"); @@ -1117,19 +1165,35 @@ describe("RealtimeCallHandler path routing", () => { expect(typeof payload?.message).toBe("string"); expect(workingCall[2]).toEqual({ willContinue: true }); }); - expect(submitToolResult).toHaveBeenCalledTimes(1); + expect( + submitToolResult.mock.calls.filter( + ([, result]) => + result && + typeof result === "object" && + "status" in result && + result.status === "working", + ), + ).toHaveLength(2); resolveConsult?.({ text: "The basement lights are on." }); await waitForRealtimeTest(() => { expect(submitToolResult).toHaveBeenLastCalledWith( - "consult-call", + "consult-call-2", { text: "The basement lights are on.", }, undefined, ); }); + expect(recentTalkEvents(call).some((event) => event.type === "tool.result")).toBe(false); + for (const resolve of resolveFinalSubmissions) { + resolve(); + } + await waitForRealtimeTest(() => { + expect(recentTalkEvents(call).some((event) => event.type === "tool.result")).toBe(true); + }); + expect(consultHandler).toHaveBeenCalledTimes(1); submitToolResult.mockClear(); callbacks?.onToolCall?.({ @@ -1147,6 +1211,25 @@ describe("RealtimeCallHandler path routing", () => { ); expect(customCallResults).toHaveLength(1); expect(customCallResults[0]?.[2]).toBeUndefined(); + + submitToolResult.mockClear(); + rejectWorkingSubmission = true; + callbacks?.onToolCall?.({ + itemId: "item-rejected", + callId: "consult-rejected", + name: "openclaw_agent_consult", + args: { question: "Do not run this twice" }, + }); + await waitForRealtimeTest(() => { + expect(submitToolResult).toHaveBeenCalledTimes(1); + }); + await vi.advanceTimersByTimeAsync(0); + expect(consultHandler).toHaveBeenCalledTimes(1); + expect(submitToolResult).toHaveBeenCalledWith( + "consult-rejected", + expect.objectContaining({ status: "working" }), + { willContinue: true }, + ); } finally { vi.useRealTimers(); if (ws.readyState !== WebSocket.CLOSED && ws.readyState !== WebSocket.CLOSING) { @@ -1158,6 +1241,97 @@ describe("RealtimeCallHandler path routing", () => { } }); + it("terminally satisfies a late native call for a cancelled forced consult", async () => { + let callbacks: + | { + onToolCall?: (event: { + itemId: string; + callId: string; + name: string; + args: unknown; + }) => void; + } + | undefined; + const submitToolResult = vi.fn(); + const createBridge = vi.fn( + (request: Parameters[0]) => { + callbacks = request; + return makeBridge({ submitToolResult }); + }, + ); + const call: CallRecord = { + callId: "call-1", + providerCallId: "CA-cancelled-consult", + provider: "twilio", + direction: "inbound", + state: "ringing", + from: "+15550001234", + to: "+15550009999", + startedAt: Date.now(), + transcript: [], + processedEventIds: [], + metadata: {}, + }; + const handler = makeHandler(undefined, { + manager: { getCallByProviderCallId: vi.fn(() => call) }, + realtimeProvider: makeRealtimeProvider(createBridge), + }); + const consult = vi.fn(async () => ({ text: "should not run" })); + handler.registerToolHandler("openclaw_agent_consult", consult); + const coordinator = ( + handler as unknown as { + forcedConsultCoordinator(callId: string): RealtimeVoiceForcedConsultCoordinator; + } + ).forcedConsultCoordinator(call.callId); + const cancelled = coordinator.prepare("cancelled question"); + if (!cancelled) { + throw new Error("expected forced consult handle"); + } + coordinator.markStarted(cancelled); + coordinator.markCancelled(cancelled); + const server = await startRealtimeServer(handler); + + try { + const ws = await connectWs(server.url); + try { + ws.send( + JSON.stringify({ + event: "start", + start: { streamSid: "MZ-cancelled-consult", callSid: call.providerCallId }, + }), + ); + await waitForRealtimeTest(() => { + expect(createBridge).toHaveBeenCalled(); + }); + + callbacks?.onToolCall?.({ + itemId: "item-cancelled", + callId: "native-cancelled", + name: "openclaw_agent_consult", + args: { question: "cancelled question" }, + }); + + await waitForRealtimeTest(() => { + expect(submitToolResult).toHaveBeenCalledWith( + "native-cancelled", + { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + undefined, + ); + }); + expect(consult).not.toHaveBeenCalled(); + } finally { + if (ws.readyState !== WebSocket.CLOSED && ws.readyState !== WebSocket.CLOSING) { + ws.close(); + } + } + } finally { + await server.close(); + } + }); + it("forces an agent consult from final user transcript when consult policy is always", async () => { let callbacks: | { diff --git a/extensions/voice-call/src/webhook/realtime-handler.ts b/extensions/voice-call/src/webhook/realtime-handler.ts index 03a933cb3804..cef5b87ea77a 100644 --- a/extensions/voice-call/src/webhook/realtime-handler.ts +++ b/extensions/voice-call/src/webhook/realtime-handler.ts @@ -849,7 +849,7 @@ export class RealtimeCallHandler { console.log( `[voice-call] realtime tool call received callId=${callId} providerCallId=${callSid} tool=${toolEvent.name}`, ); - void this.executeToolCall( + return this.executeToolCall( sessionLocal, callId, toolEvent.callId || toolEvent.itemId, @@ -1347,28 +1347,28 @@ export class RealtimeCallHandler { final: true, }); }; - const submitFinalToolResult = (result: unknown): void => { - bridge.submitToolResult(bridgeCallId, result); + const submitFinalToolResult = async (result: unknown): Promise => { + await bridge.submitToolResult(bridgeCallId, result); emitFinalToolEvent(result); }; - const submitWorkingResponse = () => { + const submitWorkingResponse = async (): Promise => { if ( handler && name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME && bridge.bridge.supportsToolResultContinuation && !this.config.fastContext.enabled ) { + await bridge.submitToolResult( + bridgeCallId, + buildRealtimeVoiceAgentConsultWorkingResponse("caller"), + { willContinue: true }, + ); emitTalkEvent?.({ type: "tool.progress", turnId, callId: bridgeCallId, payload: { name, status: "working" }, }); - bridge.submitToolResult( - bridgeCallId, - buildRealtimeVoiceAgentConsultWorkingResponse("caller"), - { willContinue: true }, - ); } }; if (name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { @@ -1381,9 +1381,19 @@ export class RealtimeCallHandler { } } const forcedConsult = this.forcedConsultsByCallId.get(callId); + if (forcedMatch.kind === "already_delivered" && coordinator.isCancelled(forcedMatch.handle)) { + if (forcedConsult) { + forcedConsult.sendSpeechPrompt = false; + } + await submitFinalToolResult({ + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }); + return; + } if (forcedConsult) { if (forcedConsult.completedAt || forcedMatch.kind === "already_delivered") { - submitFinalToolResult({ + await submitFinalToolResult({ status: "already_delivered", message: "OpenClaw already delivered this consult result internally. Do not repeat it.", }); @@ -1393,7 +1403,7 @@ export class RealtimeCallHandler { const result = await forcedConsult.promise.catch((error: unknown) => ({ error: formatErrorMessage(error), })); - submitFinalToolResult(result); + await submitFinalToolResult(result); return; } @@ -1402,32 +1412,36 @@ export class RealtimeCallHandler { console.log( `[voice-call] realtime tool call sharing in-flight agent consult callId=${callId} ageMs=${Date.now() - existingNativeConsult.startedAt}`, ); - submitWorkingResponse(); - submitFinalToolResult(await existingNativeConsult.promise); + await submitWorkingResponse(); + await submitFinalToolResult(await existingNativeConsult.promise); return; } - submitWorkingResponse(); const state: NativeConsultState = { startedAt, promise: Promise.resolve(), }; - state.promise = (async () => { - await this.waitForConsultTranscriptSettle(callId, startedAt); - const context = { - partialUserTranscript: this.resolveUserTranscriptContext(callId), - }; - state.partialUserTranscript = context.partialUserTranscript; - const handlerArgs = withFallbackConsultQuestion(args, context.partialUserTranscript); - console.log( - `[voice-call] realtime tool call executing callId=${callId} tool=${name} hasHandler=${Boolean(handler)}`, - ); - return !handler - ? { error: `Tool "${name}" not available` } - : await handler(handlerArgs, callId, context); - })().catch((error: unknown) => ({ - error: formatErrorMessage(error), - })); + const workingSubmission = submitWorkingResponse(); + state.promise = workingSubmission.then(async () => { + try { + await this.waitForConsultTranscriptSettle(callId, startedAt); + const context = { + partialUserTranscript: this.resolveUserTranscriptContext(callId), + }; + state.partialUserTranscript = context.partialUserTranscript; + const handlerArgs = withFallbackConsultQuestion(args, context.partialUserTranscript); + console.log( + `[voice-call] realtime tool call executing callId=${callId} tool=${name} hasHandler=${Boolean(handler)}`, + ); + return !handler + ? { error: `Tool "${name}" not available` } + : await handler(handlerArgs, callId, context); + } catch (error) { + return { + error: formatErrorMessage(error), + }; + } + }); this.nativeConsultsInFlightByCallId.set(callId, state); try { const result = await state.promise; @@ -1442,7 +1456,7 @@ export class RealtimeCallHandler { console.log( `[voice-call] realtime tool call completed callId=${callId} tool=${name} status=${status} elapsedMs=${Date.now() - startedAt}${error ? ` error=${error}` : ""}`, ); - submitFinalToolResult(result); + await submitFinalToolResult(result); if (status === "ok") { this.consumePartialUserTranscript(callId, state.partialUserTranscript); } @@ -1479,7 +1493,7 @@ export class RealtimeCallHandler { console.log( `[voice-call] realtime tool call completed callId=${callId} tool=${name} status=${status} elapsedMs=${Date.now() - startedAt}${error ? ` error=${error}` : ""}`, ); - submitFinalToolResult(result); + await submitFinalToolResult(result); if (name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME && status === "ok") { this.consumePartialUserTranscript(callId, context.partialUserTranscript); } diff --git a/src/gateway/server-methods/talk-session.ts b/src/gateway/server-methods/talk-session.ts index ffffaf087e97..132023368f94 100644 --- a/src/gateway/server-methods/talk-session.ts +++ b/src/gateway/server-methods/talk-session.ts @@ -623,7 +623,7 @@ export const talkSessionHandlers: GatewayRequestHandlers = { return; } const connId = requireUnifiedTalkSessionConn(session, client?.connId); - submitTalkRealtimeRelayToolResult({ + await submitTalkRealtimeRelayToolResult({ relaySessionId: session.relaySessionId, connId, callId: params.callId, diff --git a/src/gateway/server-methods/talk.test.ts b/src/gateway/server-methods/talk.test.ts index 136ae14e7f58..67d4677b8fa6 100644 --- a/src/gateway/server-methods/talk.test.ts +++ b/src/gateway/server-methods/talk.test.ts @@ -1521,8 +1521,14 @@ describe("talk.session unified handlers", () => { reason: "barge-in", }); + let acceptToolResult!: () => void; + mocks.submitTalkRealtimeRelayToolResult.mockReturnValueOnce( + new Promise((resolve) => { + acceptToolResult = resolve; + }), + ); const toolRespond = vi.fn(); - await talkHandlers["talk.session.submitToolResult"]({ + const toolRequest = talkHandlers["talk.session.submitToolResult"]({ req: { type: "req", id: "4", method: "talk.session.submitToolResult" }, params: { sessionId: "relay-unified-1", @@ -1535,6 +1541,9 @@ describe("talk.session unified handlers", () => { respond: toolRespond as never, context: {} as never, }); + expect(toolRespond).not.toHaveBeenCalled(); + acceptToolResult(); + await toolRequest; expect(mocks.submitTalkRealtimeRelayToolResult).toHaveBeenCalledWith({ relaySessionId: "relay-unified-1", connId: "conn-1", @@ -1542,6 +1551,28 @@ describe("talk.session unified handlers", () => { result: { status: "working" }, options: { suppressResponse: true, willContinue: true }, }); + expectRespondOk(toolRespond, { ok: true }); + + mocks.submitTalkRealtimeRelayToolResult.mockRejectedValueOnce( + new Error("provider rejected tool result"), + ); + const rejectedToolRespond = vi.fn(); + await talkHandlers["talk.session.submitToolResult"]({ + req: { type: "req", id: "4-rejected", method: "talk.session.submitToolResult" }, + params: { + sessionId: "relay-unified-1", + callId: "call-rejected", + result: { ok: true }, + }, + client: { connId: "conn-1" } as never, + isWebchatConnect: () => false, + respond: rejectedToolRespond as never, + context: {} as never, + }); + expectRespondError(rejectedToolRespond, { + code: ErrorCodes.UNAVAILABLE, + message: "Error: provider rejected tool result", + }); const steerRespond = vi.fn(); await talkHandlers["talk.session.steer"]({ diff --git a/src/gateway/talk-realtime-relay.test.ts b/src/gateway/talk-realtime-relay.test.ts index d9c5755b66ea..d0e615ee7454 100644 --- a/src/gateway/talk-realtime-relay.test.ts +++ b/src/gateway/talk-realtime-relay.test.ts @@ -7,7 +7,10 @@ import { testing as embeddedRunTesting, } from "../agents/embedded-agent-runner/runs.js"; import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js"; -import type { RealtimeVoiceBridgeCreateRequest } from "../talk/provider-types.js"; +import type { + RealtimeVoiceBridge, + RealtimeVoiceBridgeCreateRequest, +} from "../talk/provider-types.js"; import { cancelTalkRealtimeRelayTurn, clearTalkRealtimeRelaySessionsForTest, @@ -44,6 +47,89 @@ describe("talk realtime gateway relay", () => { }; } + function createDeferredVoid(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((accept) => { + resolve = accept; + }); + return { promise, resolve }; + } + + async function createSuppressionUnsupportedForcedConsultFixture( + nativeCallIds: string[], + options: { + firstSubmission?: Promise; + supportsToolResultContinuation?: boolean; + } = {}, + ) { + vi.useFakeTimers(); + let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined; + const submitToolResult = vi.fn(); + if (options.firstSubmission) { + submitToolResult.mockReturnValueOnce(options.firstSubmission); + } + const sendUserMessage = vi.fn(); + const bridge = { + supportsToolResultContinuation: options.supportsToolResultContinuation ?? false, + supportsToolResultSuppression: false, + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + sendUserMessage, + handleBargeIn: vi.fn(), + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }; + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: (request) => { + bridgeRequest = request; + return bridge; + }, + }; + const events: Array<{ event: string; payload: unknown; connIds: string[] }> = []; + const session = createTalkRealtimeRelaySession({ + context: { + broadcastToConnIds: (event: string, payload: unknown, connIds: ReadonlySet) => { + events.push({ event, payload, connIds: [...connIds] }); + }, + } as never, + connId: "conn-1", + provider, + providerConfig: {}, + instructions: "be brief", + tools: [], + forceAgentConsultOnFinalTranscript: true, + }); + await Promise.resolve(); + bridgeRequest?.onTranscript?.("user", "Can you check this?", true); + await vi.advanceTimersByTimeAsync(250); + const forcedToolCall = findEventPayload( + events, + (payload) => payload.type === "toolCall" && payload.forced === true, + ); + for (const callId of nativeCallIds) { + bridgeRequest?.onToolCall?.({ + itemId: `item-${callId}`, + callId, + name: "openclaw_agent_consult", + args: { question: "Can you check this?" }, + }); + } + return { + bridgeRequest, + callId: String(forcedToolCall.callId), + events, + sendUserMessage, + session, + submitToolResult, + }; + } + it("rejects session creation when relay expiry would exceed Date range", () => { vi.useFakeTimers(); vi.setSystemTime(new Date(8_640_000_000_000_000)); @@ -62,6 +148,7 @@ describe("talk realtime gateway relay", () => { function createAbortableRelayRunFixture(provider = createIdleRelayProvider()) { const abortController = new AbortController(); + const broadcastToConnIds = vi.fn(); const broadcast = vi.fn(); const nodeSendToSession = vi.fn(); const removeChatRun = vi.fn(() => ({ sessionKey: "main", clientRunId: "run-1" })); @@ -85,7 +172,7 @@ describe("talk realtime gateway relay", () => { ], ]); const context = { - broadcastToConnIds: vi.fn(), + broadcastToConnIds, broadcast, nodeSendToSession, chatAbortControllers: new Map([ @@ -143,6 +230,7 @@ describe("talk realtime gateway relay", () => { removeChatRun, agentDeltaSentAt, bufferedAgentEvents, + broadcastToConnIds, session, }; } @@ -363,20 +451,20 @@ describe("talk realtime gateway relay", () => { audioBase64: Buffer.from("audio-in").toString("base64"), timestamp: 123, }); - submitTalkRealtimeRelayToolResult({ + void submitTalkRealtimeRelayToolResult({ relaySessionId: session.relaySessionId, connId: "conn-1", callId: "call-1", result: { status: "working" }, options: { willContinue: true }, }); - submitTalkRealtimeRelayToolResult({ + void submitTalkRealtimeRelayToolResult({ relaySessionId: session.relaySessionId, connId: "conn-1", callId: "call-1", result: { ok: true }, }); - submitTalkRealtimeRelayToolResult({ + void submitTalkRealtimeRelayToolResult({ relaySessionId: session.relaySessionId, connId: "conn-1", callId: "call-2", @@ -859,7 +947,7 @@ describe("talk realtime gateway relay", () => { }); const callId = String(forcedToolCall.callId); - submitTalkRealtimeRelayToolResult({ + void submitTalkRealtimeRelayToolResult({ relaySessionId: session.relaySessionId, connId: "conn-1", callId, @@ -887,12 +975,19 @@ describe("talk realtime gateway relay", () => { { willContinue: true }, ); - submitTalkRealtimeRelayToolResult({ + const forcedAcceptance = createDeferredVoid(); + bridge.submitToolResult.mockImplementationOnce(() => forcedAcceptance.promise); + const forcedSubmission = submitTalkRealtimeRelayToolResult({ relaySessionId: session.relaySessionId, connId: "conn-1", callId, result: { result: "Here is the checked answer." }, }); + expect(bridge.sendUserMessage).not.toHaveBeenCalledWith( + expect.stringContaining("Here is the checked answer."), + ); + forcedAcceptance.resolve(); + await forcedSubmission; expect(bridge.submitToolResult).toHaveBeenLastCalledWith( "native-call", { @@ -960,6 +1055,265 @@ describe("talk realtime gateway relay", () => { stopTalkRealtimeRelaySession({ relaySessionId: session.relaySessionId, connId: "conn-1" }); }); + it("uses the actual forced result when one native call cannot suppress responses", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture(["native-call"]); + const accepted = createDeferredVoid(); + fixture.submitToolResult.mockReturnValueOnce(accepted.promise); + + const submission = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { result: "checked" }, + }); + const duplicate = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { answer: "duplicate" }, + }); + expect(duplicate).toBe(submission); + expect(fixture.submitToolResult).toHaveBeenCalledTimes(1); + expect(fixture.submitToolResult).toHaveBeenCalledWith( + "native-call", + { result: "checked" }, + undefined, + ); + expect(fixture.sendUserMessage).not.toHaveBeenCalled(); + + accepted.resolve(); + await submission; + expect(fixture.sendUserMessage).not.toHaveBeenCalled(); + expect( + fixture.events.filter( + (entry) => + (entry.payload as { type?: string; callId?: string }).type === "toolResult" && + (entry.payload as { callId?: string }).callId === fixture.callId, + ), + ).toHaveLength(1); + }); + + it("classifies forced interim and terminal results only from willContinue", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture(["native-call"]); + + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { phase: "checking" }, + options: { willContinue: true }, + }); + expect(fixture.submitToolResult).not.toHaveBeenCalled(); + + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { status: "working" }, + }); + expect(fixture.submitToolResult).toHaveBeenCalledWith( + "native-call", + { status: "working" }, + undefined, + ); + const finalStates = fixture.events + .filter( + (entry) => + (entry.payload as { type?: string; callId?: string }).type === "toolResult" && + (entry.payload as { callId?: string }).callId === fixture.callId, + ) + .map((entry) => (entry.payload as { talkEvent?: { final?: boolean } }).talkEvent?.final); + expect(finalStates).toEqual([false, true]); + }); + + it("waits for every native forced result when response suppression is unsupported", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture([ + "native-1", + "native-2", + ]); + const first = createDeferredVoid(); + const second = createDeferredVoid(); + fixture.submitToolResult.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise); + const submission = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { result: "checked" }, + }); + + first.resolve(); + await Promise.resolve(); + expect( + fixture.events.some( + (entry) => + (entry.payload as { type?: string; callId?: string }).type === "toolResult" && + (entry.payload as { callId?: string }).callId === fixture.callId, + ), + ).toBe(false); + second.resolve(); + await submission; + expect(fixture.submitToolResult).toHaveBeenCalledTimes(2); + }); + + it("drains native calls that join while a forced terminal result is pending", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture(["native-1"]); + const first = createDeferredVoid(); + fixture.submitToolResult.mockReturnValueOnce(first.promise).mockReturnValueOnce(undefined); + const submission = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { result: "checked" }, + }); + + fixture.bridgeRequest?.onToolCall?.({ + itemId: "late-item", + callId: "native-2", + name: "openclaw_agent_consult", + args: { question: "Can you check this?" }, + }); + expect(fixture.submitToolResult.mock.calls.map((call) => call[0])).toEqual(["native-1"]); + + first.resolve(); + await submission; + expect(fixture.submitToolResult.mock.calls.map((call) => call[0])).toEqual([ + "native-1", + "native-2", + ]); + expect(fixture.sendUserMessage).not.toHaveBeenCalled(); + expect( + fixture.events.filter( + (entry) => + (entry.payload as { type?: string }).type === "toolResult" && + (entry.payload as { callId?: string }).callId === "native-2", + ), + ).toHaveLength(0); + }); + + it("keeps a suppression-unsupported forced result retryable after rejection", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture(["native-call"]); + fixture.submitToolResult.mockRejectedValueOnce(new Error("native result rejected")); + const rejected = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { answer: "checked" }, + }); + await expect(rejected).rejects.toThrow("native result rejected"); + expect(fixture.sendUserMessage).not.toHaveBeenCalled(); + + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { answer: "checked" }, + }); + expect(fixture.submitToolResult).toHaveBeenCalledTimes(2); + }); + + it("retries only rejected native forced results after partial acceptance", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture([ + "native-1", + "native-2", + ]); + fixture.submitToolResult + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error("second native rejected")); + const firstAttempt = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { answer: "checked" }, + }); + await expect(firstAttempt).rejects.toThrow("second native rejected"); + + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { answer: "checked" }, + }); + expect(fixture.submitToolResult.mock.calls.map((call) => call[0])).toEqual([ + "native-1", + "native-2", + "native-2", + ]); + }); + + it("uses the speech path without native calls even when suppression is unsupported", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture([]); + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { result: "checked" }, + }); + + expect(fixture.submitToolResult).not.toHaveBeenCalled(); + expect(fixture.sendUserMessage).toHaveBeenCalledWith(expect.stringContaining("checked")); + }); + + it("satisfies late native forced calls without suppression or duplicate speech", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture([]); + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { result: "checked" }, + }); + expect(fixture.sendUserMessage).toHaveBeenCalledTimes(1); + + fixture.bridgeRequest?.onToolCall?.({ + itemId: "late-item", + callId: "late-call", + name: "openclaw_agent_consult", + args: { question: "Can you check this?" }, + }); + expect(fixture.submitToolResult).toHaveBeenCalledWith( + "late-call", + { + status: "already_delivered", + message: "OpenClaw already delivered this consult result internally. Do not repeat it.", + }, + undefined, + ); + expect(fixture.sendUserMessage).toHaveBeenCalledTimes(1); + }); + + it("rejects direct suppressed results when the provider does not support them", () => { + const submitToolResult = vi.fn(); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + submitToolResult, + supportsToolResultSuppression: false, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }); + const session = createTalkRealtimeRelaySession({ + context: { broadcastToConnIds: vi.fn() } as never, + connId: "conn-1", + provider, + providerConfig: {}, + instructions: "brief", + tools: [], + }); + + expect(() => + submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { ok: true }, + options: { suppressResponse: true }, + }), + ).toThrow("Realtime provider does not support suppressed tool results"); + expect(submitToolResult).not.toHaveBeenCalled(); + }); + it("does not force a duplicate consult after native consult or cancellation", async () => { vi.useFakeTimers(); @@ -1211,10 +1565,60 @@ describe("talk realtime gateway relay", () => { expectNodeAbortPayload(nodeSendToSession); }); + it("terminally satisfies a late normal result after turn cancellation without a new turn", async () => { + const submitToolResult = vi.fn(); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }); + const { broadcastToConnIds, session } = createAbortableRelayRunFixture(provider); + cancelTalkRealtimeRelayTurn({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + const startedTurns = () => + broadcastToConnIds.mock.calls.filter( + (call) => (call[1] as { talkEvent?: { type?: string } }).talkEvent?.type === "turn.started", + ).length; + const beforeLateResult = startedTurns(); + + await submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { error: "aborted" }, + }); + + expect(startedTurns()).toBe(beforeLateResult); + expect(submitToolResult).toHaveBeenCalledWith( + "call-1", + { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + { suppressResponse: true }, + ); + expect( + broadcastToConnIds.mock.calls.filter( + (call) => + (call[1] as { type?: string; callId?: string }).type === "toolResult" && + (call[1] as { callId?: string }).callId === "call-1", + ), + ).toHaveLength(0); + }); + it("clears linked agent consult runs after the final tool result", () => { const { abortController, broadcast, session } = createAbortableRelayRunFixture(); - submitTalkRealtimeRelayToolResult({ + void submitTalkRealtimeRelayToolResult({ relaySessionId: session.relaySessionId, connId: "conn-1", callId: "call-1", @@ -1233,6 +1637,553 @@ describe("talk realtime gateway relay", () => { ); }); + it("serializes a final consult result behind async working acceptance", async () => { + let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined; + const working = createDeferredVoid(); + const submitToolResult = vi + .fn() + .mockReturnValueOnce(working.promise) + .mockReturnValueOnce(undefined); + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: (request) => { + bridgeRequest = request; + return { + supportsToolResultContinuation: true, + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }; + }, + }; + const events: Array<{ payload: Record }> = []; + const session = createTalkRealtimeRelaySession({ + context: { + broadcastToConnIds: (_event: string, payload: unknown) => { + events.push({ payload: payload as Record }); + }, + } as never, + connId: "conn-1", + provider, + providerConfig: {}, + instructions: "brief", + tools: [], + }); + bridgeRequest?.onToolCall?.({ + itemId: "item-1", + callId: "call-1", + name: "openclaw_agent_consult", + args: { question: "check" }, + }); + const finalSubmission = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { answer: "done" }, + }); + + expect(submitToolResult).toHaveBeenCalledTimes(1); + expect( + events.map((entry) => (entry.payload.talkEvent as { type?: string } | undefined)?.type), + ).toContain("tool.call"); + expect( + events.map((entry) => (entry.payload.talkEvent as { type?: string } | undefined)?.type), + ).not.toContain("tool.progress"); + + working.resolve(); + await finalSubmission; + expect(submitToolResult.mock.calls.map((call) => call[1])).toEqual([ + expect.objectContaining({ status: "working" }), + { answer: "done" }, + ]); + const talkTypes = events + .map((entry) => (entry.payload.talkEvent as { type?: string } | undefined)?.type) + .filter(Boolean); + expect(talkTypes.indexOf("tool.call")).toBeLessThan(talkTypes.indexOf("tool.progress")); + expect(talkTypes.indexOf("tool.progress")).toBeLessThan(talkTypes.indexOf("tool.result")); + }); + + it("serializes concurrent client interims and a final result in submission order", async () => { + const firstAccepted = createDeferredVoid(); + const secondAccepted = createDeferredVoid(); + const submitToolResult = vi + .fn() + .mockReturnValueOnce(firstAccepted.promise) + .mockReturnValueOnce(secondAccepted.promise) + .mockReturnValueOnce(undefined); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }); + const session = createTalkRealtimeRelaySession({ + context: { broadcastToConnIds: vi.fn() } as never, + connId: "conn-1", + provider, + providerConfig: {}, + instructions: "brief", + tools: [], + }); + + const firstInterim = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { phase: "first" }, + options: { willContinue: true }, + }); + const secondInterim = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { phase: "second" }, + options: { willContinue: true }, + }); + const final = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { answer: "done" }, + }); + expect(submitToolResult).toHaveBeenCalledTimes(1); + + firstAccepted.resolve(); + await vi.waitFor(() => expect(submitToolResult).toHaveBeenCalledTimes(2)); + expect(submitToolResult).not.toHaveBeenCalledWith("call-1", { answer: "done" }, undefined); + + secondAccepted.resolve(); + await Promise.all([firstInterim, secondInterim, final]); + expect(submitToolResult.mock.calls.map((call) => call[1])).toEqual([ + { phase: "first" }, + { phase: "second" }, + { answer: "done" }, + ]); + }); + + it("submits an ordinary final after an interim rejection", async () => { + let rejectInterim: ((error: Error) => void) | undefined; + const rejectedInterim = new Promise((_resolve, reject) => { + rejectInterim = reject; + }); + const submitToolResult = vi + .fn() + .mockReturnValueOnce(rejectedInterim) + .mockReturnValueOnce(undefined); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }); + const session = createTalkRealtimeRelaySession({ + context: { broadcastToConnIds: vi.fn() } as never, + connId: "conn-1", + provider, + providerConfig: {}, + instructions: "brief", + tools: [], + }); + const interim = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { phase: "checking" }, + options: { willContinue: true }, + }); + const final = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { answer: "done" }, + }); + + rejectInterim?.(new Error("interim rejected")); + await expect(interim).rejects.toThrow("interim rejected"); + await final; + expect(submitToolResult.mock.calls.map((call) => call[1])).toEqual([ + { phase: "checking" }, + { answer: "done" }, + ]); + }); + + it("supersedes queued interims and a stale final with canonical cancellation", async () => { + const interimAccepted = createDeferredVoid(); + const submitToolResult = vi + .fn() + .mockReturnValueOnce(interimAccepted.promise) + .mockReturnValueOnce(undefined); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }); + const { broadcastToConnIds, session } = createAbortableRelayRunFixture(provider); + const firstInterim = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { phase: "first" }, + options: { willContinue: true }, + }); + const secondInterim = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { phase: "second" }, + options: { willContinue: true }, + }); + const final = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { answer: "stale" }, + }); + cancelTalkRealtimeRelayTurn({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + const cancellation = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { error: "aborted" }, + }); + + interimAccepted.resolve(); + await Promise.all([firstInterim, secondInterim, final, cancellation]); + expect(submitToolResult.mock.calls.map((call) => call[1])).toEqual([ + { phase: "first" }, + { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + ]); + expect(submitToolResult.mock.calls[1]?.[2]).toEqual({ suppressResponse: true }); + expect( + broadcastToConnIds.mock.calls.some( + (call) => + (call[1] as { type?: string; callId?: string }).type === "toolResult" && + (call[1] as { callId?: string }).callId === "call-1", + ), + ).toBe(false); + }); + + it("terminally cancels a final queued behind working acceptance without a second client result", async () => { + const workingAccepted = createDeferredVoid(); + const submitToolResult = vi + .fn() + .mockReturnValueOnce(workingAccepted.promise) + .mockReturnValueOnce(undefined); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult, + supportsToolResultSuppression: false, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }); + const { broadcastToConnIds, session } = createAbortableRelayRunFixture(provider); + const working = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { status: "working" }, + options: { willContinue: true }, + }); + const final = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { answer: "stale" }, + }); + + cancelTalkRealtimeRelayTurn({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + workingAccepted.resolve(); + await Promise.all([working, final]); + + expect(submitToolResult.mock.calls.map((call) => call[1])).toEqual([ + { status: "working" }, + { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + ]); + expect(submitToolResult.mock.calls[1]?.[2]).toBeUndefined(); + expect( + broadcastToConnIds.mock.calls.some( + (call) => + (call[1] as { type?: string; callId?: string }).type === "toolResult" && + (call[1] as { callId?: string }).callId === "call-1", + ), + ).toBe(false); + + await submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { error: "late abort" }, + }); + expect(submitToolResult).toHaveBeenCalledTimes(2); + }); + + it("supersedes a rejected in-flight final with canonical cancellation", async () => { + let rejectFinal: ((error: Error) => void) | undefined; + const rejectedFinal = new Promise((_resolve, reject) => { + rejectFinal = reject; + }); + const submitToolResult = vi + .fn() + .mockReturnValueOnce(rejectedFinal) + .mockReturnValueOnce(undefined); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }); + const { session } = createAbortableRelayRunFixture(provider); + const staleFinal = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { answer: "stale" }, + }); + cancelTalkRealtimeRelayTurn({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + const cancellation = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { error: "aborted" }, + }); + + rejectFinal?.(new Error("stale final rejected")); + await expect(staleFinal).rejects.toThrow("stale final rejected"); + await cancellation; + expect(submitToolResult.mock.calls.map((call) => call[1])).toEqual([ + { answer: "stale" }, + { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + ]); + }); + + it("waits for provider acceptance before broadcasting and clearing a final tool result", async () => { + const deferred = createDeferredVoid(); + const bridge = { + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult: vi.fn(() => deferred.promise), + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }; + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: () => bridge, + }; + const { abortController, broadcast, broadcastToConnIds, session } = + createAbortableRelayRunFixture(provider); + + const submission = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { ok: true }, + }); + if (!submission) { + throw new Error("Expected asynchronous provider submission"); + } + const duplicate = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { ok: "duplicate" }, + }); + const followUp = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { status: "working" }, + options: { willContinue: true }, + }); + + expect(duplicate).toBe(submission); + expect(followUp).toBe(submission); + expect(bridge.submitToolResult).toHaveBeenCalledTimes(1); + expect( + broadcastToConnIds.mock.calls.some( + (call) => (call[1] as { type?: string }).type === "toolResult", + ), + ).toBe(false); + + deferred.resolve(); + await submission; + + const toolResultEvents = () => + broadcastToConnIds.mock.calls.filter( + (call) => (call[1] as { type?: string }).type === "toolResult", + ); + expect(toolResultEvents()).toHaveLength(1); + expect( + submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { ok: "already completed" }, + }), + ).toBeUndefined(); + expect(bridge.submitToolResult).toHaveBeenCalledTimes(1); + expect(toolResultEvents()).toHaveLength(1); + cancelTalkRealtimeRelayTurn({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + expect(abortController.signal.aborted).toBe(false); + expect(broadcast).not.toHaveBeenCalledWith( + "chat", + expect.objectContaining({ runId: "run-1", state: "aborted" }), + ); + }); + + it("keeps linked run state and omits success events when provider submission rejects", async () => { + const bridge = { + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult: vi.fn(() => Promise.reject(new Error("provider rejected tool result"))), + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }; + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: () => bridge, + }; + const { abortController, broadcastToConnIds, session } = + createAbortableRelayRunFixture(provider); + + const submission = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { ok: true }, + }); + if (!submission) { + throw new Error("Expected asynchronous provider submission"); + } + await expect(submission).rejects.toThrow("provider rejected tool result"); + expect( + broadcastToConnIds.mock.calls.some( + (call) => (call[1] as { type?: string }).type === "toolResult", + ), + ).toBe(false); + + cancelTalkRealtimeRelayTurn({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + expect(abortController.signal.aborted).toBe(true); + }); + + it("allows a final tool result to retry after provider rejection", async () => { + let attempt = 0; + const bridge = { + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult: vi.fn(() => { + attempt += 1; + return attempt === 1 ? Promise.reject(new Error("temporary rejection")) : undefined; + }), + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }; + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: () => bridge, + }; + const { broadcastToConnIds, session } = createAbortableRelayRunFixture(provider); + + const rejected = submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { ok: false }, + }); + await expect(rejected).rejects.toThrow("temporary rejection"); + + expect( + submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { ok: true }, + }), + ).toBeUndefined(); + expect(bridge.submitToolResult).toHaveBeenCalledTimes(2); + expect( + broadcastToConnIds.mock.calls.filter( + (call) => (call[1] as { type?: string }).type === "toolResult", + ), + ).toHaveLength(1); + }); + it("returns structured relay steering status and emits Talk progress", async () => { const provider = createIdleRelayProvider(); const events: Array<{ event: string; payload: unknown; connIds: string[] }> = []; @@ -1286,78 +2237,211 @@ describe("talk realtime gateway relay", () => { }); }); - it("submits a final provider result when voice cancel aborts an active relay run", async () => { - const abortEmbeddedRun = vi.fn(); + it.each([ + { + supportsSuppression: undefined, + expectedOptions: { suppressResponse: true }, + expectedSuppress: false, + }, + { supportsSuppression: false, expectedOptions: undefined, expectedSuppress: true }, + ])( + "submits a final provider result when voice cancel aborts an active relay run ($supportsSuppression)", + async ({ supportsSuppression, expectedOptions, expectedSuppress }) => { + const abortEmbeddedRun = vi.fn(); + setActiveEmbeddedRun( + "embedded-session-1", + { + queueMessage: vi.fn(async () => undefined), + isStreaming: () => true, + isCompacting: () => false, + abort: abortEmbeddedRun, + }, + "main", + ); + const bridge = { + supportsToolResultSuppression: supportsSuppression, + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult: vi.fn(), + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }; + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: () => bridge, + }; + const { abortController, broadcast, session } = createAbortableRelayRunFixture(provider); + + const result = await steerTalkRealtimeRelayAgentRun({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + text: "cancel that", + mode: "cancel", + }); + cancelTalkRealtimeRelayTurn({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + + expect(result).toMatchObject({ + ok: true, + mode: "cancel", + suppress: expectedSuppress, + providerResult: { + status: "cancelled", + message: "Cancelled the active OpenClaw run.", + }, + }); + expect(abortEmbeddedRun).toHaveBeenCalledTimes(1); + expect(bridge.submitToolResult).toHaveBeenCalledWith( + "call-1", + { + status: "cancelled", + message: "Cancelled the active OpenClaw run.", + }, + expectedOptions, + ); + expect(abortController.signal.aborted).toBe(false); + expect(broadcast).not.toHaveBeenCalledWith( + "chat", + expect.objectContaining({ runId: "run-1", state: "aborted" }), + ); + + void submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { error: "aborted" }, + }); + expect(bridge.submitToolResult).toHaveBeenCalledTimes(1); + }, + ); + + it("finalizes accepted cancel calls independently when another provider result rejects", async () => { setActiveEmbeddedRun( "embedded-session-1", { queueMessage: vi.fn(async () => undefined), isStreaming: () => true, isCompacting: () => false, - abort: abortEmbeddedRun, + abort: vi.fn(), }, "main", ); - const bridge = { + const submitToolResult = vi + .fn() + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error("second cancel rejected")); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ connect: vi.fn(async () => undefined), sendAudio: vi.fn(), setMediaTimestamp: vi.fn(), handleBargeIn: vi.fn(), - submitToolResult: vi.fn(), + submitToolResult, acknowledgeMark: vi.fn(), close: vi.fn(), isConnected: vi.fn(() => true), - }; - const provider: RealtimeVoiceProviderPlugin = { - id: "relay-test", - label: "Relay Test", - isConfigured: () => true, - createBridge: () => bridge, - }; - const { abortController, broadcast, session } = createAbortableRelayRunFixture(provider); + }); + const { broadcastToConnIds, session } = createAbortableRelayRunFixture(provider); + registerTalkRealtimeRelayAgentRun({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + sessionKey: "main", + runId: "run-1", + callId: "call-2", + }); - const result = await steerTalkRealtimeRelayAgentRun({ + await expect( + steerTalkRealtimeRelayAgentRun({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + text: "cancel that", + mode: "cancel", + }), + ).rejects.toThrow("second cancel rejected"); + + expect( + broadcastToConnIds.mock.calls.filter( + (call) => + (call[1] as { type?: string; callId?: string }).type === "toolResult" && + (call[1] as { callId?: string }).callId === "call-1", + ), + ).toHaveLength(1); + expect( + submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { late: true }, + }), + ).toBeUndefined(); + expect(submitToolResult).toHaveBeenCalledTimes(2); + + await submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId: "call-2", + result: { retry: true }, + }); + expect(submitToolResult.mock.calls.map((call) => call[0])).toEqual([ + "call-1", + "call-2", + "call-2", + ]); + }); + + it("does not broadcast steering progress after the relay closes during provider acceptance", async () => { + setActiveEmbeddedRun( + "embedded-session-1", + { + queueMessage: vi.fn(async () => undefined), + isStreaming: () => true, + isCompacting: () => false, + abort: vi.fn(), + }, + "main", + ); + const accepted = createDeferredVoid(); + const submitToolResult = vi.fn(() => accepted.promise); + const provider = createIdleRelayProvider(); + provider.createBridge = () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }); + const { broadcastToConnIds, session } = createAbortableRelayRunFixture(provider); + + const steering = steerTalkRealtimeRelayAgentRun({ relaySessionId: session.relaySessionId, connId: "conn-1", text: "cancel that", mode: "cancel", }); - cancelTalkRealtimeRelayTurn({ + await vi.waitFor(() => expect(submitToolResult).toHaveBeenCalledTimes(1)); + stopTalkRealtimeRelaySession({ relaySessionId: session.relaySessionId, connId: "conn-1", - reason: "barge-in", }); + accepted.resolve(); + await steering; - expect(result).toMatchObject({ - ok: true, - mode: "cancel", - providerResult: { - status: "cancelled", - message: "Cancelled the active OpenClaw run.", - }, - }); - expect(abortEmbeddedRun).toHaveBeenCalledTimes(1); - expect(bridge.submitToolResult).toHaveBeenCalledWith( - "call-1", - { - status: "cancelled", - message: "Cancelled the active OpenClaw run.", - }, - { suppressResponse: true }, - ); - expect(abortController.signal.aborted).toBe(false); - expect(broadcast).not.toHaveBeenCalledWith( - "chat", - expect.objectContaining({ runId: "run-1", state: "aborted" }), - ); - - submitTalkRealtimeRelayToolResult({ - relaySessionId: session.relaySessionId, - connId: "conn-1", - callId: "call-1", - result: { error: "aborted" }, - }); - expect(bridge.submitToolResult).toHaveBeenCalledTimes(1); + expect( + broadcastToConnIds.mock.calls.filter( + (call) => (call[1] as { type?: string }).type === "toolProgress", + ), + ).toHaveLength(0); }); it("does not submit cancel results for synthetic forced-consult calls", async () => { @@ -1459,6 +2543,254 @@ describe("talk realtime gateway relay", () => { }); }); + it("terminally cancels late forced working even when willContinue is set", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture(["native-1"]); + cancelTalkRealtimeRelayTurn({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + const startedTurns = () => + fixture.events.filter( + (entry) => + (entry.payload as { talkEvent?: { type?: string } }).talkEvent?.type === "turn.started", + ).length; + const beforeLateCalls = startedTurns(); + + fixture.bridgeRequest?.onToolCall?.({ + itemId: "late-native-item", + callId: "native-2", + name: "openclaw_agent_consult", + args: { question: "Can you check this?" }, + }); + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { status: "working" }, + options: { willContinue: true }, + }); + + expect(startedTurns()).toBe(beforeLateCalls); + expect(fixture.submitToolResult.mock.calls.map((call) => call[0])).toEqual([ + "native-2", + "native-1", + ]); + for (const call of fixture.submitToolResult.mock.calls) { + expect(call[1]).toEqual({ + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }); + expect(call[2]).toBeUndefined(); + } + expect( + fixture.events.filter( + (entry) => + (entry.payload as { type?: string; callId?: string }).type === "toolCall" && + (entry.payload as { callId?: string }).callId === "native-2", + ), + ).toHaveLength(0); + const terminal = findEventPayload( + fixture.events, + (payload) => + payload.type === "toolResult" && + payload.callId === fixture.callId && + (payload.talkEvent as { final?: boolean } | undefined)?.final === true, + ); + expectRecordFields((terminal.talkEvent as Record).payload, { + result: { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + forced: true, + }); + }); + + it("terminally cancels a forced final queued behind native working acceptance", async () => { + const workingAccepted = createDeferredVoid(); + const fixture = await createSuppressionUnsupportedForcedConsultFixture(["native-call"], { + firstSubmission: workingAccepted.promise, + supportsToolResultContinuation: true, + }); + const final = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { answer: "stale" }, + }); + + cancelTalkRealtimeRelayTurn({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + workingAccepted.resolve(); + await final; + + expect(fixture.submitToolResult.mock.calls.map((call) => call[1])).toEqual([ + expect.objectContaining({ status: "working" }), + { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + ]); + expect(fixture.submitToolResult.mock.calls[1]?.[2]).toBeUndefined(); + expect( + fixture.events.some( + (entry) => + (entry.payload as { type?: string; callId?: string }).type === "toolResult" && + (entry.payload as { callId?: string }).callId === fixture.callId, + ), + ).toBe(false); + }); + + it("supersedes a rejected forced final with canonical cancellation", async () => { + const fixture = await createSuppressionUnsupportedForcedConsultFixture(["native-call"]); + let rejectFinal: ((error: Error) => void) | undefined; + const rejectedFinal = new Promise((_resolve, reject) => { + rejectFinal = reject; + }); + fixture.submitToolResult.mockReturnValueOnce(rejectedFinal).mockReturnValueOnce(undefined); + const staleFinal = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { answer: "stale" }, + }); + cancelTalkRealtimeRelayTurn({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + reason: "barge-in", + }); + const cancellation = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: fixture.callId, + result: { status: "working" }, + options: { willContinue: true }, + }); + + rejectFinal?.(new Error("forced final rejected")); + await expect(staleFinal).rejects.toThrow("forced final rejected"); + await cancellation; + expect(fixture.submitToolResult.mock.calls.map((call) => call[1])).toEqual([ + { answer: "stale" }, + { + status: "cancelled", + message: "OpenClaw cancelled this consult before completion. Do not restart it.", + }, + ]); + }); + + it("keeps a forced consult deliverable when async provider cancellation rejects", async () => { + vi.useFakeTimers(); + setActiveEmbeddedRun( + "embedded-session-1", + { + queueMessage: vi.fn(async () => undefined), + isStreaming: () => true, + isCompacting: () => false, + abort: vi.fn(), + }, + "main", + ); + + let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined; + let rejectNextSubmission = false; + const bridge = { + supportsToolResultContinuation: true, + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + sendUserMessage: vi.fn(), + triggerGreeting: vi.fn(), + handleBargeIn: vi.fn(), + submitToolResult: vi.fn(() => { + if (!rejectNextSubmission) { + return undefined; + } + rejectNextSubmission = false; + return Promise.reject(new Error("provider cancellation rejected")); + }), + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }; + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: (request) => { + bridgeRequest = request; + return bridge; + }, + }; + const events: Array<{ event: string; payload: unknown; connIds: string[] }> = []; + const session = createTalkRealtimeRelaySession({ + context: { + broadcastToConnIds: (event: string, payload: unknown, connIds: ReadonlySet) => { + events.push({ event, payload, connIds: [...connIds] }); + }, + } as never, + connId: "conn-1", + provider, + providerConfig: {}, + instructions: "be brief", + tools: [], + forceAgentConsultOnFinalTranscript: true, + }); + await Promise.resolve(); + + bridgeRequest?.onTranscript?.("user", "Can you check this?", true); + await vi.advanceTimersByTimeAsync(250); + const forcedToolCall = findEventPayload( + events, + (payload) => payload.type === "toolCall" && payload.forced === true, + ); + const callId = String(forcedToolCall.callId); + bridgeRequest?.onToolCall?.({ + itemId: "native-item", + callId: "native-call", + name: "openclaw_agent_consult", + args: { question: "Can you check this?" }, + }); + registerTalkRealtimeRelayAgentRun({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + sessionKey: "main", + runId: "run-1", + callId, + }); + + rejectNextSubmission = true; + await expect( + steerTalkRealtimeRelayAgentRun({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + text: "cancel that", + mode: "cancel", + }), + ).rejects.toThrow("provider cancellation rejected"); + + await submitTalkRealtimeRelayToolResult({ + relaySessionId: session.relaySessionId, + connId: "conn-1", + callId, + result: { result: "Here is the checked answer." }, + }); + expect(bridge.submitToolResult).toHaveBeenLastCalledWith( + "native-call", + { + status: "already_delivered", + message: "OpenClaw already delivered this consult result internally. Do not repeat it.", + }, + { suppressResponse: true }, + ); + expect(bridge.sendUserMessage).toHaveBeenLastCalledWith( + expect.stringContaining("Here is the checked answer."), + ); + }); + it("does not duplicate control-like transcripts when the linked relay run is already gone", async () => { let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined; const bridge = { diff --git a/src/gateway/talk-realtime-relay.ts b/src/gateway/talk-realtime-relay.ts index e0ef58ace27b..344ac2ed814d 100644 --- a/src/gateway/talk-realtime-relay.ts +++ b/src/gateway/talk-realtime-relay.ts @@ -8,6 +8,7 @@ import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, buildRealtimeVoiceAgentConsultWorkingResponse, } from "../talk/agent-consult-tool.js"; +import { buildRealtimeVoiceAgentCancelProviderResult } from "../talk/agent-run-control-shared.js"; import { buildRealtimeVoiceAgentControlSpeechMessage, controlRealtimeVoiceAgentRun, @@ -18,6 +19,7 @@ import { readSpeakableRealtimeVoiceToolResult } from "../talk/consult-question.j import { createRealtimeVoiceForcedConsultCoordinator, type RealtimeVoiceForcedConsultCoordinator, + type RealtimeVoiceForcedConsultHandle, } from "../talk/forced-consult-coordinator.js"; import { recordTalkObservabilityEvent } from "../talk/observability.js"; import { @@ -104,6 +106,18 @@ type TalkRealtimeRelayEventPayload = type TalkRealtimeRelayEvent = TalkRealtimeRelayEventPayload & { talkEvent?: TalkEvent }; +type ForcedTerminalProviderResult = { + result: unknown; + options?: RealtimeVoiceToolResultOptions; + turnId: string; + epoch: number; +}; + +type RelayAgentControlProviderSubmission = { + completion?: Promise; + providerResponseStarted: boolean; +}; + type RelaySession = { id: string; connId: string; @@ -116,6 +130,22 @@ type RelaySession = { activeAgentRuns: Map; activeAgentToolCalls: Map; completedAgentToolCalls: Set; + // Cancelled calls retain their original turn long enough to terminally satisfy + // late browser results without creating a replacement turn or owner success event. + cancelledAgentToolCalls: Map; + pendingFinalToolResults: Map>; + // Provider acceptance survives partial retries independently from the owner-facing + // agent-call lifecycle, so accepted native ids are never submitted twice. + completedProviderToolResults: Set; + pendingProviderToolResults: Map>; + // A final result must wait until the provider accepts its continuation result; + // otherwise async bridges can observe final-before-working ordering. + pendingWorkingToolResults: Map>; + // Keep a forced terminal result open while late matching native ids join it. + // Delivery/cancellation closes the state only after every current id accepts. + forcedTerminalProviderResults: Map; + // Turn cancellation invalidates async acceptance callbacks from the prior turn. + toolResultEpoch: number; forcedConsults: RealtimeVoiceForcedConsultCoordinator; transcript: RealtimeVoiceTranscriptEntry[]; }; @@ -233,6 +263,14 @@ function buildAlreadyDeliveredToolResult(): Record { }; } +function suppressedToolResultOptions( + session: RelaySession, +): RealtimeVoiceToolResultOptions | undefined { + return session.bridge.bridge.supportsToolResultSuppression === false + ? undefined + : { suppressResponse: true }; +} + function cancelForcedConsults(session: RelaySession): void { for (const handle of session.forcedConsults.handles()) { session.forcedConsults.markCancelled(handle); @@ -311,39 +349,223 @@ function broadcastToolResultToOwner( }); } +function completeAfterToolResultSubmissions( + session: RelaySession, + submissions: Array>, + onAccepted: () => void, +): void | Promise { + const pending = submissions.filter( + (submission): submission is Promise => submission !== undefined, + ); + const complete = () => { + if (relaySessions.get(session.id) === session) { + onAccepted(); + } + }; + if (pending.length === 0) { + complete(); + return; + } + return Promise.all(pending).then(complete); +} + +function submitFinalProviderToolResult(params: { + session: RelaySession; + callId: string; + result: unknown; + options?: RealtimeVoiceToolResultOptions; + onAccepted?: () => void; +}): void | Promise { + if (params.session.completedProviderToolResults.has(params.callId)) { + if (relaySessions.get(params.session.id) === params.session) { + params.onAccepted?.(); + } + return; + } + const pending = params.session.pendingProviderToolResults.get(params.callId); + if (pending) { + return pending; + } + const submit = () => + params.session.bridge.submitToolResult(params.callId, params.result, params.options); + const working = params.session.pendingWorkingToolResults.get(params.callId); + const epoch = params.session.toolResultEpoch; + const submitAfterWorking = async () => { + if (relaySessions.get(params.session.id) !== params.session) { + return false; + } + if (params.session.toolResultEpoch !== epoch) { + if (!params.session.cancelledAgentToolCalls.has(params.callId)) { + return false; + } + // The browser already considers this final submitted while it waits behind + // the provider's working-result acknowledgement. Finish the cancelled call + // here so the provider is not left waiting for a terminal result. + await params.session.bridge.submitToolResult( + params.callId, + buildRealtimeVoiceAgentCancelProviderResult( + "OpenClaw cancelled this consult before completion. Do not restart it.", + ), + suppressedToolResultOptions(params.session), + ); + params.session.completedProviderToolResults.add(params.callId); + params.session.cancelledAgentToolCalls.delete(params.callId); + params.session.completedAgentToolCalls.add(params.callId); + return false; + } + await submit(); + return true; + }; + const submission = working ? working.then(submitAfterWorking, submitAfterWorking) : submit(); + const accept = () => { + params.session.completedProviderToolResults.add(params.callId); + if (relaySessions.get(params.session.id) === params.session) { + params.onAccepted?.(); + } + }; + if (!submission) { + accept(); + return; + } + const tracked = submission + .then((submitted) => { + if (submitted !== false) { + accept(); + } + }) + .finally(() => { + if (params.session.pendingProviderToolResults.get(params.callId) === tracked) { + params.session.pendingProviderToolResults.delete(params.callId); + } + }); + params.session.pendingProviderToolResults.set(params.callId, tracked); + return tracked; +} + +function trackAgentFinalToolResult( + session: RelaySession, + callId: string, + completion: void | Promise, +): void | Promise { + if (!completion) { + return; + } + const tracked = completion.finally(() => { + if (session.pendingFinalToolResults.get(callId) === tracked) { + session.pendingFinalToolResults.delete(callId); + } + }); + session.pendingFinalToolResults.set(callId, tracked); + return tracked; +} + +function trackPendingWorkingToolResult( + session: RelaySession, + callId: string, + completion: void | Promise, +): void | Promise { + if (!completion) { + return; + } + const tracked = completion.finally(() => { + if (session.pendingWorkingToolResults.get(callId) === tracked) { + session.pendingWorkingToolResults.delete(callId); + } + }); + session.pendingWorkingToolResults.set(callId, tracked); + return tracked; +} + +function clearRelayAgentToolCall(session: RelaySession, callId: string): void { + const runId = session.activeAgentToolCalls.get(callId); + session.activeAgentToolCalls.delete(callId); + if (!runId) { + return; + } + const runStillActive = [...session.activeAgentToolCalls.values()].includes(runId); + if (!runStillActive) { + session.activeAgentRuns.delete(runId); + } +} + function submitRelayAgentControlProviderResults( session: RelaySession, result: RealtimeVoiceAgentControlResult, turnId: string, -): void { +): RelayAgentControlProviderSubmission | undefined { if (result.mode !== "cancel" || !result.ok || !result.providerResult) { - return; + return undefined; } - const activeCallIds = [...session.activeAgentToolCalls.keys()]; - for (const callId of activeCallIds) { - const forcedConsult = session.forcedConsults.handles().find((handle) => handle.id === callId); + const providerResult = result.providerResult; + const epoch = session.toolResultEpoch; + const callIds = [...session.activeAgentToolCalls.keys()]; + const activeCallIds = callIds.filter((callId) => !session.pendingFinalToolResults.has(callId)); + const submissions: Array> = callIds + .map((callId) => session.pendingFinalToolResults.get(callId)) + .filter((pending): pending is Promise => pending !== undefined); + const toolResultOptions = suppressedToolResultOptions(session); + let providerResponseStarted = toolResultOptions === undefined && submissions.length > 0; + const finalizeAgentCall = (callId: string, forcedConsult?: RealtimeVoiceForcedConsultHandle) => { + if (session.toolResultEpoch !== epoch) { + return; + } if (forcedConsult) { - // Forced consults may have both synthetic and provider-native call ids; - // cancelling must satisfy every native id or the provider keeps waiting. session.forcedConsults.markCancelled(forcedConsult); - for (const nativeCallId of session.forcedConsults.nativeCallIds(forcedConsult)) { - session.bridge.submitToolResult(nativeCallId, result.providerResult, { - suppressResponse: true, - }); - } - } else { - session.bridge.submitToolResult(callId, result.providerResult, { suppressResponse: true }); } broadcastToolResultToOwner(session, { callId, turnId, - result: result.providerResult, + result: providerResult, final: true, }); - session.activeAgentToolCalls.delete(callId); + clearRelayAgentToolCall(session, callId); session.completedAgentToolCalls.add(callId); + }; + for (const callId of activeCallIds) { + const forcedConsult = session.forcedConsults.handles().find((handle) => handle.id === callId); + if (forcedConsult) { + const nativeCallIds = session.forcedConsults.nativeCallIds(forcedConsult); + providerResponseStarted ||= toolResultOptions === undefined && nativeCallIds.length > 0; + const terminal: ForcedTerminalProviderResult = { + result: providerResult, + options: toolResultOptions, + turnId, + epoch, + }; + session.forcedTerminalProviderResults.set(callId, terminal); + const clearTerminal = () => { + if (session.forcedTerminalProviderResults.get(callId) === terminal) { + session.forcedTerminalProviderResults.delete(callId); + } + }; + const drained = drainForcedTerminalProviderResultsAfterPending( + session, + forcedConsult, + terminal, + ); + const completed = completeAfterToolResultSubmissions(session, [drained], () => { + clearTerminal(); + finalizeAgentCall(callId, forcedConsult); + }); + const tracked = trackAgentFinalToolResult(session, callId, completed?.finally(clearTerminal)); + submissions.push(tracked); + continue; + } + providerResponseStarted ||= toolResultOptions === undefined; + const submitted = submitFinalProviderToolResult({ + session, + callId, + result: providerResult, + options: toolResultOptions, + onAccepted: () => finalizeAgentCall(callId), + }); + submissions.push(trackAgentFinalToolResult(session, callId, submitted)); } - session.activeAgentRuns.clear(); + const completion = completeAfterToolResultSubmissions(session, submissions, () => {}); + return { + ...(completion ? { completion } : {}), + providerResponseStarted, + }; } function closeRelaySession(session: RelaySession, reason: "completed" | "error"): void { @@ -582,7 +804,7 @@ export function createTalkRealtimeRelaySession( }, onToolCall: (toolCall) => { const relay = relayRef.current; - const turnId = relay ? ensureRelayTurn(relay) : undefined; + let shouldSubmitWorkingResult = false; if (relay && toolCall.name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { const forcedConsult = relay.forcedConsults.recordNativeConsult( toolCall.args, @@ -590,14 +812,26 @@ export function createTalkRealtimeRelaySession( ); if (forcedConsult.kind === "in_flight" || forcedConsult.kind === "already_delivered") { if (forcedConsult.kind === "already_delivered") { - submitAlreadyDeliveredToolResult(relay, toolCall.callId, turnId); - } else { - submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId, turnId); + const result = relay.forcedConsults.isCancelled(forcedConsult.handle) + ? buildRealtimeVoiceAgentCancelProviderResult( + "OpenClaw cancelled this consult before completion. Do not restart it.", + ) + : buildAlreadyDeliveredToolResult(); + return submitForcedConsultProviderResult( + relay, + toolCall.callId, + result, + suppressedToolResultOptions(relay), + ); } - return; + if (relay.forcedTerminalProviderResults.has(forcedConsult.handle.id)) { + return relay.pendingFinalToolResults.get(forcedConsult.handle.id); + } + return submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId); } - submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId, turnId); + shouldSubmitWorkingResult = true; } + const turnId = relay ? ensureRelayTurn(relay) : undefined; emit( { relaySessionId, @@ -615,6 +849,9 @@ export function createTalkRealtimeRelaySession( payload: { name: toolCall.name, args: toolCall.args }, }, ); + if (relay && shouldSubmitWorkingResult) { + return submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId, turnId); + } }, onReady: () => { ready = true; @@ -680,6 +917,13 @@ export function createTalkRealtimeRelaySession( activeAgentRuns: new Map(), activeAgentToolCalls: new Map(), completedAgentToolCalls: new Set(), + cancelledAgentToolCalls: new Map(), + pendingFinalToolResults: new Map(), + completedProviderToolResults: new Set(), + pendingProviderToolResults: new Map(), + pendingWorkingToolResults: new Map(), + forcedTerminalProviderResults: new Map(), + toolResultEpoch: 0, forcedConsults: createRealtimeVoiceForcedConsultCoordinator(), transcript: [], }; @@ -770,49 +1014,97 @@ function scheduleForcedAgentConsult(session: RelaySession | undefined, question: }); } -function submitAlreadyDeliveredToolResult( +function submitForcedConsultProviderResult( session: RelaySession, callId: string, - turnId = ensureRelayTurn(session), -): void { - const result = buildAlreadyDeliveredToolResult(); - session.bridge.submitToolResult(callId, result, { suppressResponse: true }); - broadcastToOwner(session.context, session.connId, { - relaySessionId: session.id, - type: "toolResult", + result: unknown, + options: RealtimeVoiceToolResultOptions | undefined, +): void | Promise { + return submitFinalProviderToolResult({ + session, callId, - talkEvent: session.talk.emit({ - type: "tool.result", - callId, - turnId, - payload: { name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, result }, - final: true, - }), + result, + options, }); } +function drainForcedTerminalProviderResults( + session: RelaySession, + handle: RealtimeVoiceForcedConsultHandle, + terminal: ForcedTerminalProviderResult, +): void | Promise { + if (session.forcedTerminalProviderResults.get(handle.id) !== terminal) { + return; + } + const submissions = session.forcedConsults + .nativeCallIds(handle) + .map((callId) => + submitForcedConsultProviderResult(session, callId, terminal.result, terminal.options), + ); + const pending = submissions.filter( + (submission): submission is Promise => submission !== undefined, + ); + if (pending.length > 0) { + return Promise.all(pending).then(() => + drainForcedTerminalProviderResults(session, handle, terminal), + ); + } + const hasUnsubmittedCall = session.forcedConsults + .nativeCallIds(handle) + .some((callId) => !session.completedProviderToolResults.has(callId)); + if (hasUnsubmittedCall) { + return drainForcedTerminalProviderResults(session, handle, terminal); + } +} + +function drainForcedTerminalProviderResultsAfterPending( + session: RelaySession, + handle: RealtimeVoiceForcedConsultHandle, + terminal: ForcedTerminalProviderResult, +): void | Promise { + const pending = session.forcedConsults + .nativeCallIds(handle) + .map((callId) => session.pendingProviderToolResults.get(callId)) + .filter((submission): submission is Promise => submission !== undefined); + if (pending.length === 0) { + return drainForcedTerminalProviderResults(session, handle, terminal); + } + return Promise.allSettled(pending).then(() => + drainForcedTerminalProviderResults(session, handle, terminal), + ); +} + function submitRealtimeAgentConsultWorkingResponse( session: RelaySession, callId: string, turnId = ensureRelayTurn(session), -): void { +): void | Promise { if (!session.bridge.bridge.supportsToolResultContinuation) { return; } - session.bridge.submitToolResult(callId, buildRealtimeVoiceAgentConsultWorkingResponse("person"), { - willContinue: true, - }); - broadcastToOwner(session.context, session.connId, { - relaySessionId: session.id, - type: "toolResult", + const epoch = session.toolResultEpoch; + const submission = session.bridge.submitToolResult( callId, - talkEvent: session.talk.emit({ - type: "tool.progress", + buildRealtimeVoiceAgentConsultWorkingResponse("person"), + { willContinue: true }, + ); + const completion = completeAfterToolResultSubmissions(session, [submission], () => { + if (session.toolResultEpoch !== epoch) { + return; + } + broadcastToOwner(session.context, session.connId, { + relaySessionId: session.id, + type: "toolResult", callId, - turnId, - payload: { name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, status: "working" }, - }), + talkEvent: session.talk.emit({ + type: "tool.progress", + callId, + turnId, + payload: { name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, status: "working" }, + }), + }); }); + return trackPendingWorkingToolResult(session, callId, completion); } function ensureRelayTurn(session: RelaySession): string { @@ -874,64 +1166,200 @@ export function submitTalkRealtimeRelayToolResult(params: { callId: string; result: unknown; options?: RealtimeVoiceToolResultOptions; -}): void { +}): void | Promise { const session = getRelaySession(params.relaySessionId, params.connId); if (session.completedAgentToolCalls.has(params.callId)) { return; } + const pendingFinal = session.pendingFinalToolResults.get(params.callId); + const cancelledAgentCall = session.cancelledAgentToolCalls.has(params.callId); + if (pendingFinal && !cancelledAgentCall) { + return pendingFinal; + } const forcedConsult = session.forcedConsults .handles() .find((handle) => handle.id === params.callId); if (forcedConsult) { - const turnId = ensureRelayTurn(session); const cancelled = session.forcedConsults.isCancelled(forcedConsult); + const turnId = cancelled + ? (session.cancelledAgentToolCalls.get(params.callId) ?? session.talk.activeTurnId) + : ensureRelayTurn(session); + if (!turnId) { + throw new Error("Cancelled realtime consult is missing its original turn"); + } if (cancelled) { - if (params.options?.willContinue !== true) { + const providerResult = buildRealtimeVoiceAgentCancelProviderResult( + "OpenClaw cancelled this consult before completion. Do not restart it.", + ); + const terminal: ForcedTerminalProviderResult = { + result: providerResult, + options: suppressedToolResultOptions(session), + turnId, + epoch: session.toolResultEpoch, + }; + session.forcedTerminalProviderResults.set(forcedConsult.id, terminal); + const clearTerminal = () => { + if (session.forcedTerminalProviderResults.get(forcedConsult.id) === terminal) { + session.forcedTerminalProviderResults.delete(forcedConsult.id); + } + }; + const drained = drainForcedTerminalProviderResultsAfterPending( + session, + forcedConsult, + terminal, + ); + const completion = completeAfterToolResultSubmissions(session, [drained], () => { + clearTerminal(); + if (session.toolResultEpoch !== terminal.epoch) { + return; + } session.forcedConsults.markCancelled(forcedConsult); - } - } else if (isWorkingToolResult(params.result)) { - session.bridge.sendUserMessage(buildForcedConsultCheckingPrompt()); - } else { - session.forcedConsults.markDelivered(forcedConsult); - const text = readSpeakableRealtimeVoiceToolResult(params.result, { - maxChars: FORCED_CONSULT_RESULT_MAX_CHARS, + clearRelayAgentToolCall(session, params.callId); + session.cancelledAgentToolCalls.delete(params.callId); + session.completedAgentToolCalls.add(params.callId); + broadcastToolResultToOwner(session, { + callId: params.callId, + turnId, + result: providerResult, + forced: true, + final: true, + }); }); - for (const nativeCallId of session.forcedConsults.nativeCallIds(forcedConsult)) { - submitAlreadyDeliveredToolResult(session, nativeCallId, turnId); - } - if (text) { - session.bridge.sendUserMessage(buildForcedConsultSpeechPrompt(text)); - } + return trackAgentFinalToolResult(session, params.callId, completion?.finally(clearTerminal)); } const final = params.options?.willContinue !== true; - if (final && !cancelled && !isWorkingToolResult(params.result)) { + if (!final) { + if (isWorkingToolResult(params.result)) { + session.bridge.sendUserMessage(buildForcedConsultCheckingPrompt()); + } + broadcastToolResultToOwner(session, { + callId: params.callId, + turnId, + result: params.result, + forced: true, + final: false, + }); + return; + } + const text = readSpeakableRealtimeVoiceToolResult(params.result, { + maxChars: FORCED_CONSULT_RESULT_MAX_CHARS, + }); + const providerOptions = suppressedToolResultOptions(session); + const providerResult = providerOptions ? buildAlreadyDeliveredToolResult() : params.result; + const terminal: ForcedTerminalProviderResult = { + result: providerResult, + options: providerOptions, + turnId, + epoch: session.toolResultEpoch, + }; + session.forcedTerminalProviderResults.set(forcedConsult.id, terminal); + const submission = drainForcedTerminalProviderResults(session, forcedConsult, terminal); + const clearTerminal = () => { + if (session.forcedTerminalProviderResults.get(forcedConsult.id) === terminal) { + session.forcedTerminalProviderResults.delete(forcedConsult.id); + } + }; + const completion = completeAfterToolResultSubmissions(session, [submission], () => { + clearTerminal(); + if (session.toolResultEpoch !== terminal.epoch) { + return; + } session.forcedConsults.markDelivered(forcedConsult); + clearRelayAgentToolCall(session, params.callId); + session.completedAgentToolCalls.add(params.callId); + const hasNativeCalls = session.forcedConsults.nativeCallIds(forcedConsult).length > 0; + if (text && (!hasNativeCalls || providerOptions)) { + session.bridge.sendUserMessage(buildForcedConsultSpeechPrompt(text)); + } + broadcastToolResultToOwner(session, { + callId: params.callId, + turnId, + result: params.result, + forced: true, + final: true, + }); + }); + const trackedCompletion = completion?.finally(clearTerminal); + return trackAgentFinalToolResult(session, params.callId, trackedCompletion); + } + if (cancelledAgentCall) { + const providerResult = buildRealtimeVoiceAgentCancelProviderResult( + "OpenClaw cancelled this consult before completion. Do not restart it.", + ); + const submitCancellation = () => + submitFinalProviderToolResult({ + session, + callId: params.callId, + result: providerResult, + options: suppressedToolResultOptions(session), + onAccepted: () => { + session.cancelledAgentToolCalls.delete(params.callId); + session.completedAgentToolCalls.add(params.callId); + }, + }); + const pendingProvider = session.pendingProviderToolResults.get(params.callId); + const completion = pendingProvider + ? pendingProvider.then(submitCancellation, submitCancellation) + : submitCancellation(); + return trackAgentFinalToolResult(session, params.callId, completion); + } + if ( + params.options?.suppressResponse === true && + session.bridge.bridge.supportsToolResultSuppression === false + ) { + throw new Error("Realtime provider does not support suppressed tool results"); + } + // A final result owns provider completion for this call. Follow-up RPCs share it so + // only one accepted submission can clear the linked run and emit the success event. + const final = params.options?.willContinue !== true; + const turnId = ensureRelayTurn(session); + const epoch = session.toolResultEpoch; + const onAccepted = () => { + if (session.toolResultEpoch !== epoch) { + return; + } + if (final) { + clearRelayAgentToolCall(session, params.callId); + session.completedAgentToolCalls.add(params.callId); } broadcastToolResultToOwner(session, { callId: params.callId, turnId, result: params.result, - forced: true, final, }); - return; - } - session.bridge.submitToolResult(params.callId, params.result, params.options); - const turnId = ensureRelayTurn(session); - const final = params.options?.willContinue !== true; + }; if (final) { - const runId = session.activeAgentToolCalls.get(params.callId); - if (runId) { - session.activeAgentRuns.delete(runId); - session.activeAgentToolCalls.delete(params.callId); - } + const completion = submitFinalProviderToolResult({ + session, + callId: params.callId, + result: params.result, + options: params.options, + onAccepted, + }); + return trackAgentFinalToolResult(session, params.callId, completion); } - broadcastToolResultToOwner(session, { - callId: params.callId, - turnId, - result: params.result, - final, - }); + const submit = () => + session.bridge.submitToolResult(params.callId, params.result, params.options); + const pendingWorking = session.pendingWorkingToolResults.get(params.callId); + if (pendingWorking) { + const submission = pendingWorking.then(async () => { + if (relaySessions.get(session.id) !== session || session.toolResultEpoch !== epoch) { + return false; + } + await submit(); + return true; + }); + const completion = submission.then((submitted) => { + if (submitted && relaySessions.get(session.id) === session) { + onAccepted(); + } + }); + return trackPendingWorkingToolResult(session, params.callId, completion); + } + const submission = submit(); + const completion = completeAfterToolResultSubmissions(session, [submission], onAccepted); + return trackPendingWorkingToolResult(session, params.callId, completion); } /** Tracks the chat run started for a realtime agent-consult tool call. */ @@ -975,24 +1403,36 @@ export async function steerTalkRealtimeRelayAgentRun(params: { mode: params.mode, recentEvents: session.talk.recentEvents, }); + if (relaySessions.get(session.id) !== session) { + throw new Error("Realtime relay session closed while steering the agent run"); + } const turnId = ensureRelayTurn(session); + const providerSubmission = submitRelayAgentControlProviderResults(session, result, turnId); + if (providerSubmission?.completion) { + await providerSubmission.completion; + } + const finalResult = providerSubmission?.providerResponseStarted + ? { ...result, suppress: true } + : result; + if (relaySessions.get(session.id) !== session) { + return finalResult; + } broadcastToOwner(session.context, session.connId, { relaySessionId: session.id, type: "toolProgress", - result, + result: finalResult, talkEvent: session.talk.emit({ type: "tool.progress", turnId, payload: { name: "openclaw_agent_control", - phase: result.mode, - result, + phase: finalResult.mode, + result: finalResult, }, - final: result.mode === "cancel" || result.mode === "status", + final: finalResult.mode === "cancel" || finalResult.mode === "status", }), }); - submitRelayAgentControlProviderResults(session, result, turnId); - return result; + return finalResult; } /** Cancels the active relay turn, aborts agent work, and clears provider audio. */ @@ -1002,9 +1442,22 @@ export function cancelTalkRealtimeRelayTurn(params: { reason?: string; }): void { const session = getRelaySession(params.relaySessionId, params.connId); + session.toolResultEpoch += 1; + session.forcedTerminalProviderResults.clear(); const turnId = ensureRelayTurn(session); const reason = params.reason ?? "client-cancelled"; cancelForcedConsults(session); + for (const callId of session.activeAgentToolCalls.keys()) { + session.cancelledAgentToolCalls.set(callId, turnId); + } + for (const forcedConsult of session.forcedConsults.handles()) { + if (session.forcedConsults.isCancelled(forcedConsult)) { + session.cancelledAgentToolCalls.set(forcedConsult.id, turnId); + for (const nativeCallId of session.forcedConsults.nativeCallIds(forcedConsult)) { + session.cancelledAgentToolCalls.set(nativeCallId, turnId); + } + } + } session.bridge.handleBargeIn({ audioPlaybackActive: true }); abortRelayAgentRuns(session, reason); const cancelled = session.talk.cancelTurn({ diff --git a/src/talk/forced-consult-coordinator.test.ts b/src/talk/forced-consult-coordinator.test.ts index f87d262df874..cd56e9387674 100644 --- a/src/talk/forced-consult-coordinator.test.ts +++ b/src/talk/forced-consult-coordinator.test.ts @@ -47,6 +47,20 @@ describe("realtime voice forced consult coordinator", () => { } }); + it("does not retroactively cancel a delivered handle", () => { + const coordinator = createRealtimeVoiceForcedConsultCoordinator(); + const handle = coordinator.prepare("Can you check this?", { id: "forced-1" }); + coordinator.markStarted(handle!); + coordinator.markDelivered(handle!); + + coordinator.markCancelled(handle!); + + expect(coordinator.isCancelled(handle!)).toBe(false); + expect( + coordinator.recordNativeConsult({ question: "Can you check this?" }, "native-call"), + ).toMatchObject({ kind: "already_delivered", handle: { id: "forced-1" } }); + }); + it("tracks native-first consults during the dedupe window", () => { vi.useFakeTimers(); try { @@ -174,7 +188,7 @@ describe("realtime voice forced consult coordinator", () => { expect(scheduledDelays).toEqual([MAX_TIMER_TIMEOUT_MS]); }); - it("reports cancelled handles until the dedupe window expires", () => { + it("matches cancelled handles until the dedupe window expires", () => { vi.useFakeTimers(); try { const coordinator = createRealtimeVoiceForcedConsultCoordinator(); @@ -183,8 +197,15 @@ describe("realtime voice forced consult coordinator", () => { coordinator.markCancelled(pending!); expect(coordinator.isCancelled(pending!)).toBe(true); + expect( + coordinator.recordNativeConsult({ question: "check status" }, "native-call"), + ).toMatchObject({ kind: "already_delivered", handle: { id: "forced-1" } }); + expect(coordinator.nativeCallIds(pending!)).toEqual(["native-call"]); vi.advanceTimersByTime(2_001); expect(coordinator.isCancelled(pending!)).toBe(false); + expect(coordinator.recordNativeConsult({ question: "check status" })).toMatchObject({ + kind: "none", + }); } finally { vi.useRealTimers(); } diff --git a/src/talk/forced-consult-coordinator.ts b/src/talk/forced-consult-coordinator.ts index 62dd61d3fbca..956502aea1c9 100644 --- a/src/talk/forced-consult-coordinator.ts +++ b/src/talk/forced-consult-coordinator.ts @@ -326,13 +326,16 @@ export function createRealtimeVoiceForcedConsultCoordinator( return { kind: "pending", question, handle: pending.handle }; } const stored = findMatching(question); - if (!stored || stored.cancelled) { + if (!stored) { return { kind: "none", question }; } if (nativeCallId) { stored.nativeCallIds.add(nativeCallId); } rememberStoredQuestion(stored, question); + if (stored.cancelled) { + return { kind: "already_delivered", question, handle: stored.handle }; + } if (stored.delivered) { return { kind: "already_delivered", question, handle: stored.handle }; } @@ -363,7 +366,7 @@ export function createRealtimeVoiceForcedConsultCoordinator( }, markCancelled(handle) { const stored = getStored(handle); - if (!stored) { + if (!stored || stored.delivered) { return; } clearTimer(stored); diff --git a/src/talk/provider-types.ts b/src/talk/provider-types.ts index df8d20d614de..ebec88633c15 100644 --- a/src/talk/provider-types.ts +++ b/src/talk/provider-types.ts @@ -188,13 +188,23 @@ export type RealtimeVoiceBrowserSession = export type RealtimeVoiceBridge = { supportsToolResultContinuation?: boolean; + /** False when the provider cannot accept a tool result without starting a response. */ + supportsToolResultSuppression?: boolean; connect(): Promise; sendAudio(audio: Buffer): void; setMediaTimestamp(ts: number): void; sendUserMessage?(text: string): void; triggerGreeting?(instructions?: string): void; handleBargeIn?(options?: RealtimeVoiceBargeInOptions): void; - submitToolResult(callId: string, result: unknown, options?: RealtimeVoiceToolResultOptions): void; + /** + * Returns void when submission completes synchronously, or a Promise that resolves at the + * asynchronous completion boundary exposed by the provider and rejects on submission failure. + */ + submitToolResult( + callId: string, + result: unknown, + options?: RealtimeVoiceToolResultOptions, + ): void | Promise; acknowledgeMark(): void; close(): void; isConnected(): boolean; diff --git a/src/talk/session-runtime.test.ts b/src/talk/session-runtime.test.ts index ba7d5feecf99..ec94e62ea4a1 100644 --- a/src/talk/session-runtime.test.ts +++ b/src/talk/session-runtime.test.ts @@ -227,8 +227,126 @@ describe("realtime voice bridge session runtime", () => { expect(onToolCall).toHaveBeenCalledWith(event, session); }); - it("forwards tool result continuation options to the provider bridge", () => { - const bridge = makeBridge(); + it("routes synchronous and asynchronous tool-call callback failures to onError", async () => { + let callbacks: Parameters[0] | undefined; + const onError = vi.fn(); + const syncFailure = new Error("sync callback failed"); + const provider: RealtimeVoiceProviderPlugin = { + id: "test", + label: "Test", + isConfigured: () => true, + createBridge: (request) => { + callbacks = request; + return makeBridge(); + }, + }; + const onToolCall = vi + .fn() + .mockImplementationOnce(() => { + throw syncFailure; + }) + .mockImplementationOnce(() => Promise.reject(new Error("async callback failed"))); + createRealtimeVoiceBridgeSession({ + provider, + providerConfig: {}, + audioSink: { sendAudio: vi.fn() }, + onToolCall, + onError, + }); + const event = { + itemId: "item-1", + callId: "call-1", + name: "lookup", + args: {}, + }; + + callbacks?.onToolCall?.(event); + callbacks?.onToolCall?.(event); + await Promise.resolve(); + + expect(onError).toHaveBeenNthCalledWith(1, syncFailure); + expect(onError).toHaveBeenNthCalledWith(2, new Error("async callback failed")); + }); + + it("contains an onError exception after an asynchronous tool-call failure", async () => { + let callbacks: Parameters[0] | undefined; + const onError = vi.fn(() => { + throw new Error("error callback failed"); + }); + const provider: RealtimeVoiceProviderPlugin = { + id: "test", + label: "Test", + isConfigured: () => true, + createBridge: (request) => { + callbacks = request; + return makeBridge(); + }, + }; + createRealtimeVoiceBridgeSession({ + provider, + providerConfig: {}, + audioSink: { sendAudio: vi.fn() }, + onToolCall: () => Promise.reject(new Error("tool callback failed")), + onError, + }); + + callbacks?.onToolCall?.({ + itemId: "item-1", + callId: "call-1", + name: "lookup", + args: {}, + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(onError).toHaveBeenCalledWith(new Error("tool callback failed")); + }); + + it("does not report an asynchronous tool-call failure after the session closes", async () => { + let callbacks: Parameters[0] | undefined; + let rejectToolCall: ((error: Error) => void) | undefined; + const close = vi.fn(); + const bridge = makeBridge({ close }); + const onError = vi.fn(); + const provider: RealtimeVoiceProviderPlugin = { + id: "test", + label: "Test", + isConfigured: () => true, + createBridge: (request) => { + callbacks = request; + return bridge; + }, + }; + const session = createRealtimeVoiceBridgeSession({ + provider, + providerConfig: {}, + audioSink: { sendAudio: vi.fn() }, + onToolCall: () => + new Promise((_resolve, reject) => { + rejectToolCall = reject; + }), + onError, + }); + + callbacks?.onToolCall?.({ + itemId: "item-1", + callId: "call-1", + name: "lookup", + args: {}, + }); + session.close(); + rejectToolCall?.(new Error("late tool callback failure")); + await Promise.resolve(); + + expect(close).toHaveBeenCalledTimes(1); + expect(onError).not.toHaveBeenCalled(); + }); + + it("forwards tool result continuation options and async acceptance to the provider bridge", () => { + const acceptance = Promise.resolve(); + const submitToolResult = vi.fn(() => acceptance); + const bridge = makeBridge({ submitToolResult }); const provider: RealtimeVoiceProviderPlugin = { id: "test", label: "Test", @@ -241,13 +359,42 @@ describe("realtime voice bridge session runtime", () => { audioSink: { sendAudio: vi.fn() }, }); - session.submitToolResult("call-1", { status: "working" }, { willContinue: true }); - - expect(bridge["submitToolResult"]).toHaveBeenCalledWith( + const submitted = session.submitToolResult( "call-1", { status: "working" }, { willContinue: true }, ); + + expect(submitted).toBe(acceptance); + expect(submitToolResult).toHaveBeenCalledWith( + "call-1", + { status: "working" }, + { willContinue: true }, + ); + }); + + it("rejects suppressed results before calling an unsupported provider bridge", () => { + const submitToolResult = vi.fn(); + const bridge = makeBridge({ + submitToolResult, + supportsToolResultSuppression: false, + }); + const provider: RealtimeVoiceProviderPlugin = { + id: "test", + label: "Test", + isConfigured: () => true, + createBridge: () => bridge, + }; + const session = createRealtimeVoiceBridgeSession({ + provider, + providerConfig: {}, + audioSink: { sendAudio: vi.fn() }, + }); + + expect(() => + session.submitToolResult("call-1", { ok: true }, { suppressResponse: true }), + ).toThrow("Realtime provider does not support suppressed tool results"); + expect(submitToolResult).not.toHaveBeenCalled(); }); it("does not expose session callbacks until the provider returns its bridge", () => { diff --git a/src/talk/session-runtime.ts b/src/talk/session-runtime.ts index e80ac207cbe1..252397517e39 100644 --- a/src/talk/session-runtime.ts +++ b/src/talk/session-runtime.ts @@ -42,7 +42,11 @@ export type RealtimeVoiceBridgeSession = { sendUserMessage(text: string): void; handleBargeIn(options?: RealtimeVoiceBargeInOptions): void; setMediaTimestamp(ts: number): void; - submitToolResult(callId: string, result: unknown, options?: RealtimeVoiceToolResultOptions): void; + submitToolResult( + callId: string, + result: unknown, + options?: RealtimeVoiceToolResultOptions, + ): void | Promise; triggerGreeting(instructions?: string): void; }; @@ -64,7 +68,10 @@ export type RealtimeVoiceBridgeSessionParams = { tools?: RealtimeVoiceTool[]; onTranscript?: (role: RealtimeVoiceRole, text: string, isFinal: boolean) => void; onEvent?: (event: RealtimeVoiceBridgeEvent) => void; - onToolCall?: (event: RealtimeVoiceToolCallEvent, session: RealtimeVoiceBridgeSession) => void; + onToolCall?: ( + event: RealtimeVoiceToolCallEvent, + session: RealtimeVoiceBridgeSession, + ) => void | Promise; onReady?: (session: RealtimeVoiceBridgeSession) => void; onError?: (error: Error) => void; onClose?: (reason: RealtimeVoiceCloseReason) => void; @@ -77,6 +84,7 @@ export function createRealtimeVoiceBridgeSession( params: RealtimeVoiceBridgeSessionParams, ): RealtimeVoiceBridgeSession { const bridgeRef: { current?: RealtimeVoiceBridge } = {}; + let isActive = true; const requireBridge = () => { if (!bridgeRef.current) { throw new Error("Realtime voice bridge is not ready"); @@ -90,17 +98,38 @@ export function createRealtimeVoiceBridgeSession( return requireBridge(); }, acknowledgeMark: () => requireBridge().acknowledgeMark(), - close: () => requireBridge().close(), + close: () => { + const bridge = requireBridge(); + isActive = false; + bridge.close(); + }, connect: () => requireBridge().connect(), sendAudio: (audio) => requireBridge().sendAudio(audio), sendUserMessage: (text) => requireBridge().sendUserMessage?.(text), handleBargeIn: (options) => requireBridge().handleBargeIn?.(options), setMediaTimestamp: (ts) => requireBridge().setMediaTimestamp(ts), - submitToolResult: (callId, result, options) => - requireBridge().submitToolResult(callId, result, options), + submitToolResult: (callId, result, options) => { + const bridge = requireBridge(); + if (options?.suppressResponse && bridge.supportsToolResultSuppression === false) { + throw new Error("Realtime provider does not support suppressed tool results"); + } + return bridge.submitToolResult(callId, result, options); + }, triggerGreeting: (instructions) => requireBridge().triggerGreeting?.(instructions), }; const canSendAudio = () => params.audioSink.isOpen?.() ?? true; + const reportCallbackError = (error: unknown) => { + // Async tool handlers can settle after the provider closes. Once inactive, no + // callback may report stale failures into the next session lifecycle. + if (!isActive) { + return; + } + try { + params.onError?.(error instanceof Error ? error : new Error(String(error))); + } catch { + // An error callback is the terminal boundary for provider callback failures. + } + }; const bridge = params.provider.createBridge({ cfg: params.cfg, providerConfig: params.providerConfig, @@ -136,10 +165,17 @@ export function createRealtimeVoiceBridgeSession( onTranscript: params.onTranscript, onEvent: params.onEvent, onToolCall: (event) => { - if (!bridgeRef.current) { + if (!bridgeRef.current || !isActive) { return; } - params.onToolCall?.(event, session); + try { + const pending = params.onToolCall?.(event, session); + if (pending) { + void pending.catch(reportCallbackError); + } + } catch (error) { + reportCallbackError(error); + } }, onReady: () => { if (!bridgeRef.current) { @@ -151,7 +187,10 @@ export function createRealtimeVoiceBridgeSession( params.onReady?.(session); }, onError: params.onError, - onClose: params.onClose, + onClose: (reason) => { + isActive = false; + params.onClose?.(reason); + }, }); bridgeRef.current = bridge; diff --git a/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts b/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts index 1a231b80f352..b47c4e02a9fe 100644 --- a/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts +++ b/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts @@ -489,6 +489,115 @@ describe("GatewayRelayRealtimeTalkTransport", () => { transport.stop(); }); + it("waits for provider tool-result submission before returning to listening", async () => { + let resolveSubmission: () => void = () => undefined; + const submission = new Promise((resolve) => { + resolveSubmission = resolve; + }); + const onStatus = vi.fn(); + const client = createClient(); + vi.mocked(client["request"]).mockImplementation(async (method) => { + if (method === "talk.client.toolCall") { + return { runId: "run-1" }; + } + if (method === "talk.session.submitToolResult") { + await submission; + } + return {}; + }); + const transport = new GatewayRelayRealtimeTalkTransport(createSession(), { + callbacks: { onStatus }, + client, + sessionKey: "main", + }); + + await transport.start(); + emitGatewayFrame({ + event: "talk.event", + payload: { + relaySessionId: "relay-1", + type: "toolCall", + callId: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + args: { question: "status?" }, + }, + }); + await vi.waitFor(() => expect(requestCallsFor(client, "talk.client.toolCall")).toHaveLength(1)); + emitGatewayFrame({ + event: "chat", + payload: { + runId: "run-1", + state: "final", + message: { text: "All systems green." }, + }, + }); + await vi.waitFor(() => + expect(requestCallsFor(client, "talk.session.submitToolResult")).toHaveLength(1), + ); + emitGatewayFrame({ + event: "talk.event", + payload: { + relaySessionId: "relay-1", + type: "toolResult", + callId: "call-1", + }, + }); + + expect(onStatus).not.toHaveBeenCalledWith("listening"); + expect(requestCallsFor(client, "chat.abort")).toHaveLength(0); + resolveSubmission(); + await vi.waitFor(() => expect(onStatus).toHaveBeenCalledWith("listening")); + transport.stop(); + }); + + it("surfaces rejected provider tool-result submission without an unhandled rejection", async () => { + const onStatus = vi.fn(); + const client = createClient(); + vi.mocked(client["request"]).mockImplementation(async (method) => { + if (method === "talk.client.toolCall") { + return { runId: "run-1" }; + } + if (method === "talk.session.submitToolResult") { + throw new Error("Provider rejected the tool result"); + } + return {}; + }); + const transport = new GatewayRelayRealtimeTalkTransport(createSession(), { + callbacks: { onStatus }, + client, + sessionKey: "main", + }); + + await transport.start(); + emitGatewayFrame({ + event: "talk.event", + payload: { + relaySessionId: "relay-1", + type: "toolCall", + callId: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + args: { question: "status?" }, + }, + }); + await vi.waitFor(() => expect(requestCallsFor(client, "talk.client.toolCall")).toHaveLength(1)); + emitGatewayFrame({ + event: "chat", + payload: { + runId: "run-1", + state: "final", + message: { text: "All systems green." }, + }, + }); + + await vi.waitFor(() => + expect(onStatus).toHaveBeenCalledWith("error", "Provider rejected the tool result"), + ); + expect(onStatus).toHaveBeenLastCalledWith("error", "Provider rejected the tool result"); + expect(onStatus).not.toHaveBeenCalledWith("listening"); + expect(requestCallsFor(client, "talk.session.submitToolResult")).toHaveLength(1); + transport.stop(); + }); + it("submits an interim working result for forced consult tool calls", async () => { const client = createClient(); vi.mocked(client["request"]).mockImplementation(async (method) => { @@ -532,6 +641,47 @@ describe("GatewayRelayRealtimeTalkTransport", () => { transport.stop(); }); + it("does not start a forced consult when the working result is terminally cancelled", async () => { + const client = createClient(); + vi.mocked(client["request"]).mockImplementation(async (method) => { + if (method === "talk.session.submitToolResult") { + emitGatewayFrame({ + event: "talk.event", + payload: { + relaySessionId: "relay-1", + type: "toolResult", + callId: "call-1", + }, + }); + } + return {}; + }); + const transport = new GatewayRelayRealtimeTalkTransport(createSession(), { + callbacks: {}, + client, + sessionKey: "main", + }); + + await transport.start(); + emitGatewayFrame({ + event: "talk.event", + payload: { + relaySessionId: "relay-1", + type: "toolCall", + callId: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + forced: true, + args: { question: "status?" }, + }, + }); + + await vi.waitFor(() => + expect(requestCallsFor(client, "talk.session.submitToolResult")).toHaveLength(1), + ); + expect(requestCallsFor(client, "talk.client.toolCall")).toHaveLength(0); + transport.stop(); + }); + it("treats server relay tool results as terminal for active consult calls", async () => { const client = createClient(); vi.mocked(client["request"]).mockImplementation(async (method) => { diff --git a/ui/src/pages/chat/realtime-talk-gateway-relay.ts b/ui/src/pages/chat/realtime-talk-gateway-relay.ts index 90d20c13dbd3..b1cb417dd81c 100644 --- a/ui/src/pages/chat/realtime-talk-gateway-relay.ts +++ b/ui/src/pages/chat/realtime-talk-gateway-relay.ts @@ -61,6 +61,7 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport private readonly outputQueue = new RealtimeTalkPcmOutputQueue(); private readonly consultAbortControllers = new Map(); private readonly completedToolCalls = new Set(); + private readonly submittingToolCalls = new Set(); private cancelRequestedForPlayback = false; private speechFramesDuringPlayback = 0; private lastRelayError: string | undefined; @@ -201,6 +202,9 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport return; case "clear": this.stopOutput(); + if (event.talkEvent?.type === "turn.cancelled") { + this.abortConsults(); + } return; case "mark": this.scheduleMarkAck(); @@ -215,7 +219,9 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport } return; case "toolCall": - void this.handleToolCall(event); + void this.handleToolCall(event).catch((error: unknown) => { + this.reportToolResultSubmissionError(error); + }); return; case "toolResult": if (this.isFinalToolResult(event)) { @@ -278,14 +284,16 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport return; } if (name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { - this.submitToolResult(callId, { error: `Tool "${name}" not available in browser Talk` }); + await this.submitToolResult(callId, { + error: `Tool "${name}" not available in browser Talk`, + }); return; } const abortController = new AbortController(); this.consultAbortControllers.set(callId, abortController); try { if (event.forced) { - this.submitToolResult( + await this.submitToolResult( callId, { status: "working", @@ -295,6 +303,13 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport }, { willContinue: true }, ); + if (this.completedToolCalls.has(callId)) { + return; + } + if (abortController.signal.aborted) { + await this.submitToolResult(callId, { status: "cancelled" }); + return; + } } await submitRealtimeTalkConsult({ ctx: this.ctx, @@ -309,20 +324,34 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport } } - private submitToolResult( + private async submitToolResult( callId: string, result: unknown, options?: { suppressResponse?: boolean; willContinue?: boolean }, - ): void { + ): Promise { if (this.completedToolCalls.has(callId)) { return; } - void this.ctx.client.request("talk.session.submitToolResult", { - sessionId: this.session.relaySessionId, - callId, - result, - ...(options ? { options } : {}), - }); + this.submittingToolCalls.add(callId); + try { + await this.ctx.client.request("talk.session.submitToolResult", { + sessionId: this.session.relaySessionId, + callId, + result, + ...(options ? { options } : {}), + }); + } finally { + this.submittingToolCalls.delete(callId); + } + } + + private reportToolResultSubmissionError(error: unknown): void { + if (this.closed) { + return; + } + const message = error instanceof Error ? error.message : String(error); + this.lastRelayError = message; + this.ctx.callbacks.onStatus?.("error", message); } private completeToolCall(callIdRaw: string | undefined): void { @@ -331,6 +360,11 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport return; } this.completedToolCalls.add(callId); + // The Gateway broadcasts acceptance before resolving the matching RPC. + // Do not turn our own accepted result into a late consult cancellation. + if (this.submittingToolCalls.has(callId)) { + return; + } this.consultAbortControllers.get(callId)?.abort(); this.consultAbortControllers.delete(callId); } diff --git a/ui/src/pages/chat/realtime-talk-google-live.test.ts b/ui/src/pages/chat/realtime-talk-google-live.test.ts index 3f393ba3f967..9e3d7472f338 100644 --- a/ui/src/pages/chat/realtime-talk-google-live.test.ts +++ b/ui/src/pages/chat/realtime-talk-google-live.test.ts @@ -4,7 +4,10 @@ import { buildGoogleLiveUrl, GoogleLiveRealtimeTalkTransport, } from "./realtime-talk-google-live.ts"; -import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME } from "./realtime-talk-shared.ts"; +import { + REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, +} from "./realtime-talk-shared.ts"; import type { RealtimeTalkJsonPcmWebSocketSessionResult, RealtimeTalkTransportContext, @@ -495,6 +498,69 @@ describe("GoogleLiveRealtimeTalkTransport", () => { transport.stop(); }); + it("surfaces Google Live tool-result send failures without an unhandled rejection", async () => { + const onStatus = vi.fn(); + const onTalkEvent = vi.fn(); + const client = createClient(); + vi.mocked(client["request"]).mockImplementation(async (method) => { + if (method === "talk.client.steer") { + return { + ok: true, + mode: "status", + sessionKey: "main", + active: true, + message: "Still working.", + }; + } + throw new Error(`unexpected request: ${method}`); + }); + const transport = createTransport({ onStatus, onTalkEvent }, client); + + await transport.start(); + const ws = latestWebSocket(); + vi.spyOn(ws, "send").mockImplementation(() => { + throw new Error("Google Live socket rejected the tool result"); + }); + ws.emitMessage( + encodeJsonFrame({ + toolCall: { + functionCalls: [ + { + id: "call-control", + name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, + args: { text: "status", mode: "status" }, + }, + ], + }, + }), + ); + + await vi.waitFor(() => + expect(onStatus).toHaveBeenCalledWith("error", "Google Live socket rejected the tool result"), + ); + expect( + onTalkEvent.mock.calls.some( + ([event]) => + (event.type === "tool.progress" || event.type === "tool.error") && event.final === true, + ), + ).toBe(false); + expect( + ( + transport as unknown as { + pendingCalls: Map; + } + ).pendingCalls.has("call-control"), + ).toBe(true); + expect(() => + ( + transport as unknown as { + submitToolResult: (callId: string, result: unknown) => void; + } + ).submitToolResult("missing-call", { ok: true }), + ).toThrow("Google Live has no pending tool call for missing-call"); + transport.stop(); + }); + it("sends spoken active-control acknowledgements through Google Live", async () => { const client = createClient(); vi.mocked(client["request"]).mockImplementation(async (method) => { diff --git a/ui/src/pages/chat/realtime-talk-google-live.ts b/ui/src/pages/chat/realtime-talk-google-live.ts index a66296d76fdf..cee2adf3241e 100644 --- a/ui/src/pages/chat/realtime-talk-google-live.ts +++ b/ui/src/pages/chat/realtime-talk-google-live.ts @@ -213,10 +213,12 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { this.inputProcessor.connect(this.inputContext.destination); } - private send(message: unknown): void { + private send(message: unknown): boolean { if (!this.closed && this.ws?.readyState === WebSocket.OPEN) { this.ws.send(JSON.stringify(message)); + return true; } + return false; } private async handleMessage(data: unknown): Promise { @@ -310,7 +312,9 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { this.emitTalkEvent({ type: "turn.ended", final: true }); } for (const call of message.toolCall?.functionCalls ?? []) { - void this.handleToolCall(call); + void this.handleToolCall(call).catch((error: unknown) => { + this.reportToolResultSubmissionError(error); + }); } } @@ -393,10 +397,9 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { private submitToolResult(callId: string, result: unknown): void { const pending = this.pendingCalls.get(callId); if (!pending) { - return; + throw new Error(`Google Live has no pending tool call for ${callId}`); } - this.pendingCalls.delete(callId); - this.send({ + const sent = this.send({ toolResponse: { functionResponses: [ { @@ -411,6 +414,18 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { ], }, }); + if (!sent) { + throw new Error("Google Live socket is not open"); + } + this.pendingCalls.delete(callId); + } + + private reportToolResultSubmissionError(error: unknown): void { + if (this.closed) { + return; + } + const message = error instanceof Error ? error.message : String(error); + this.ctx.callbacks.onStatus?.("error", message); } private sendControlSpeechMessage(message: string): void { diff --git a/ui/src/pages/chat/realtime-talk-shared.ts b/ui/src/pages/chat/realtime-talk-shared.ts index 1c5a1cf47d9e..0d90ba8e8612 100644 --- a/ui/src/pages/chat/realtime-talk-shared.ts +++ b/ui/src/pages/chat/realtime-talk-shared.ts @@ -452,14 +452,16 @@ export async function steerRealtimeTalkActiveConsult(params: { export async function submitRealtimeTalkAgentControl(params: { ctx: RealtimeTalkTransportContext; args: unknown; - submit: (callId: string, result: unknown) => void; + submit: (callId: string, result: unknown) => void | Promise; callId: string; sessionId?: string; emitTalkEvent?: (input: RealtimeTalkEventInput) => void; }): Promise { + let result: unknown; + let talkEvent: RealtimeTalkEventInput; try { const parsed = parseRealtimeVoiceAgentControlToolArgs(params.args); - const result = + result = params.sessionId && params.sessionId.trim() ? await params.ctx.client.request("talk.session.steer", { sessionId: params.sessionId, @@ -472,7 +474,7 @@ export async function submitRealtimeTalkAgentControl(params: { text: parsed.text, mode: parsed.mode, }); - params.emitTalkEvent?.({ + talkEvent = { type: "tool.progress", callId: params.callId, payload: { @@ -483,18 +485,19 @@ export async function submitRealtimeTalkAgentControl(params: { result && typeof result === "object" && "mode" in result ? result.mode === "status" || result.mode === "cancel" : undefined, - }); - params.submit(params.callId, result); + }; } catch (error) { const message = error instanceof Error ? error.message : String(error); - params.emitTalkEvent?.({ + talkEvent = { type: "tool.error", callId: params.callId, payload: { message }, final: true, - }); - params.submit(params.callId, { error: message }); + }; + result = { error: message }; } + await params.submit(params.callId, result); + params.emitTalkEvent?.(talkEvent); } function maybeSpeakRealtimeTalkControlResult( @@ -523,7 +526,7 @@ function maybeSpeakRealtimeTalkControlResult( export async function submitRealtimeTalkConsult(params: { ctx: RealtimeTalkTransportContext; args: unknown; - submit: (callId: string, result: unknown) => void; + submit: (callId: string, result: unknown) => void | Promise; callId: string; relaySessionId?: string; emitTalkEvent?: (input: RealtimeTalkEventInput) => void; @@ -535,16 +538,18 @@ export async function submitRealtimeTalkConsult(params: { let runId: string | undefined; let aborted = false; let submitted = false; - const submitOnce = (result: unknown) => { + let submissionCompleted = false; + const submitOnce = async (result: unknown): Promise => { if (submitted) { return; } submitted = true; - submit(callId, result); + await submit(callId, result); + submissionCompleted = true; }; - const submitAbortResult = () => { + const submitAbortResult = async (): Promise => { if (params.submitAbortResult !== false) { - submitOnce(buildRealtimeVoiceAgentCancelProviderResult()); + await submitOnce(buildRealtimeVoiceAgentCancelProviderResult()); } }; const abortRun = () => { @@ -554,7 +559,7 @@ export async function submitRealtimeTalkConsult(params: { } }; if (params.signal?.aborted) { - submitAbortResult(); + await submitAbortResult(); return; } params.signal?.addEventListener("abort", abortRun, { once: true }); @@ -577,7 +582,7 @@ export async function submitRealtimeTalkConsult(params: { } if (params.signal?.aborted) { abortRun(); - submitAbortResult(); + await submitAbortResult(); return; } const result = await waitForChatResult({ @@ -587,18 +592,21 @@ export async function submitRealtimeTalkConsult(params: { emitTalkEvent: params.emitTalkEvent, signal: params.signal, }); - submitOnce({ result }); + await submitOnce({ result }); } catch (error) { + if (submitted) { + throw error; + } if (aborted || params.signal?.aborted || isAbortError(error)) { - submitAbortResult(); + await submitAbortResult(); return; } - submitOnce({ + await submitOnce({ error: error instanceof Error ? error.message : String(error), }); } finally { params.signal?.removeEventListener("abort", abortRun); - if (!aborted && !params.signal?.aborted) { + if (submissionCompleted && !aborted && !params.signal?.aborted) { ctx.callbacks.onStatus?.("listening"); } } diff --git a/ui/src/pages/chat/realtime-talk-webrtc.test.ts b/ui/src/pages/chat/realtime-talk-webrtc.test.ts index 99dafb43b1cf..c2c7e1d3c953 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.test.ts @@ -896,4 +896,50 @@ describe("WebRtcSdpRealtimeTalkTransport", () => { }); transport.stop(); }); + + it("surfaces OpenAI tool-result send failures without an unhandled rejection", async () => { + stubAnswerSdpFetch(); + const onStatus = vi.fn(); + const onTalkEvent = vi.fn(); + const request = vi.fn(async (method: string) => { + if (method === "talk.client.steer") { + return { + ok: true, + mode: "status", + sessionKey: "main", + active: true, + message: "Still working.", + }; + } + throw new Error(`unexpected request: ${method}`); + }); + const transport = createOpenAiTransport({ request }, { onStatus, onTalkEvent }); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + peer?.channel.send.mockImplementation(() => { + throw new Error("OpenAI data channel rejected the tool result"); + }); + dispatchRealtimeEvent(peer, { + type: "response.function_call_arguments.done", + item_id: "item-control", + call_id: "call-control", + name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, + arguments: JSON.stringify({ text: "status", mode: "status" }), + }); + + await vi.waitFor(() => + expect(onStatus).toHaveBeenCalledWith( + "error", + "OpenAI data channel rejected the tool result", + ), + ); + expect( + onTalkEvent.mock.calls.some( + ([event]) => + (event.type === "tool.progress" || event.type === "tool.error") && event.final === true, + ), + ).toBe(false); + transport.stop(); + }); }); diff --git a/ui/src/pages/chat/realtime-talk-webrtc.ts b/ui/src/pages/chat/realtime-talk-webrtc.ts index 1a644c6a1e61..b3bc133c9806 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.ts @@ -261,7 +261,9 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { this.bufferToolDelta(event); return; case "response.function_call_arguments.done": - void this.handleToolCall(event); + void this.handleToolCall(event).catch((error: unknown) => { + this.reportToolResultSubmissionError(error); + }); return; case "input_audio_buffer.speech_started": this.ctx.callbacks.onStatus?.("listening", "Speech detected"); @@ -409,6 +411,14 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { this.requestResponseCreate(); } + private reportToolResultSubmissionError(error: unknown): void { + if (this.closed) { + return; + } + const message = error instanceof Error ? error.message : String(error); + this.ctx.callbacks.onStatus?.("error", message); + } + private sendControlSpeechMessage(message: string): void { if (this.responseActive) { this.send({ type: "response.cancel" });