diff --git a/extensions/discord/src/voice/manager.e2e.test-support.ts b/extensions/discord/src/voice/manager.e2e.test-support.ts index de0c39d76515..736fdf97df96 100644 --- a/extensions/discord/src/voice/manager.e2e.test-support.ts +++ b/extensions/discord/src/voice/manager.e2e.test-support.ts @@ -44,7 +44,8 @@ export type TestRealtimeBridgeParams = { cfg?: unknown; instructions?: string; interruptResponseOnInputAudio?: boolean; - onEvent?: (event: { detail?: string; direction: "server"; type: string }) => void; + onEvent?: (event: { detail?: string; direction: "client" | "server"; type: string }) => void; + onReady?: () => void; onToolCall?: ( event: { args: unknown; callId: string; itemId: string; name: string }, session: unknown, diff --git a/extensions/discord/src/voice/manager.e2e.test.ts b/extensions/discord/src/voice/manager.e2e.test.ts index 3effe9bb3921..8bbf1d9b581b 100644 --- a/extensions/discord/src/voice/manager.e2e.test.ts +++ b/extensions/discord/src/voice/manager.e2e.test.ts @@ -3263,6 +3263,98 @@ describe("DiscordVoiceManager", () => { expect(player.stop).toHaveBeenCalledTimes(stopCallsAfterControl + 1); }); + it("drops stale active-run control after provider continuity reset", async () => { + let resolveOldControl: ((result: RealtimeVoiceAgentControlResult) => void) | undefined; + controlRealtimeVoiceAgentRunMock + .mockReturnValueOnce( + new Promise((resolve) => { + resolveOldControl = resolve; + }), + ) + .mockResolvedValueOnce({ + ok: true, + mode: "cancel", + sessionKey: "discord:g1:c1", + sessionId: "embedded-fresh", + active: true, + aborted: true, + message: "Fresh control result.", + speak: true, + show: true, + suppress: false, + }); + const manager = createAgentProxyManager(); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const bridgeParams = lastRealtimeBridgeParams(); + bridgeParams?.onTranscript?.("user", "cancel that", true); + await vi.waitFor(() => expect(controlRealtimeVoiceAgentRunMock).toHaveBeenCalledTimes(1)); + + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + resolveOldControl?.({ + ok: true, + mode: "cancel", + sessionKey: "discord:g1:c1", + sessionId: "embedded-old", + active: true, + aborted: true, + message: "Stale control result.", + speak: true, + show: true, + suppress: false, + }); + await Promise.resolve(); + await Promise.resolve(); + + expectUserMessageNotIncludes("Stale control result."); + expect(realtimeSessionMock.handleBargeIn).not.toHaveBeenCalled(); + + bridgeParams?.onReady?.(); + bridgeParams?.onTranscript?.("user", "stop that", true); + await vi.waitFor(() => expectUserMessageIncludes("Fresh control result.")); + expect(realtimeSessionMock.handleBargeIn).toHaveBeenCalledTimes(1); + }); + + it("replaces stale talkback work across provider continuity reset", async () => { + let resolveOldTalkback: ((result: { payloads: Array<{ text: string }> }) => void) | undefined; + agentCommandMock + .mockReturnValueOnce( + new Promise((resolve) => { + resolveOldTalkback = resolve; + }), + ) + .mockResolvedValueOnce({ payloads: [{ text: "fresh talkback" }] }); + const manager = createManager({ + groupPolicy: "open", + voice: { + enabled: true, + mode: "agent-proxy", + realtime: { provider: "openai", debounceMs: 1, toolPolicy: "none" }, + }, + }); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const entry = getSessionEntry(manager); + const bridgeParams = lastRealtimeBridgeParams(); + beginSpeakerTurn(entry); + await emitFinalRealtimeUserTranscript(bridgeParams, "old question"); + await vi.waitFor(() => expect(agentCommandMock).toHaveBeenCalledTimes(1)); + + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onReady?.(); + beginSpeakerTurn(entry); + await emitFinalRealtimeUserTranscript(bridgeParams, "fresh question"); + + await vi.waitFor(() => expect(agentCommandMock).toHaveBeenCalledTimes(2)); + await vi.waitFor(() => expectUserMessageIncludes("fresh talkback")); + resolveOldTalkback?.({ payloads: [{ text: "stale talkback" }] }); + await Promise.resolve(); + await Promise.resolve(); + expectUserMessageNotIncludes("stale talkback"); + }); + it("preserves realtime forced consults when no active run accepts steering", async () => { agentCommandMock.mockResolvedValueOnce({ payloads: [{ text: "normal answer" }] }); const manager = createAgentProxyManager(); @@ -3418,6 +3510,203 @@ describe("DiscordVoiceManager", () => { expectUserMessageIncludes("wake answer"); }); + it("does not carry partial wake-name state across provider continuity resets", async () => { + const { entry, bridgeParams } = await createWakeNameFixture(); + const wakeAckCount = () => + sentUserMessages().filter((message) => message.includes('Answer: "Yeah."')).length; + + beginSpeakerTurn(entry); + bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" }); + bridgeParams?.onTranscript?.("user", "Hey, Mol", false); + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onTranscript?.("user", "ty", false); + + expect(wakeAckCount()).toBe(0); + + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onReady?.(); + bridgeParams?.onTranscript?.("user", "Hey, Molty", false); + expect(wakeAckCount()).toBe(1); + bridgeParams?.onEvent?.({ direction: "server", type: "response.done" }); + }); + + it("preserves the wake-name acknowledgement across provider continuity resets", async () => { + const { entry, bridgeParams } = await createWakeNameFixture(); + const wakeAckCount = () => + sentUserMessages().filter((message) => message.includes('Answer: "')).length; + + beginSpeakerTurn(entry); + bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" }); + bridgeParams?.onTranscript?.("user", "Hey, Molty", false); + expect(wakeAckCount()).toBe(1); + bridgeParams?.onEvent?.({ direction: "server", type: "response.done" }); + + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onReady?.(); + bridgeParams?.onTranscript?.("user", "Hey, Molty", false); + expect(wakeAckCount()).toBe(1); + + bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" }); + bridgeParams?.onTranscript?.("user", "Hey, Molty", false); + expect(wakeAckCount()).toBe(2); + }); + + it("replays zero-audio exact speech once after provider continuity reset", async () => { + agentCommandMock + .mockResolvedValueOnce({ payloads: [{ text: "first answer" }] }) + .mockResolvedValueOnce({ payloads: [{ text: "second answer" }] }); + const manager = createAgentProxyManager(); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const entry = getSessionEntry(manager); + const player = getLastAudioPlayer(); + const bridgeParams = lastRealtimeBridgeParams(); + + beginSpeakerTurn(entry); + await emitFinalRealtimeUserTranscript(bridgeParams, "first question"); + await vi.waitFor(() => expectUserMessageIncludes("first answer")); + beginSpeakerTurn(entry); + await emitFinalRealtimeUserTranscript(bridgeParams, "second question"); + expectUserMessageNotIncludes("second answer"); + + const stopCallsBeforeReset = player.stop.mock.calls.length; + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + expectUserMessageNotIncludes("second answer"); + expect(player.stop).toHaveBeenCalledTimes(stopCallsBeforeReset + 1); + expect(realtimeSessionMock.handleBargeIn).not.toHaveBeenCalled(); + expect(realtimeSessionMock.close).not.toHaveBeenCalled(); + + bridgeParams?.onReady?.(); + expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength( + 2, + ); + expectUserMessageNotIncludes("second answer"); + bridgeParams?.onEvent?.({ direction: "server", type: "response.done" }); + expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength( + 1, + ); + }); + + it("replays exact speech buffered below playback preroll after continuity reset", async () => { + agentCommandMock + .mockResolvedValueOnce({ payloads: [{ text: "first answer" }] }) + .mockResolvedValueOnce({ payloads: [{ text: "second answer" }] }); + const manager = createAgentProxyManager(); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const entry = getSessionEntry(manager); + const player = getLastAudioPlayer(); + const bridgeParams = lastRealtimeBridgeParams(); + + beginSpeakerTurn(entry); + await emitFinalRealtimeUserTranscript(bridgeParams, "first question"); + await vi.waitFor(() => expectUserMessageIncludes("first answer")); + bridgeParams?.audioSink?.sendAudio(Buffer.alloc(480)); + expect(player.play).not.toHaveBeenCalled(); + beginSpeakerTurn(entry); + await emitFinalRealtimeUserTranscript(bridgeParams, "second question"); + expectUserMessageNotIncludes("second answer"); + + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onReady?.(); + + expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength( + 2, + ); + expectUserMessageNotIncludes("second answer"); + bridgeParams?.onEvent?.({ direction: "server", type: "response.done" }); + expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength( + 1, + ); + }); + + it("does not replay exact speech after Discord playback starts", async () => { + agentCommandMock + .mockResolvedValueOnce({ payloads: [{ text: "first answer" }] }) + .mockResolvedValueOnce({ payloads: [{ text: "second answer" }] }); + const manager = createAgentProxyManager(); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const entry = getSessionEntry(manager); + const player = getLastAudioPlayer(); + const bridgeParams = lastRealtimeBridgeParams(); + + beginSpeakerTurn(entry); + await emitFinalRealtimeUserTranscript(bridgeParams, "first question"); + await vi.waitFor(() => expectUserMessageIncludes("first answer")); + for (let index = 0; index < 50; index += 1) { + bridgeParams?.audioSink?.sendAudio(Buffer.alloc(480)); + } + expect(player.play).toHaveBeenCalledOnce(); + beginSpeakerTurn(entry); + await emitFinalRealtimeUserTranscript(bridgeParams, "second question"); + expectUserMessageNotIncludes("second answer"); + + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + bridgeParams?.onReady?.(); + + expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength( + 1, + ); + expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength( + 1, + ); + }); + + it("drops stale native consult delivery after provider continuity reset", async () => { + let resolveOld: ((result: { payloads: Array<{ text: string }> }) => void) | undefined; + agentCommandMock + .mockReturnValueOnce( + new Promise((resolve) => { + resolveOld = resolve; + }), + ) + .mockResolvedValueOnce({ payloads: [{ text: "fresh answer" }] }); + const manager = createAgentProxyManager(); + + await manager.join({ guildId: "g1", channelId: "1001" }); + const entry = getSessionEntry(manager); + const bridgeParams = lastRealtimeBridgeParams(); + beginSpeakerTurn(entry); + const oldSubmission = bridgeParams?.onToolCall?.( + { + itemId: "item-old", + callId: "call-old", + name: "openclaw_agent_consult", + args: { question: "same question" }, + }, + realtimeSessionMock, + ); + await Promise.resolve(); + + bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" }); + resolveOld?.({ payloads: [{ text: "stale answer" }] }); + await oldSubmission; + expect( + realtimeSessionMock.submitToolResult.mock.calls.some(([callId]) => callId === "call-old"), + ).toBe(false); + + bridgeParams?.onReady?.(); + beginSpeakerTurn(entry); + await bridgeParams?.onToolCall?.( + { + itemId: "item-fresh", + callId: "call-fresh", + name: "openclaw_agent_consult", + args: { question: "same question" }, + }, + realtimeSessionMock, + ); + expect(realtimeSessionMock.submitToolResult).toHaveBeenCalledWith("call-fresh", { + text: "fresh answer", + }); + }); + it("treats a bare wake name as an activation for the next realtime transcript", async () => { agentCommandMock.mockResolvedValueOnce({ payloads: [{ text: "follow-up answer" }] }); const onUtterance = vi.fn(); diff --git a/extensions/discord/src/voice/realtime.ts b/extensions/discord/src/voice/realtime.ts index ade1b709766c..67ff7ba01a16 100644 --- a/extensions/discord/src/voice/realtime.ts +++ b/extensions/discord/src/voice/realtime.ts @@ -10,6 +10,7 @@ import { buildRealtimeVoiceAgentConsultPolicyInstructions, classifySkippableRealtimeVoiceConsultTranscript, controlRealtimeVoiceAgentRun, + createRealtimeVoiceAgentTalkbackQueue, createRealtimeVoiceSessionHarness, createRealtimeVoiceTurnContextTracker, matchRealtimeVoiceActivationName, @@ -26,6 +27,7 @@ import { type RealtimeVoiceBridgeEvent, type RealtimeVoiceAgentConsultToolPolicy, type RealtimeVoiceAgentControlResult, + type RealtimeVoiceAgentTalkbackQueue, type RealtimeVoiceBridgeSession, type RealtimeVoiceProviderConfig, type RealtimeVoiceToolCallEvent, @@ -133,8 +135,10 @@ type RecentAgentProxyConsultResult = type AgentProxyConsultState = { speaker: DiscordRealtimeSpeakerContext; + providerEpoch: number; handledByForcedPlayback?: boolean; providerDelivery?: Promise; + settleProviderDelivery?: (accepted: boolean) => void; promise?: Promise; result?: RecentAgentProxyConsultResult; }; @@ -324,6 +328,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { private bridge: RealtimeVoiceBridgeSession | null = null; private outputStream: PassThrough | null = null; private readonly harness: RealtimeVoiceSessionHarness; + private talkback: RealtimeVoiceAgentTalkbackQueue; private stopped = false; private consultToolPolicy: RealtimeVoiceAgentConsultToolPolicy = "safe-read-only"; private consultToolsAllow: string[] | undefined; @@ -347,6 +352,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { private queuedExactSpeechMessages: string[] = []; private exactSpeechResponseActive = false; private exactSpeechAudioStarted = false; + private activeExactSpeechMessage: string | undefined; + private bridgeReady = false; + private providerGenerationObserved = false; + private providerContinuityEpoch = 0; private partialUserTranscript = ""; private wakeNameAckedForTurn = false; private wakeNameAckIndex = 0; @@ -397,34 +406,40 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { outputAudioDelta: discordRealtimeTalkPayload, outputAudioDone: discordRealtimeTalkPayload, }, - talkback: { - debounceMs: this.realtimeConfig?.debounceMs ?? DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS, - logger, - logPrefix: "[discord] realtime agent", - responseStyle: "Brief, natural spoken answer for a Discord voice channel.", - fallbackText: DISCORD_REALTIME_FALLBACK_TEXT, - consult: async ({ question, responseStyle, metadata }) => { - const context = isDiscordRealtimeSpeakerContext(metadata) ? metadata : undefined; - return { - text: await this.runAgentTurn({ - context, - message: formatVoiceIngressPrompt( - [question, responseStyle ? `Spoken style: ${responseStyle}` : undefined] - .filter(Boolean) - .join("\n\n"), - context?.speakerLabel ?? "Discord voice speaker", - ), - }), - }; - }, - deliver: (text) => this.enqueueExactSpeechMessage(text), - }, forcedConsults: { limit: DISCORD_REALTIME_RECENT_AGENT_PROXY_CONSULT_LIMIT, nativeDedupeMs: DISCORD_REALTIME_RECENT_AGENT_PROXY_CONSULT_TTL_MS, questionsMatch: matchRealtimeVoiceConsultQuestions, }, }); + this.talkback = this.createTalkbackQueue(); + } + + private createTalkbackQueue(): RealtimeVoiceAgentTalkbackQueue { + const providerEpoch = this.providerContinuityEpoch; + return createRealtimeVoiceAgentTalkbackQueue({ + debounceMs: this.realtimeConfig?.debounceMs ?? DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS, + isStopped: () => this.stopped || providerEpoch !== this.providerContinuityEpoch, + logger, + logPrefix: "[discord] realtime agent", + responseStyle: "Brief, natural spoken answer for a Discord voice channel.", + fallbackText: DISCORD_REALTIME_FALLBACK_TEXT, + consult: async ({ question, responseStyle, metadata }) => { + const context = isDiscordRealtimeSpeakerContext(metadata) ? metadata : undefined; + return { + text: await this.runAgentTurn({ + context, + message: formatVoiceIngressPrompt( + [question, responseStyle ? `Spoken style: ${responseStyle}` : undefined] + .filter(Boolean) + .join("\n\n"), + context?.speakerLabel ?? "Discord voice speaker", + ), + }), + }; + }, + deliver: (text) => this.enqueueExactSpeechMessage(text), + }); } async connect(): Promise { @@ -493,10 +508,14 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { audioSink: { isOpen: () => !this.stopped, sendAudio: (audio) => this.sendOutputAudio(audio), - clearAudio: () => - this.harness.flushOutput(() => this.clearOutputAudio("provider-clear-audio")), + clearAudio: () => { + this.markProviderGenerationObserved(); + this.harness.flushOutput(() => this.clearOutputAudio("provider-clear-audio")); + }, }, onTranscript: (role, text, isFinal) => { + this.markProviderGenerationObserved(); + const providerEpoch = this.providerContinuityEpoch; if (isFinal && text.trim()) { logger.info( `discord voice: realtime ${role} transcript (${text.length} chars): ${formatVoiceLogPreview(text)}`, @@ -512,11 +531,28 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { this.handlePartialUserTranscript(text); return; } - void this.handleFinalUserTranscript(text, { usesRealtimeAgentHandoff }); + void this.handleFinalUserTranscript(text, { + providerEpoch, + usesRealtimeAgentHandoff, + }); + }, + onToolCall: (event, session) => { + this.markProviderGenerationObserved(); + return this.handleToolCall(event, session); + }, + onReady: () => { + this.markProviderGenerationObserved(); + this.bridgeReady = true; + this.drainQueuedExactSpeechMessages("provider-ready"); }, - onToolCall: (event, session) => this.handleToolCall(event, session), onEvent: (event) => { + if (!(event.direction === "client" && event.type === "session.continuity.reset")) { + this.markProviderGenerationObserved(); + } const detail = event.detail ? ` ${event.detail}` : ""; + if (event.direction === "client" && event.type === "session.continuity.reset") { + this.resetProviderContinuity(event.type); + } if (event.direction === "server" && event.type === "input_audio_buffer.speech_started") { this.resetPartialWakeNameTracking(); } @@ -571,6 +607,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { const voiceSdk = loadDiscordVoiceSdk(); this.params.entry.player.on(voiceSdk.AudioPlayerStatus.Idle, this.playerIdleHandler); await this.bridge.connect(); + // Some provider/test bridges do not expose an explicit ready callback. + this.markProviderGenerationObserved(); + this.bridgeReady = true; + this.drainQueuedExactSpeechMessages("provider-connected"); logger.info( `discord voice: realtime bridge ready mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"}`, ); @@ -578,13 +618,18 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { close(): void { this.stopped = true; + this.bridgeReady = false; + this.providerContinuityEpoch += 1; this.outputBackpressure = undefined; this.flushSuppressedRealtimeErrors(); + this.clearProviderConsultState(); + this.talkback.close(); this.harness.close(); this.speakerTurns.clear(); this.queuedExactSpeechMessages = []; this.exactSpeechResponseActive = false; this.exactSpeechAudioStarted = false; + this.activeExactSpeechMessage = undefined; this.resetPartialWakeNameTracking(); this.pendingWakeNameFollowup = undefined; this.clearOutputAudio("session-close"); @@ -733,6 +778,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { } private sendOutputAudio(realtimePcm24kMono: Buffer): void { + this.markProviderGenerationObserved(); if (this.stopped || this.outputBackpressure) { return; } @@ -948,7 +994,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { if (this.stopped || !text.trim()) { return; } - if (this.exactSpeechResponseActive || this.hasInterruptibleOutputAudio()) { + if (!this.bridgeReady || this.exactSpeechResponseActive || this.hasInterruptibleOutputAudio()) { this.queuedExactSpeechMessages.push(text); logger.info( `discord voice: realtime exact speech queued guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()}`, @@ -964,6 +1010,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { } this.exactSpeechResponseActive = true; this.exactSpeechAudioStarted = false; + this.activeExactSpeechMessage = text; this.bridge?.sendUserMessage(buildDiscordSpeakExactUserMessage(text)); } @@ -983,7 +1030,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { logger.info( `discord voice: realtime wake-name ack canonical=${result.activationName} heard=${result.heardName} match=${result.match} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`, ); - this.sendExactSpeechMessage(ack ?? "Yeah."); + this.enqueueExactSpeechMessage(ack ?? "Yeah."); } private speakControlResult(text: string): void { @@ -1001,7 +1048,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { sentAt: Date.now(), assistantTranscriptCount: 0, }; - this.sendExactSpeechMessage(trimmed); + this.enqueueExactSpeechMessage(trimmed); } private suppressDuplicateControlSpeech(text: string): void { @@ -1034,6 +1081,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { } this.exactSpeechResponseActive = false; this.exactSpeechAudioStarted = false; + this.activeExactSpeechMessage = undefined; if (options?.drain === false) { return; } @@ -1043,6 +1091,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { private drainQueuedExactSpeechMessages(reason: string): void { if ( this.stopped || + !this.bridgeReady || this.exactSpeechResponseActive || this.queuedExactSpeechMessages.length === 0 || this.hasInterruptibleOutputAudio() @@ -1123,9 +1172,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { event: RealtimeVoiceToolCallEvent, session: RealtimeVoiceBridgeSession, ): Promise { + const providerEpoch = this.providerContinuityEpoch; const callId = event.callId || event.itemId || "unknown"; if (event.name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) { - await this.handleAgentControlToolCall(event, session, callId); + await this.handleAgentControlToolCall(event, session, callId, providerEpoch); return; } if (event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { @@ -1223,11 +1273,17 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { try { text = await promise; } catch (error) { + if (providerEpoch !== this.providerContinuityEpoch) { + return; + } const message = formatErrorMessage(error); logger.warn(`discord voice: realtime consult failed call=${callId || "unknown"}: ${message}`); await session.submitToolResult(callId, { error: message }); return; } + if (providerEpoch !== this.providerContinuityEpoch) { + return; + } logger.info( `discord voice: realtime consult answer (${text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(text)}`, ); @@ -1238,6 +1294,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { event: RealtimeVoiceToolCallEvent, session: RealtimeVoiceBridgeSession, callId: string, + providerEpoch: number, ): Promise { let result: RealtimeVoiceAgentControlResult; try { @@ -1248,9 +1305,15 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { mode: parsed.mode, }); } catch (error) { + if (providerEpoch !== this.providerContinuityEpoch) { + return; + } await session.submitToolResult(callId, { error: formatErrorMessage(error) }); return; } + if (providerEpoch !== this.providerContinuityEpoch) { + return; + } this.logAgentControlResult(result); await session.submitToolResult(callId, result); } @@ -1273,7 +1336,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { private async handleFinalUserTranscript( text: string, - params: { usesRealtimeAgentHandoff: boolean }, + params: { providerEpoch: number; usesRealtimeAgentHandoff: boolean }, ): Promise { const trimmed = text.trim(); if (!trimmed) { @@ -1316,15 +1379,24 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { usesAgentProxy && params.usesRealtimeAgentHandoff ? this.prepareForcedAgentProxyConsult(acceptedText, forcedSpeakerContext) : undefined; - const control = await maybeControlDiscordVoiceAgentRun({ - entry: this.params.entry, - text: acceptedText, - }).catch((error: unknown) => { + let control: Awaited> | undefined; + try { + control = await maybeControlDiscordVoiceAgentRun({ + entry: this.params.entry, + text: acceptedText, + }); + } catch (error) { + if (params.providerEpoch !== this.providerContinuityEpoch) { + return; + } logger.warn( `discord voice: realtime active-run control failed; falling back to normal transcript handling: ${formatErrorMessage(error)}`, ); - return undefined; - }); + control = undefined; + } + if (params.providerEpoch !== this.providerContinuityEpoch) { + return; + } if (control?.handled) { if (pendingForcedConsult) { this.harness.forcedConsults.remove(pendingForcedConsult); @@ -1345,7 +1417,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { } return; } - this.harness.talkback?.enqueue( + this.talkback.enqueue( acceptedText, forcedSpeakerContext ?? this.consumePendingSpeakerContext(), ); @@ -1372,6 +1444,52 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { this.wakeNameAckedForTurn = false; } + private markProviderGenerationObserved(): void { + this.providerGenerationObserved = true; + } + + private resetProviderContinuity(reason: string): void { + if (!this.providerGenerationObserved) { + return; + } + this.providerGenerationObserved = false; + this.bridgeReady = false; + this.providerContinuityEpoch += 1; + this.talkback.close(); + this.talkback = this.createTalkbackQueue(); + this.outputBackpressure = undefined; + this.partialUserTranscript = ""; + this.pendingWakeNameFollowup = undefined; + this.lastControlSpeech = undefined; + this.clearProviderConsultState(); + const replayExactSpeech = + this.exactSpeechResponseActive && !this.harness.outputActivity.snapshot().playbackStarted + ? this.activeExactSpeechMessage + : undefined; + this.exactSpeechResponseActive = false; + this.exactSpeechAudioStarted = false; + this.activeExactSpeechMessage = undefined; + if (replayExactSpeech) { + this.queuedExactSpeechMessages.unshift(replayExactSpeech); + } + this.harness.flushOutput(() => this.clearOutputAudio(reason)); + this.harness.finishOutputAudio(reason); + } + + private clearProviderConsultState(): void { + for (const handle of this.harness.forcedConsults.handles()) { + const state = handle.context; + if (!state) { + continue; + } + state.handledByForcedPlayback = false; + state.settleProviderDelivery?.(false); + state.settleProviderDelivery = undefined; + state.providerDelivery = undefined; + } + this.harness.forcedConsults.clear(); + } + private resolveWakeNameTranscript( text: string, requireWakeName: boolean, @@ -1477,7 +1595,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { return undefined; } return this.harness.forcedConsults.prepare(question, { - context: { speaker: context }, + context: { speaker: context, providerEpoch: this.providerContinuityEpoch }, }); } @@ -1498,7 +1616,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { } const context = state.speaker; const { question } = pending; - if (this.stopped) { + if (this.stopped || state.providerEpoch !== this.providerContinuityEpoch) { this.harness.forcedConsults.markCancelled(pending); return; } @@ -1523,6 +1641,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { this.setRecentAgentProxyConsultPromise(pending, promise); const text = await promise; await state.providerDelivery; + if (state.providerEpoch !== this.providerContinuityEpoch) { + return; + } logger.info( `discord voice: realtime forced agent consult answer (${text.length} chars) elapsedMs=${Date.now() - startedAt} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}: ${formatVoiceLogPreview(text)}`, ); @@ -1531,6 +1652,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { } } catch (error) { await state.providerDelivery; + if (state.providerEpoch !== this.providerContinuityEpoch) { + return; + } logger.warn( `discord voice: realtime forced agent consult failed elapsedMs=${Date.now() - startedAt}: ${formatErrorMessage(error)}`, ); @@ -1612,7 +1736,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { options: { id?: string; started?: boolean } = {}, ): AgentProxyConsultHandle { const handle = this.harness.forcedConsults.prepare(question, { - context: { speaker: context }, + context: { speaker: context, providerEpoch: this.providerContinuityEpoch }, ...(options.id ? { id: options.id } : {}), }); if (!handle) { @@ -1636,10 +1760,16 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { state.promise = promise; void promise .then((text) => { + if (state.providerEpoch !== this.providerContinuityEpoch) { + return; + } state.result = { status: "fulfilled", text }; this.harness.forcedConsults.markDelivered(recent); }) .catch((error: unknown) => { + if (state.providerEpoch !== this.providerContinuityEpoch) { + return; + } state.result = { status: "rejected", error: formatErrorMessage(error) }; this.harness.forcedConsults.markDelivered(recent); }); @@ -1674,6 +1804,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { if (!state) { return false; } + if (state.providerEpoch !== this.providerContinuityEpoch) { + return true; + } const providerOwnsDelivery = Boolean( state.handledByForcedPlayback && state.promise && @@ -1686,15 +1819,22 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { // the local success/fallback path instead of losing the answer entirely. state.providerDelivery = new Promise((resolve) => { resolveProviderDelivery = resolve; + state.settleProviderDelivery = resolve; }); } const submitAlreadyDelivered = async (): Promise => { + if (state.providerEpoch !== this.providerContinuityEpoch) { + return; + } await this.submitTerminalRealtimeToolResult(callId, session, { status: "already_delivered", message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it.", }); }; const submitResult = async (result: RecentAgentProxyConsultResult): Promise => { + if (state.providerEpoch !== this.providerContinuityEpoch) { + return; + } if (state.handledByForcedPlayback && !providerOwnsDelivery) { await submitAlreadyDelivered(); return; @@ -1720,6 +1860,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { ); if (state.handledByForcedPlayback && !providerOwnsDelivery) { await state.promise.catch(() => undefined); + if (state.providerEpoch !== this.providerContinuityEpoch) { + return true; + } await submitAlreadyDelivered(); return true; } @@ -1729,13 +1872,18 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession { } catch (error) { result = { status: "rejected", error: formatErrorMessage(error) }; } + if (state.providerEpoch !== this.providerContinuityEpoch) { + return true; + } try { await submitResult(result); if (providerOwnsDelivery) { state.handledByForcedPlayback = false; + state.settleProviderDelivery = undefined; resolveProviderDelivery?.(true); } } catch (error) { + state.settleProviderDelivery = undefined; resolveProviderDelivery?.(false); throw error; } diff --git a/extensions/google/realtime-voice-provider.test.ts b/extensions/google/realtime-voice-provider.test.ts index 4f4be238e5b4..7257aa57b7f0 100644 --- a/extensions/google/realtime-voice-provider.test.ts +++ b/extensions/google/realtime-voice-provider.test.ts @@ -815,6 +815,9 @@ describe("buildGoogleRealtimeVoiceProvider", () => { sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" }, serverContent: { inputTranscription: { text: "Before " } }, }); + firstSession.onmessage({ + sessionResumptionUpdate: { newHandle: "unconfirmed-handle" }, + }); firstSession.onclose({ code: 1011, reason: "temporary" }); await vi.advanceTimersByTimeAsync(250); lastConnectParams().callbacks.onmessage({ @@ -827,6 +830,46 @@ describe("buildGoogleRealtimeVoiceProvider", () => { ]); }); + it("preserves continuity when resumability recovers before reconnect", async () => { + vi.useFakeTimers(); + const provider = buildGoogleRealtimeVoiceProvider(); + const onEvent = vi.fn(); + const onTranscript = vi.fn(); + const bridge = provider.createBridge({ + providerConfig: { apiKey: "gemini-key" }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), + onEvent, + onTranscript, + }); + + await bridge.connect(); + const firstSession = lastConnectParams().callbacks; + firstSession.onmessage({ + sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" }, + serverContent: { inputTranscription: { text: "Before " } }, + }); + firstSession.onmessage({ + sessionResumptionUpdate: { resumable: false }, + }); + firstSession.onmessage({ + sessionResumptionUpdate: { resumable: true, newHandle: "resume-2" }, + }); + expect(onEvent).not.toHaveBeenCalled(); + + firstSession.onclose({ code: 1011, reason: "temporary" }); + await vi.advanceTimersByTimeAsync(250); + lastConnectParams().callbacks.onmessage({ + serverContent: { inputTranscription: { text: "after", finished: true } }, + }); + + expect(lastConnectParams().config.sessionResumption).toEqual({ handle: "resume-2" }); + expect(onEvent).not.toHaveBeenCalled(); + expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([ + ["user", "Before after", true], + ]); + }); + it("drops unfinished hypotheses when a new session has no continuity", async () => { vi.useFakeTimers(); const provider = buildGoogleRealtimeVoiceProvider(); @@ -854,28 +897,37 @@ describe("buildGoogleRealtimeVoiceProvider", () => { ]); }); - it("reconnects unexpected Google Live closes with the latest resumption handle", async () => { + it.each([undefined, "invalidated-handle"])("invalidates resume handles %#", async (newHandle) => { vi.useFakeTimers(); try { const provider = buildGoogleRealtimeVoiceProvider(); const onClose = vi.fn(); const onError = vi.fn(); + const onEvent = vi.fn(); + const onTranscript = vi.fn(); const bridge = provider.createBridge({ providerConfig: { apiKey: "gemini-key" }, onAudio: vi.fn(), onClearAudio: vi.fn(), onClose, onError, + onEvent, + onTranscript, }); await bridge.connect(); lastConnectParams().callbacks.onmessage({ setupComplete: { sessionId: "session-1" }, sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" }, + serverContent: { + inputTranscription: { text: "Previous caller " }, + outputTranscription: { text: "Previous assistant " }, + }, }); lastConnectParams().callbacks.onmessage({ - sessionResumptionUpdate: { resumable: false }, + sessionResumptionUpdate: { resumable: false, ...(newHandle ? { newHandle } : {}) }, }); + expect(onEvent).not.toHaveBeenCalled(); lastConnectParams().callbacks.onclose({ code: 1011, reason: "temporary upstream close", @@ -883,13 +935,38 @@ describe("buildGoogleRealtimeVoiceProvider", () => { }); expect(onClose).not.toHaveBeenCalled(); + expect(onEvent).toHaveBeenCalledOnce(); + expect(onEvent).toHaveBeenCalledWith({ + direction: "client", + type: "session.continuity.reset", + }); const error = requireFirstError(onError); expect(error.message).toContain("reconnecting 1/3"); await vi.advanceTimersByTimeAsync(250); expect(connectMock).toHaveBeenCalledTimes(2); - expect(lastConnectParams().config.sessionResumption).toEqual({ handle: "resume-1" }); + expect(lastConnectParams().config.sessionResumption).toEqual({}); + expect(onEvent).toHaveBeenCalledOnce(); + const resetOrder = onEvent.mock.invocationCallOrder[0]; + const reconnectOrder = connectMock.mock.invocationCallOrder[1]; + if (resetOrder === undefined || reconnectOrder === undefined) { + throw new Error("expected continuity reset before reconnect"); + } + expect(resetOrder).toBeLessThan(reconnectOrder); + + lastConnectParams().callbacks.onmessage({ + setupComplete: { sessionId: "session-2" }, + serverContent: { + inputTranscription: { text: "Fresh caller", finished: true }, + outputTranscription: { text: "Fresh assistant", finished: true }, + }, + }); + + expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([ + ["user", "Fresh caller", true], + ["assistant", "Fresh assistant", true], + ]); } finally { vi.useRealTimers(); } @@ -995,6 +1072,103 @@ describe("buildGoogleRealtimeVoiceProvider", () => { ); }); + it("emits one continuity reset across failed fresh reconnect attempts", async () => { + vi.useFakeTimers(); + const provider = buildGoogleRealtimeVoiceProvider(); + const onClose = vi.fn(); + const onEvent = vi.fn(); + const bridge = provider.createBridge({ + providerConfig: { apiKey: "gemini-key", sessionResumption: false }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), + onClose, + onEvent, + }); + + await bridge.connect(); + const firstSession = lastConnectParams().callbacks; + firstSession.onmessage({ setupComplete: {} }); + connectMock + .mockRejectedValueOnce(new Error("connect failed 1")) + .mockRejectedValueOnce(new Error("connect failed 2")) + .mockRejectedValueOnce(new Error("connect failed 3")); + firstSession.onclose({ code: 1011, reason: "temporary" }); + + await vi.advanceTimersByTimeAsync(1_750); + + expect(connectMock).toHaveBeenCalledTimes(4); + expect(onEvent.mock.calls).toEqual([ + [{ direction: "client", type: "session.continuity.reset" }], + ]); + expect(onClose).toHaveBeenCalledWith("error"); + }); + + it("rearms continuity reset after pre-return setup selects a fresh session", async () => { + vi.useFakeTimers(); + const pendingSession = createDeferred(); + const freshSession = createMockGoogleLiveSession(); + connectMock + .mockReturnValueOnce(Promise.resolve(session)) + .mockReturnValueOnce(pendingSession.promise); + const provider = buildGoogleRealtimeVoiceProvider(); + const onEvent = vi.fn(); + const onReady = vi.fn(); + const onTranscript = vi.fn(); + const bridge = provider.createBridge({ + providerConfig: { apiKey: "gemini-key", sessionResumption: false }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), + onEvent, + onReady, + onTranscript, + }); + + await bridge.connect(); + const firstCallbacks = lastConnectParams().callbacks; + firstCallbacks.onopen(); + firstCallbacks.onmessage({ setupComplete: {} }); + firstCallbacks.onclose({ code: 1011, reason: "temporary" }); + await vi.advanceTimersByTimeAsync(250); + + const freshCallbacks = lastConnectParams().callbacks; + expect(onEvent).toHaveBeenCalledTimes(1); + freshCallbacks.onopen(); + freshCallbacks.onmessage({ + setupComplete: {}, + serverContent: { inputTranscription: { text: "Fresh partial " } }, + }); + expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([ + "session.continuity.reset", + "session.created", + ]); + expect(onReady).toHaveBeenCalledTimes(1); + + pendingSession.resolve(freshSession); + await vi.waitFor(() => { + expect(onReady).toHaveBeenCalledTimes(2); + }); + const sessionCreatedOrder = onEvent.mock.invocationCallOrder[1]; + const freshReadyOrder = onReady.mock.invocationCallOrder[1]; + if (sessionCreatedOrder === undefined || freshReadyOrder === undefined) { + throw new Error("expected fresh session creation before readiness"); + } + expect(sessionCreatedOrder).toBeLessThan(freshReadyOrder); + freshCallbacks.onclose({ code: 1011, reason: "temporary again" }); + await vi.advanceTimersByTimeAsync(250); + + expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([ + "session.continuity.reset", + "session.created", + "session.continuity.reset", + ]); + lastConnectParams().callbacks.onmessage({ + serverContent: { inputTranscription: { text: "Next", finished: true } }, + }); + expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([ + ["user", "Next", true], + ]); + }); + it("waits for the returned session after setup completion before activating", async () => { const pendingSession = createDeferred(); const connectedSession = createMockGoogleLiveSession(); diff --git a/extensions/google/realtime-voice-provider.ts b/extensions/google/realtime-voice-provider.ts index 4b2b5ea82276..09b63d02b6be 100644 --- a/extensions/google/realtime-voice-provider.ts +++ b/extensions/google/realtime-voice-provider.ts @@ -482,6 +482,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { private reconnectAttempts = 0; private reconnectTimer: ReturnType | undefined; private hasConnectedSession = false; + private continuityResetEmitted = false; private terminalError: Error | undefined; private closeNotified = false; private connectionOwner: GoogleLiveConnectionAttempt | undefined; @@ -530,13 +531,6 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { } private async connectOwned(attempt: GoogleLiveConnectionAttempt): Promise { - const canResumeSession = - this.config.sessionResumption !== false && Boolean(this.resumptionHandle); - if (this.hasConnectedSession && !canResumeSession) { - // An unfinished recognition hypothesis cannot cross into a fresh server session. - // Dropping it avoids treating a transport break as a completed user utterance. - this.resetPendingTranscripts(); - } this.intentionallyClosed = false; this.closeNotified = false; this.setupCompleteReceived = false; @@ -852,7 +846,9 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { sessionResumptionUpdate?: { newHandle?: string; resumable?: boolean }; }; const update = raw.sessionResumptionUpdate; - if (update?.resumable && update.newHandle) { + if (update?.resumable === false) { + this.resumptionHandle = undefined; + } else if (update?.resumable && update.newHandle) { this.resumptionHandle = update.newHandle; } if (raw.goAway?.timeLeft) { @@ -861,6 +857,14 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { } private handleSetupComplete(): void { + if (!this.setupCompleteReceived) { + // setupComplete proves Google selected a new server session. A later + // continuity loss therefore owns a new reset generation. + if (this.continuityResetEmitted) { + this.config.onEvent?.({ direction: "server", type: "session.created" }); + } + this.continuityResetEmitted = false; + } this.setupCompleteReceived = true; this.maybeActivateSession(); } @@ -1046,6 +1050,18 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { if (this.reconnectAttempts >= GOOGLE_REALTIME_RECONNECT_MAX_ATTEMPTS) { return false; } + const canResumeSession = + this.config.sessionResumption !== false && Boolean(this.resumptionHandle); + if (this.hasConnectedSession && !canResumeSession && !this.continuityResetEmitted) { + // A non-resumable close ends the provider generation immediately. Reset + // consumers before backoff so stale work cannot finish into the replacement. + this.continuityResetEmitted = true; + this.resetPendingTranscripts(); + this.config.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + } const attempt = ++this.reconnectAttempts; const delayMs = Math.min( GOOGLE_REALTIME_RECONNECT_MAX_DELAY_MS, diff --git a/extensions/voice-call/src/webhook/realtime-handler.test.ts b/extensions/voice-call/src/webhook/realtime-handler.test.ts index 4f85c402b4f1..e3f779ae3210 100644 --- a/extensions/voice-call/src/webhook/realtime-handler.test.ts +++ b/extensions/voice-call/src/webhook/realtime-handler.test.ts @@ -254,6 +254,7 @@ async function withBargeInHarness( createBridge: ReturnType; handleBargeIn: ReturnType; outboundMessages: Array>; + processEvent: ReturnType; sendAudio: ReturnType; ws: WebSocket; }) => Promise, @@ -261,6 +262,7 @@ async function withBargeInHarness( let callbacks: RealtimeBridgeRequest | undefined; const sendAudio = vi.fn(); const handleBargeIn = vi.fn(); + const processEvent = vi.fn(); const call = makeCallRecord(params.providerCallId); const createBridge = vi.fn((request: RealtimeBridgeRequest) => { callbacks = request; @@ -278,6 +280,7 @@ async function withBargeInHarness( const handler = makeHandler(undefined, { manager: { getCallByProviderCallId: vi.fn((): CallRecord => call), + processEvent, }, providerConfig: { apiKey: "test-key", @@ -313,6 +316,7 @@ async function withBargeInHarness( createBridge, handleBargeIn, outboundMessages, + processEvent, sendAudio, ws, }); @@ -948,6 +952,63 @@ describe("RealtimeCallHandler path routing", () => { ); }); + it("starts fresh transcript and Talk state after provider continuity resets", async () => { + await withBargeInHarness( + { providerCallId: "CA-continuity-reset" }, + async ({ callbacks, call, outboundMessages, processEvent }) => { + callbacks.onTranscript?.("user", "Old caller ", false); + callbacks.onTranscript?.("assistant", "Old assistant ", false); + callbacks.onAudio?.(Buffer.alloc(320, 0xff)); + const oldTurnId = recentTalkEvents(call).findLast( + (event) => event.type === "turn.started", + )?.turnId; + expect(oldTurnId).toBeTruthy(); + + callbacks.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + callbacks.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + + await waitForRealtimeTest(() => { + expect(outboundMessages.some((message) => message.event === "clear")).toBe(true); + }); + expect(requireCancelledTurn(call).turnId).toBe(oldTurnId); + const resetEvents = recentTalkEvents(call); + expect(resetEvents.filter((event) => event.type === "turn.cancelled")).toHaveLength(1); + expect(resetEvents.findIndex((event) => event.type === "output.audio.done")).toBeLessThan( + resetEvents.findIndex((event) => event.type === "turn.cancelled"), + ); + + callbacks.onTranscript?.("user", "Fresh caller", true); + callbacks.onTranscript?.("assistant", "Fresh assistant", true); + callbacks.onEvent?.({ direction: "server", type: "response.done" }); + + const processedEvents = processEvent.mock.calls.map(([event]) => event as NormalizedEvent); + expect( + processedEvents + .filter((event) => event.type === "call.speech") + .map((event) => (event.type === "call.speech" ? event.transcript : undefined)), + ).toEqual(["Fresh caller"]); + expect( + processedEvents + .filter((event) => event.type === "call.assistant-speech") + .map((event) => + event.type === "call.assistant-speech" ? event.transcript : undefined, + ), + ).toEqual(["Fresh assistant"]); + const startedTurns = recentTalkEvents(call).filter( + (event) => event.type === "turn.started", + ); + expect(startedTurns).toHaveLength(2); + expect(startedTurns[1]?.turnId).not.toBe(oldTurnId); + }, + ); + }); + it("passes the disabled input-interruption policy without cancelling speech-start", async () => { await withBargeInHarness( { @@ -1492,6 +1553,87 @@ describe("RealtimeCallHandler path routing", () => { } }); + it("clears cancelled consult dedupe for a fresh provider session", async () => { + let callbacks: RealtimeBridgeRequest | undefined; + let sessionHarness: RealtimeVoiceSessionHarness | undefined; + realtimeVoiceHarnessTestHooks.onCreate = (harness) => { + sessionHarness = harness; + }; + const submitToolResult = vi.fn(); + const createBridge = vi.fn((request: RealtimeBridgeRequest) => { + callbacks = request; + return makeBridge({ submitToolResult }); + }); + const handler = makeHandler(undefined, { + manager: { + getCallByProviderCallId: vi.fn((providerCallId: string) => makeCallRecord(providerCallId)), + }, + realtimeProvider: makeRealtimeProvider(createBridge), + }); + const consult = vi.fn(async () => ({ text: "fresh consult answer" })); + handler.registerToolHandler("openclaw_agent_consult", consult); + const server = await startRealtimeServer(handler); + + try { + const ws = await connectWs(server.url); + try { + ws.send( + JSON.stringify({ + event: "start", + start: { streamSid: "MZ-continuity-consult", callSid: "CA-continuity-consult" }, + }), + ); + await waitForRealtimeTest(() => { + expect(callbacks).toBeDefined(); + expect(sessionHarness).toBeDefined(); + }); + + const coordinator = expectDefined( + sessionHarness, + "voice-call realtime session harness", + ).forcedConsults; + const cancelled = expectDefined( + coordinator.prepare("same question"), + "cancelled forced consult", + ); + coordinator.markStarted(cancelled); + coordinator.markCancelled(cancelled); + + callbacks?.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + callbacks?.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + expect(coordinator.handles()).toEqual([]); + + callbacks?.onToolCall?.({ + itemId: "item-fresh", + callId: "native-fresh", + name: "openclaw_agent_consult", + args: { question: "same question" }, + }); + + await waitForRealtimeTest(() => { + expect(consult).toHaveBeenCalledTimes(1); + expect(submitToolResult).toHaveBeenCalledWith( + "native-fresh", + { text: "fresh consult answer" }, + undefined, + ); + }); + } finally { + if (ws.readyState !== WebSocket.CLOSED && ws.readyState !== WebSocket.CLOSING) { + ws.close(); + } + } + } finally { + await server.close(); + } + }); + it("does not deliver a forced consult after its realtime session closes", async () => { let callbacks: | { @@ -1727,6 +1869,125 @@ describe("RealtimeCallHandler path routing", () => { } }); + it("ignores late continuity reset and close from a replaced bridge", async () => { + const callbacks: RealtimeBridgeRequest[] = []; + const oldCloseBridge = vi.fn(); + const replacementCloseBridge = vi.fn(); + const bridges = [ + makeBridge({ close: oldCloseBridge }), + makeBridge({ close: replacementCloseBridge }), + ]; + const createBridge = vi.fn((request: RealtimeBridgeRequest) => { + callbacks.push(request); + const bridge = bridges[callbacks.length - 1]; + if (!bridge) { + throw new Error("unexpected replacement bridge"); + } + return bridge; + }); + const processEvent = vi.fn(); + const hangupCall = vi.fn(async () => {}); + const sharedCallSid = "CA-continuity-shared"; + const call = makeCallRecord(sharedCallSid); + const handler = makeHandler(undefined, { + manager: { + getCallByProviderCallId: vi.fn(() => call), + processEvent, + }, + provider: { hangupCall }, + realtimeProvider: makeRealtimeProvider(createBridge), + }); + const oldServer = await startRealtimeServer(handler); + let replacementServer: Awaited> | undefined; + let oldWs: WebSocket | undefined; + + try { + oldWs = await connectWs(oldServer.url); + oldWs.send( + JSON.stringify({ + event: "start", + start: { streamSid: "MZ-continuity-old", callSid: sharedCallSid }, + }), + ); + await waitForRealtimeTest(() => { + expect(callbacks).toHaveLength(1); + }); + + replacementServer = await startRealtimeServer(handler); + const replacementWs = await connectWs(replacementServer.url); + try { + replacementWs.send( + JSON.stringify({ + event: "start", + start: { + streamSid: "MZ-continuity-replacement", + callSid: sharedCallSid, + }, + }), + ); + await waitForRealtimeTest(() => { + expect(callbacks).toHaveLength(2); + }); + + callbacks[1]?.onTranscript?.("user", "Fresh ", false); + callbacks[0]?.onTranscript?.("user", "stale partial", false); + callbacks[0]?.onTranscript?.("user", "stale final", true); + callbacks[0]?.onTranscript?.("assistant", "stale assistant", true); + callbacks[0]?.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + callbacks[0]?.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + const oldClosed = waitForClose(oldWs); + callbacks[0]?.onClose?.("error"); + await oldClosed; + expect(oldCloseBridge).toHaveBeenCalledOnce(); + expect(replacementCloseBridge).not.toHaveBeenCalled(); + expect(hangupCall).not.toHaveBeenCalled(); + expect( + processEvent.mock.calls + .map(([event]) => event as NormalizedEvent) + .filter((event) => event.type === "call.ended"), + ).toHaveLength(0); + callbacks[1]?.onTranscript?.("user", "caller", true); + + await waitForRealtimeTest(() => { + expect( + processEvent.mock.calls + .map(([event]) => event as NormalizedEvent) + .filter((event) => event.type === "call.speech") + .map((event) => (event.type === "call.speech" ? event.transcript : undefined)), + ).toEqual(["Fresh caller"]); + }); + expect( + processEvent.mock.calls + .map(([event]) => event as NormalizedEvent) + .filter((event) => event.type === "call.assistant-speech"), + ).toHaveLength(0); + } finally { + if ( + replacementWs.readyState !== WebSocket.CLOSED && + replacementWs.readyState !== WebSocket.CLOSING + ) { + replacementWs.close(); + } + } + } finally { + if ( + oldWs && + oldWs.readyState !== WebSocket.CLOSED && + oldWs.readyState !== WebSocket.CLOSING + ) { + oldWs.close(); + } + await replacementServer?.close(); + await oldServer.close(); + } + }); + it("does not share a native consult with a replacement realtime session", async () => { const callbacks: RealtimeBridgeRequest[] = []; const oldSubmitToolResult = vi.fn(); diff --git a/extensions/voice-call/src/webhook/realtime-handler.ts b/extensions/voice-call/src/webhook/realtime-handler.ts index 7417857d4fe8..cb8a0c3a7950 100644 --- a/extensions/voice-call/src/webhook/realtime-handler.ts +++ b/extensions/voice-call/src/webhook/realtime-handler.ts @@ -339,7 +339,10 @@ export class RealtimeCallHandler { private readonly activeBridgesByCallId = new Map(); private readonly activeTelephonyClosersByCallId = new Map< string, - (reason: TelephonyCloseReason) => void + { + owner: ActiveRealtimeVoiceBridge; + close: (reason: TelephonyCloseReason) => void; + } >(); private readonly partialUserTranscriptsByCallId = new Map(); private readonly rawPartialUserTranscriptsByCallId = new Map(); @@ -797,6 +800,10 @@ export class RealtimeCallHandler { }, }, onTranscript: (role, text, isFinal) => { + const owner = nativeConsultOwner.current; + if (owner && !this.isActiveBridgeOwner(callId, owner)) { + return; + } const turnId = harness.ensureTurn(); const eventType = role === "assistant" @@ -898,6 +905,27 @@ export class RealtimeCallHandler { ); }, onEvent: (event) => { + if (event.direction === "client" && event.type === "session.continuity.reset") { + // A fresh provider session cannot complete the prior session's text, + // audio, tool work, or Talk turn. + const turnId = harness.talk.activeTurnId; + const owner = nativeConsultOwner.current; + if (owner && this.isActiveBridgeOwner(callId, owner)) { + this.clearUserTranscriptState(callId); + this.resetConsultSessionForContinuity(callId, owner); + } + harness.flushOutput(() => { + audioPacer.clearAudio(); + harness.finishOutputAudio(event.type); + }); + if (turnId) { + harness.talk.cancelTurn({ + turnId, + payload: { callId, providerCallId: callSid, reason: event.type }, + }); + } + return; + } if (event.type === "input_audio_buffer.speech_started") { harness.ensureTurn(); return; @@ -943,11 +971,15 @@ export class RealtimeCallHandler { }); }, onClose: (reason) => { - if (nativeConsultOwner.current) { - this.clearActiveBridgeMappings(callId, callSid, nativeConsultOwner.current); - this.cancelConsultSession(callId, nativeConsultOwner.current); + const owner = nativeConsultOwner.current; + const ownsCallState = owner ? this.isActiveBridgeOwner(callId, owner) : false; + if (owner) { + this.clearActiveBridgeMappings(callId, callSid, owner); + this.cancelConsultSession(callId, owner); + } + if (ownsCallState) { + this.clearUserTranscriptState(callId); } - this.clearUserTranscriptState(callId); harness.finishOutputAudio(reason); harness.emit({ type: "session.closed", @@ -957,10 +989,13 @@ export class RealtimeCallHandler { if (reason !== "error") { return; } - emitCallEnd("error"); if (ws.readyState === WebSocket.OPEN) { ws.close(1011, "Bridge disconnected"); } + if (owner && !ownsCallState) { + return; + } + emitCallEnd("error"); void this.provider .hangupCall({ callId, providerCallId: callSid, reason: "error" }) .catch((error: unknown) => { @@ -992,8 +1027,9 @@ export class RealtimeCallHandler { }); this.activeBridgesByCallId.set(callId, session); this.activeBridgesByCallId.set(callSid, session); - this.activeTelephonyClosersByCallId.set(callId, closeTelephony); - this.activeTelephonyClosersByCallId.set(callSid, closeTelephony); + const telephonyCloser = { owner: session, close: closeTelephony }; + this.activeTelephonyClosersByCallId.set(callId, telephonyCloser); + this.activeTelephonyClosersByCallId.set(callSid, telephonyCloser); const sendAudioToSession = session.sendAudio.bind(session); session.sendAudio = (audio) => { if (speechDetector.accept({ rms: calculateMulawRms(audio), peak: 0 })) { @@ -1021,9 +1057,12 @@ export class RealtimeCallHandler { try { closeSession(); } finally { + const ownsCallState = this.isActiveBridgeOwner(callId, session); this.clearActiveBridgeMappings(callId, callSid, session); this.cancelConsultSession(callId, session); - this.clearUserTranscriptState(callId); + if (ownsCallState) { + this.clearUserTranscriptState(callId); + } harness.close(); audioPacer.close(); } @@ -1031,8 +1070,11 @@ export class RealtimeCallHandler { session.connect().catch((error: unknown) => { console.error("[voice-call] Failed to connect realtime bridge:", error); + const ownsCallState = this.isActiveBridgeOwner(callId, session); session.close(); - emitCallEnd("error"); + if (ownsCallState) { + emitCallEnd("error"); + } ws.close(1011, "Failed to connect"); }); @@ -1105,22 +1147,43 @@ export class RealtimeCallHandler { this.forcedConsultsByCallId.delete(callId); } - private cancelConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge | undefined): void { - if (!owner) { - return; - } + private resetConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge): boolean { const session = this.consultSessionsByCallId.get(callId); if (!session || session.owner !== owner) { - return; + return false; } - // Forced and native consults share bridge ownership. Replacement or close - // must invalidate both before a newer bridge can observe call-scoped state. session.coordinator.clearPending(); this.cancelForcedConsult(callId, owner); this.cancelNativeConsult(callId, owner); + return true; + } + + private resetConsultSessionForContinuity( + callId: string, + owner: ActiveRealtimeVoiceBridge, + ): boolean { + const session = this.consultSessionsByCallId.get(callId); + if (!session || session.owner !== owner) { + return false; + } + this.cancelForcedConsult(callId, owner); + this.cancelNativeConsult(callId, owner); + // A fresh provider session must not inherit cancelled/recent consult dedupe. + session.coordinator.clear(); + return true; + } + + private cancelConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge | undefined): void { + if (!owner || !this.resetConsultSession(callId, owner)) { + return; + } this.consultSessionsByCallId.delete(callId); } + private isActiveBridgeOwner(callId: string, owner: ActiveRealtimeVoiceBridge): boolean { + return this.activeBridgesByCallId.get(callId) === owner; + } + private clearActiveBridgeMappings( callId: string, callSid: string, @@ -1197,8 +1260,8 @@ export class RealtimeCallHandler { reason: TelephonyCloseReason, ): void { const closer = this.activeTelephonyClosersByCallId.get(callIdOrSid); - if (closer) { - closer(reason); + if (closer && closer.owner === bridge) { + closer.close(reason); return; } bridge?.close(); diff --git a/src/gateway/talk-realtime-relay-forced-consults.ts b/src/gateway/talk-realtime-relay-forced-consults.ts index 96031076c021..94e1ac8199ba 100644 --- a/src/gateway/talk-realtime-relay-forced-consults.ts +++ b/src/gateway/talk-realtime-relay-forced-consults.ts @@ -22,6 +22,7 @@ import { ensureRelayTurn, noFallbackRelayOutputFlush, relaySessions, + resolveRelayProviderToolCallId, type ForcedTerminalProviderResult, type RelayAgentControlProviderSubmission, type RelaySession, @@ -275,7 +276,7 @@ export function submitRealtimeAgentConsultWorkingResponse( } const epoch = session.toolResultEpoch; const submission = session.bridge.submitToolResult( - callId, + resolveRelayProviderToolCallId(session, callId), buildRealtimeVoiceAgentConsultWorkingResponse("person"), { willContinue: true }, ); diff --git a/src/gateway/talk-realtime-relay-operations.ts b/src/gateway/talk-realtime-relay-operations.ts index 7ebd5325def8..582606ffa5a5 100644 --- a/src/gateway/talk-realtime-relay-operations.ts +++ b/src/gateway/talk-realtime-relay-operations.ts @@ -5,6 +5,7 @@ import { } from "../talk/agent-run-control.js"; import { registerClientVoiceConsultRun } from "../talk/client-voice-session.js"; import type { RealtimeVoiceToolResultOptions } from "../talk/provider-types.js"; +import type { TalkEvent } from "../talk/talk-session-controller.js"; import { abortChatRunById } from "./chat-abort.js"; import { formatError } from "./server-utils.js"; import { @@ -30,6 +31,7 @@ import { ensureRelayTurn, noFallbackRelayOutputFlush, relaySessions, + resolveRelayProviderToolCallId, type RelaySession, } from "./talk-realtime-relay-state.js"; import { @@ -298,7 +300,11 @@ export function submitTalkRealtimeRelayToolResult(params: { return trackAgentFinalToolResult(session, params.callId, completion); } const submit = () => - session.bridge.submitToolResult(params.callId, params.result, params.options); + session.bridge.submitToolResult( + resolveRelayProviderToolCallId(session, params.callId), + params.result, + params.options, + ); const pendingWorking = session.pendingWorkingToolResults.get(params.callId); if (pendingWorking) { const submission = pendingWorking.then(async () => { @@ -451,6 +457,54 @@ export function cancelTalkRealtimeRelayTurn(params: { }); } +/** Drops one provider generation without sending cancellation into its replacement. */ +export function resetTalkRealtimeRelayContinuity( + session: RelaySession, + reason = "session.continuity.reset", +): TalkEvent | undefined { + session.toolResultEpoch += 1; + const retiredCallIds = new Set([ + ...session.activeAgentToolCalls.keys(), + ...session.cancelledAgentToolCalls.keys(), + ...session.providerToolCallIds.keys(), + ...session.providerToolCallIds.values(), + ...session.pendingFinalToolResults.keys(), + ...session.pendingProviderToolResults.keys(), + ...session.pendingWorkingToolResults.keys(), + ...session.forcedTerminalProviderResults.keys(), + ]); + for (const handle of session.harness.forcedConsults.handles()) { + retiredCallIds.add(handle.id); + for (const nativeCallId of session.harness.forcedConsults.nativeCallIds(handle)) { + retiredCallIds.add(nativeCallId); + } + } + for (const callId of retiredCallIds) { + session.completedAgentToolCalls.add(callId); + } + session.cancelledAgentToolCalls.clear(); + session.providerToolCallIds.clear(); + session.relayToolCallIdsByProviderId.clear(); + session.pendingFinalToolResults.clear(); + session.completedProviderToolResults.clear(); + session.pendingProviderToolResults.clear(); + session.pendingWorkingToolResults.clear(); + session.forcedTerminalProviderResults.clear(); + session.harness.forcedConsults.clear(); + abortRelayAgentRuns(session, reason); + const turnId = session.harness.talk.activeTurnId; + session.harness.flushOutput(noFallbackRelayOutputFlush); + session.harness.finishOutputAudio(reason); + if (!turnId) { + return undefined; + } + const cancelled = session.harness.talk.cancelTurn({ + turnId, + payload: { reason }, + }); + return cancelled.ok ? cancelled.event : undefined; +} + /** Closes a realtime relay session owned by the current connection. */ export function stopTalkRealtimeRelaySession(params: { relaySessionId: string; diff --git a/src/gateway/talk-realtime-relay-provider-results.ts b/src/gateway/talk-realtime-relay-provider-results.ts index 552784688ce2..31630e960f4e 100644 --- a/src/gateway/talk-realtime-relay-provider-results.ts +++ b/src/gateway/talk-realtime-relay-provider-results.ts @@ -1,6 +1,11 @@ import { buildRealtimeVoiceAgentCancelProviderResult } from "../talk/agent-run-control-shared.js"; import type { RealtimeVoiceToolResultOptions } from "../talk/provider-types.js"; -import { broadcastToOwner, relaySessions, type RelaySession } from "./talk-realtime-relay-state.js"; +import { + broadcastToOwner, + relaySessions, + resolveRelayProviderToolCallId, + type RelaySession, +} from "./talk-realtime-relay-state.js"; export function suppressedToolResultOptions( session: RelaySession, @@ -63,8 +68,13 @@ export function submitFinalProviderToolResult(params: { options?: RealtimeVoiceToolResultOptions; onAccepted?: () => void; }): void | Promise { - if (params.session.completedProviderToolResults.has(params.callId)) { - if (relaySessions.get(params.session.id) === params.session) { + const epoch = params.session.toolResultEpoch; + const providerCallId = resolveRelayProviderToolCallId(params.session, params.callId); + if (params.session.completedProviderToolResults.has(providerCallId)) { + if ( + relaySessions.get(params.session.id) === params.session && + params.session.toolResultEpoch === epoch + ) { params.onAccepted?.(); } return; @@ -74,9 +84,8 @@ export function submitFinalProviderToolResult(params: { return pending; } const submit = () => - params.session.bridge.submitToolResult(params.callId, params.result, params.options); + params.session.bridge.submitToolResult(providerCallId, params.result, params.options); const working = params.session.pendingWorkingToolResults.get(params.callId); - const epoch = params.session.toolResultEpoch; const submitAfterWorking = async () => { if (relaySessions.get(params.session.id) !== params.session) { return false; @@ -89,13 +98,13 @@ export function submitFinalProviderToolResult(params: { // the provider's working-result acknowledgement. Finish the cancelled call // here so the provider is not left waiting for a terminal result. await params.session.bridge.submitToolResult( - params.callId, + providerCallId, buildRealtimeVoiceAgentCancelProviderResult( "OpenClaw cancelled this consult before completion. Do not restart it.", ), suppressedToolResultOptions(params.session), ); - params.session.completedProviderToolResults.add(params.callId); + params.session.completedProviderToolResults.add(providerCallId); params.session.cancelledAgentToolCalls.delete(params.callId); params.session.completedAgentToolCalls.add(params.callId); return false; @@ -105,7 +114,10 @@ export function submitFinalProviderToolResult(params: { }; const submission = working ? working.then(submitAfterWorking, submitAfterWorking) : submit(); const accept = () => { - params.session.completedProviderToolResults.add(params.callId); + if (params.session.toolResultEpoch !== epoch) { + return; + } + params.session.completedProviderToolResults.add(providerCallId); if (relaySessions.get(params.session.id) === params.session) { params.onAccepted?.(); } diff --git a/src/gateway/talk-realtime-relay-session-create.ts b/src/gateway/talk-realtime-relay-session-create.ts index 110ed9cb3891..baff1c98d77f 100644 --- a/src/gateway/talk-realtime-relay-session-create.ts +++ b/src/gateway/talk-realtime-relay-session-create.ts @@ -35,12 +35,14 @@ import { enforceRelaySessionLimits, pruneInactiveRelayAgentRuns, registerTalkRealtimeRelayAgentRun, + resetTalkRealtimeRelayContinuity, steerTalkRealtimeRelayAgentRun, } from "./talk-realtime-relay-operations.js"; import { suppressedToolResultOptions } from "./talk-realtime-relay-provider-results.js"; import { RELAY_SESSION_TTL_MS, RELAY_TRANSCRIPT_ECHO_LOOKBACK_MS, + adoptRelayProviderToolCallId, broadcastToOwner, ensureRelayTurn, relayEventDeliveryOptions, @@ -105,6 +107,7 @@ export function createTalkRealtimeRelaySession( let currentOutputItemId: string | undefined; let currentOutputResponseId: string | undefined; let ready = false; + let continuityResetActive = false; let failureEmitted = false; let transcriptPersistenceFailed = false; const constructionTerminal: { @@ -253,12 +256,34 @@ export function createTalkRealtimeRelaySession( }, }, onEvent: (event) => { - if (!getActiveRelay()) { + const relay = getActiveRelay(); + if (!relay) { + return; + } + if (event.direction === "client" && event.type === "session.continuity.reset") { + if (continuityResetActive) { + return; + } + continuityResetActive = true; + ready = false; + currentOutputItemId = undefined; + currentOutputResponseId = undefined; + const talkEvent = resetTalkRealtimeRelayContinuity(relay, event.type); + const clearEvent = { relaySessionId, type: "clear" as const }; + broadcastToOwner( + params.context, + params.connId, + { ...clearEvent, ...(talkEvent ? { talkEvent } : {}) }, + relayEventDeliveryOptions(clearEvent), + ); return; } if (event.direction !== "server") { return; } + if (event.type === "session.created") { + continuityResetActive = false; + } if ( event.type === "conversation.output_audio.delta" || event.type === "response.audio.delta" || @@ -365,11 +390,13 @@ export function createTalkRealtimeRelaySession( if (!relay) { return; } + const providerCallId = toolCall.callId; + const relayCallId = adoptRelayProviderToolCallId(relay, providerCallId); let shouldSubmitWorkingResult = false; if (toolCall.name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { const forcedConsult = relay.harness.forcedConsults.recordNativeConsult( toolCall.args, - toolCall.callId, + providerCallId, ); if (forcedConsult.kind === "in_flight" || forcedConsult.kind === "already_delivered") { if (forcedConsult.kind === "already_delivered") { @@ -380,7 +407,7 @@ export function createTalkRealtimeRelaySession( : buildAlreadyDeliveredToolResult(); return submitForcedConsultProviderResult( relay, - toolCall.callId, + providerCallId, result, suppressedToolResultOptions(relay), ); @@ -388,7 +415,7 @@ export function createTalkRealtimeRelaySession( if (relay.forcedTerminalProviderResults.has(forcedConsult.handle.id)) { return relay.pendingFinalToolResults.get(forcedConsult.handle.id); } - return submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId); + return submitRealtimeAgentConsultWorkingResponse(relay, relayCallId); } shouldSubmitWorkingResult = true; } @@ -398,20 +425,20 @@ export function createTalkRealtimeRelaySession( relaySessionId, type: "toolCall", itemId: toolCall.itemId, - callId: toolCall.callId, + callId: relayCallId, name: toolCall.name, args: toolCall.args, }, { type: "tool.call", itemId: toolCall.itemId, - callId: toolCall.callId, + callId: relayCallId, turnId, payload: { name: toolCall.name, args: toolCall.args }, }, ); if (shouldSubmitWorkingResult) { - return submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId, turnId); + return submitRealtimeAgentConsultWorkingResponse(relay, relayCallId, turnId); } }, onReady: () => { @@ -419,6 +446,7 @@ export function createTalkRealtimeRelaySession( return; } ready = true; + continuityResetActive = false; emit({ relaySessionId, type: "ready" }, { type: "session.ready", payload: null }); }, onError: (error) => { @@ -536,6 +564,8 @@ export function createTalkRealtimeRelaySession( provider: params.provider.id, activeAgentToolCalls: new Map(), completedAgentToolCalls: new Set(), + providerToolCallIds: new Map(), + relayToolCallIdsByProviderId: new Map(), cancelledAgentToolCalls: new Map(), pendingFinalToolResults: new Map(), completedProviderToolResults: new Set(), diff --git a/src/gateway/talk-realtime-relay-state.ts b/src/gateway/talk-realtime-relay-state.ts index 5531e207fd86..406b3b64d9c5 100644 --- a/src/gateway/talk-realtime-relay-state.ts +++ b/src/gateway/talk-realtime-relay-state.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import type { OpenClawConfig } from "../config/types.js"; import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js"; import type { BoundedSerialQueue } from "../shared/bounded-serial-queue.js"; @@ -94,6 +95,8 @@ export type RelaySession = { provider: string; activeAgentToolCalls: Map; completedAgentToolCalls: Set; + providerToolCallIds: Map; + relayToolCallIdsByProviderId: Map; // Cancelled calls retain their original turn long enough to terminally satisfy // late browser results without creating a replacement turn or owner success event. cancelledAgentToolCalls: Map; @@ -150,6 +153,28 @@ export const relaySessions = new Map(); // until durable close settles. Session limits count both maps. export const drainingRelaySessions = new Set(); +export function adoptRelayProviderToolCallId( + session: RelaySession, + providerCallId: string, +): string { + const current = session.relayToolCallIdsByProviderId.get(providerCallId); + if (current) { + return current; + } + const relayCallId = session.completedAgentToolCalls.has(providerCallId) + ? `relay-${randomUUID()}` + : providerCallId; + session.completedProviderToolResults.delete(providerCallId); + session.completedAgentToolCalls.delete(relayCallId); + session.providerToolCallIds.set(relayCallId, providerCallId); + session.relayToolCallIdsByProviderId.set(providerCallId, relayCallId); + return relayCallId; +} + +export function resolveRelayProviderToolCallId(session: RelaySession, relayCallId: string): string { + return session.providerToolCallIds.get(relayCallId) ?? relayCallId; +} + export function broadcastToOwner( context: GatewayRequestContext, connId: string, diff --git a/src/gateway/talk-realtime-relay.test.ts b/src/gateway/talk-realtime-relay.test.ts index 731a5b51238f..be6aa43fef80 100644 --- a/src/gateway/talk-realtime-relay.test.ts +++ b/src/gateway/talk-realtime-relay.test.ts @@ -989,6 +989,165 @@ describe("talk realtime gateway relay", () => { expectRecordFields(mockCallArg(mock, 0, 2), { runId: "run-1", state: "aborted" }); } + it("resets provider continuity without cancelling the replacement provider", async () => { + let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined; + const pendingWorking = createDeferredVoid(); + const handleBargeIn = vi.fn(); + const submitToolResult = vi + .fn() + .mockReturnValueOnce(pendingWorking.promise); + const bridge = { + supportsToolResultContinuation: true, + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + sendUserMessage: vi.fn(), + handleBargeIn, + submitToolResult, + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }; + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: (request) => { + bridgeRequest = request; + return bridge; + }, + }; + const fixture = createAbortableRelayRunFixture(provider); + const relay = relaySessions.get(fixture.session.relaySessionId); + if (!relay || !bridgeRequest) { + throw new Error("expected active relay continuity fixture"); + } + const request = bridgeRequest; + request.onReady?.(); + request.onEvent?.({ + direction: "server", + type: "response.audio.delta", + itemId: "old-item", + responseId: "old-response", + }); + request.onAudio(Buffer.from("old audio")); + request.onToolCall?.({ + itemId: "old-item", + callId: "call-1", + name: "custom_tool", + args: {}, + }); + const working = submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { status: "working" }, + options: { willContinue: true }, + }); + + request.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + request.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + + expect(fixture.abortController.signal.aborted).toBe(true); + expect(handleBargeIn).not.toHaveBeenCalled(); + expect(bridge.close).not.toHaveBeenCalled(); + expect(submitToolResult).toHaveBeenCalledTimes(1); + expect(relay.pendingWorkingToolResults.size).toBe(0); + expect(relay.activeAgentRuns.size).toBe(0); + expect(relay.activeAgentToolCalls.size).toBe(0); + const payloads = fixture.broadcastToConnIds.mock.calls.map(([, payload]) => payload); + const clears = payloads.filter( + (payload): payload is Record => + typeof payload === "object" && + payload !== null && + (payload as Record).type === "clear", + ); + expect(clears).toHaveLength(1); + expectRecordFields(clears[0]?.talkEvent, { + type: "turn.cancelled", + payload: { reason: "session.continuity.reset" }, + final: true, + }); + + pendingWorking.resolve(); + await working; + expect(submitToolResult).toHaveBeenCalledTimes(1); + + request.onEvent?.({ direction: "server", type: "session.created" }); + request.onAudio(Buffer.from("fresh audio")); + const freshAudio = fixture.broadcastToConnIds.mock.calls + .map(([, payload]) => payload) + .findLast( + (payload): payload is Record => + typeof payload === "object" && + payload !== null && + (payload as Record).type === "audio", + ); + expectRecordFields(freshAudio, { + type: "audio", + audioBase64: Buffer.from("fresh audio").toString("base64"), + }); + expect(freshAudio).not.toHaveProperty("itemId"); + expect(freshAudio).not.toHaveProperty("responseId"); + + request.onToolCall?.({ + itemId: "fresh-item", + callId: "call-1", + name: "custom_tool", + args: {}, + }); + const freshToolCall = fixture.broadcastToConnIds.mock.calls + .map(([, payload]) => payload) + .findLast( + (payload): payload is Record => + typeof payload === "object" && + payload !== null && + (payload as Record).type === "toolCall", + ); + const freshCallId = freshToolCall?.callId; + expect(typeof freshCallId).toBe("string"); + expect(freshCallId).not.toBe("call-1"); + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: "call-1", + result: { stale: true }, + }); + expect(submitToolResult).toHaveBeenCalledTimes(1); + await submitTalkRealtimeRelayToolResult({ + relaySessionId: fixture.session.relaySessionId, + connId: "conn-1", + callId: String(freshCallId), + result: { ok: true }, + }); + expect(submitToolResult).toHaveBeenLastCalledWith("call-1", { ok: true }, undefined); + + request.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + request.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + expect( + fixture.broadcastToConnIds.mock.calls + .map(([, payload]) => payload) + .filter( + (payload) => + typeof payload === "object" && + payload !== null && + (payload as Record).type === "clear", + ), + ).toHaveLength(2); + }); + it("bridges browser audio, transcripts, and tool results through a backend provider", async () => { let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined; const bridge = { diff --git a/src/meeting-bot/agent-consult.ts b/src/meeting-bot/agent-consult.ts index 5e098e8f1338..4acf307ade0e 100644 --- a/src/meeting-bot/agent-consult.ts +++ b/src/meeting-bot/agent-consult.ts @@ -24,6 +24,7 @@ import type { MeetingRealtimeToolCallParams, MeetingRuntimePlatform, } from "./realtime-engine.js"; +import { readMeetingRealtimeToolAbortSignal } from "./realtime-tool-continuity.js"; function resolveMeetingRealtimeTools( policy: RealtimeVoiceAgentConsultToolPolicy, @@ -77,7 +78,8 @@ export function createMeetingRealtimeEngineBindings(params: { ...consult, }), tools: resolveMeetingRealtimeTools(params.config.realtime.toolPolicy), - handleToolCall: async (call) => + handleToolCall: async (call) => { + const abortSignal = readMeetingRealtimeToolAbortSignal(call.session); await handleMeetingRealtimeConsultToolCall({ surface, config: params.fullConfig, @@ -85,17 +87,20 @@ export function createMeetingRealtimeEngineBindings(params: { logger: params.logger, agentId: params.config.realtime.agentId, toolPolicy: params.config.realtime.toolPolicy, + abortSignal, ...call, - }), + }); + }, }; } async function submitMeetingConsultWorkingResponse(params: { session: RealtimeVoiceBridgeSession; + abortSignal?: AbortSignal; callId: string; label: string; }): Promise { - if (!params.session.bridge.supportsToolResultContinuation) { + if (params.abortSignal?.aborted || !params.session.bridge.supportsToolResultContinuation) { return; } await params.session.submitToolResult( @@ -116,6 +121,7 @@ async function consultMeetingAgent(params: { requesterSessionKey?: string; args: unknown; transcript: Array<{ role: "user" | "assistant"; text: string }>; + abortSignal?: AbortSignal; }): Promise<{ text: string }> { const agentId = params.agentId ? normalizeAgentId(params.agentId) @@ -142,6 +148,7 @@ async function consultMeetingAgent(params: { questionSourceLabel: params.surface.questionSourceLabel, toolsAllow: resolveRealtimeVoiceAgentConsultToolsAllow(params.toolPolicy), extraSystemPrompt: params.surface.extraSystemPrompt, + abortSignal: params.abortSignal, }); } @@ -158,12 +165,19 @@ async function handleMeetingRealtimeConsultToolCall(params: { meetingSessionId: string; requesterSessionKey?: string; transcript: Array<{ role: "user" | "assistant"; text: string }>; + abortSignal?: AbortSignal; onTalkEvent?: (event: TalkEventInput) => void; }): Promise { const callId = params.event.callId || params.event.itemId; + if (params.abortSignal?.aborted) { + return; + } if (params.strategy !== "bidi") { const error = `Tool "${params.event.name}" is only available in bidi realtime strategy`; await params.session.submitToolResult(callId, { error }); + if (params.abortSignal?.aborted) { + return; + } params.onTalkEvent?.({ type: "tool.error", callId, @@ -175,6 +189,9 @@ async function handleMeetingRealtimeConsultToolCall(params: { if (params.event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { const error = `Tool "${params.event.name}" not available`; await params.session.submitToolResult(callId, { error }); + if (params.abortSignal?.aborted) { + return; + } params.onTalkEvent?.({ type: "tool.error", callId, @@ -185,9 +202,13 @@ async function handleMeetingRealtimeConsultToolCall(params: { } await submitMeetingConsultWorkingResponse({ session: params.session, + abortSignal: params.abortSignal, callId, label: params.surface.workingResponseLabel, }); + if (params.abortSignal?.aborted) { + return; + } params.onTalkEvent?.({ type: "tool.progress", callId, @@ -206,10 +227,17 @@ async function handleMeetingRealtimeConsultToolCall(params: { requesterSessionKey: params.requesterSessionKey, args: params.event.args, transcript: params.transcript, + abortSignal: params.abortSignal, }); } catch (error) { + if (params.abortSignal?.aborted) { + return; + } const message = formatErrorMessage(error); await params.session.submitToolResult(callId, { error: message }); + if (params.abortSignal?.aborted) { + return; + } params.onTalkEvent?.({ type: "tool.error", callId, @@ -218,7 +246,13 @@ async function handleMeetingRealtimeConsultToolCall(params: { }); return; } + if (params.abortSignal?.aborted) { + return; + } await params.session.submitToolResult(callId, result); + if (params.abortSignal?.aborted) { + return; + } params.onTalkEvent?.({ type: "tool.result", callId, diff --git a/src/meeting-bot/realtime-engine.test.ts b/src/meeting-bot/realtime-engine.test.ts index 4468b76e31ce..8506099f23f9 100644 --- a/src/meeting-bot/realtime-engine.test.ts +++ b/src/meeting-bot/realtime-engine.test.ts @@ -5,16 +5,22 @@ import type { RealtimeVoiceBridgeCreateRequest, } from "../talk/provider-types.js"; import type { MeetingRealtimeAudioTransport } from "./realtime-audio-transport.js"; -import { startMeetingRealtimeEngine } from "./realtime-engine.js"; +import { + startMeetingRealtimeEngine, + type MeetingRealtimeToolCallParams, +} from "./realtime-engine.js"; type PendingWrite = { resolve: () => void; }; -async function createEngineFixture() { +async function createEngineFixture(options?: { + handleToolCall?: (params: MeetingRealtimeToolCallParams) => Promise; +}) { let callbacks: RealtimeVoiceBridgeCreateRequest | undefined; let onHumanBargeIn: ((audio: Buffer) => boolean) | undefined; const handleBargeIn = vi.fn(); + const submitToolResult = vi.fn(); const bridge: RealtimeVoiceBridge = { acknowledgeMark: vi.fn(), close: vi.fn(), @@ -23,7 +29,7 @@ async function createEngineFixture() { isConnected: vi.fn(() => true), sendAudio: vi.fn(), setMediaTimestamp: vi.fn(), - submitToolResult: vi.fn(), + submitToolResult, }; const provider: RealtimeVoiceProviderPlugin = { id: "test", @@ -66,7 +72,7 @@ async function createEngineFixture() { }, consultAgent: vi.fn(async () => ({ text: "unused" })), fullConfig: {} as never, - handleToolCall: vi.fn(async () => {}), + handleToolCall: options?.handleToolCall ?? vi.fn(async () => {}), logger: { debug: vi.fn(), error: vi.fn(), @@ -94,6 +100,7 @@ async function createEngineFixture() { clearOutput, handle, handleBargeIn, + submitToolResult, releaseWrite(index: number) { const pending = pendingWrites[index]; if (!pending) { @@ -127,6 +134,117 @@ async function createEngineFixture() { } describe("meeting realtime engine output ownership", () => { + it("rearms continuity reset when the provider creates a fresh session before ready", async () => { + const fixture = await createEngineFixture(); + try { + fixture.callbacks.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + fixture.callbacks.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + await vi.waitFor(() => { + expect(fixture.clearOutput).toHaveBeenCalledOnce(); + }); + + fixture.callbacks.onEvent?.({ + direction: "server", + type: "session.created", + }); + fixture.callbacks.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + await vi.waitFor(() => { + expect(fixture.clearOutput).toHaveBeenCalledTimes(2); + }); + } finally { + await fixture.handle.stop(); + } + }); + + it("resets provider continuity without replaying old output or tool work", async () => { + let releaseTool: (() => void) | undefined; + const toolGate = new Promise((resolve) => { + releaseTool = resolve; + }); + const fixture = await createEngineFixture({ + handleToolCall: async ({ session, event, onTalkEvent }) => { + await toolGate; + await session.submitToolResult(event.callId, { text: "stale result" }); + onTalkEvent({ + type: "tool.result", + callId: event.callId, + payload: { name: event.name }, + final: true, + }); + }, + }); + try { + const active = Buffer.from([1]); + const stale = Buffer.from([2]); + const fresh = Buffer.from([3]); + fixture.callbacks.onReady?.(); + fixture.callbacks.onTranscript?.("user", "old turn", true); + fixture.sendOutputAudio(active, "response-1"); + await vi.waitFor(() => { + expect(fixture.writeOutput).toHaveBeenCalledOnce(); + }); + fixture.sendOutputAudio(stale, "response-1"); + fixture.callbacks.onToolCall?.({ + itemId: "item-old", + callId: "call-old", + name: "openclaw_agent_consult", + args: { question: "old work" }, + }); + + fixture.callbacks.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + fixture.callbacks.onEvent?.({ + direction: "client", + type: "session.continuity.reset", + }); + + await vi.waitFor(() => { + expect(fixture.clearOutput).toHaveBeenCalledOnce(); + }); + expect(fixture.handleBargeIn).not.toHaveBeenCalled(); + expect( + fixture.handle + .getHealth() + .recentTalkEvents.filter((event) => event.type === "turn.cancelled"), + ).toHaveLength(1); + + releaseTool?.(); + await Promise.resolve(); + await Promise.resolve(); + expect(fixture.submitToolResult).not.toHaveBeenCalled(); + expect( + fixture.handle.getHealth().recentTalkEvents.some((event) => event.type === "tool.result"), + ).toBe(false); + + fixture.releaseWrite(0); + await vi.waitFor(() => { + expect(fixture.clearOutput).toHaveBeenCalledTimes(2); + }); + expect(fixture.writeOutput).not.toHaveBeenCalledWith(stale); + + fixture.callbacks.onReady?.(); + fixture.sendOutputAudio(fresh, "response-1"); + await vi.waitFor(() => { + expect(fixture.writeOutput).toHaveBeenCalledTimes(2); + }); + expect(fixture.writeOutput).toHaveBeenLastCalledWith(fresh); + fixture.releaseWrite(1); + } finally { + await fixture.handle.stop(); + } + }); + it("serializes transport writes and coalesces queued 20 ms frames", async () => { const fixture = await createEngineFixture(); try { diff --git a/src/meeting-bot/realtime-engine.ts b/src/meeting-bot/realtime-engine.ts index eab0d81acc56..a12b29aba6b8 100644 --- a/src/meeting-bot/realtime-engine.ts +++ b/src/meeting-bot/realtime-engine.ts @@ -26,6 +26,7 @@ import { resolveMeetingRealtimeProvider, } from "./realtime-engine-support.js"; import { createMeetingRealtimeOutputOwner } from "./realtime-output-owner.js"; +import { createMeetingRealtimeToolContinuity } from "./realtime-tool-continuity.js"; export { formatMeetingAgentAudioModelLog, @@ -35,7 +36,6 @@ export { normalizeMeetingTtsPromptText, resolveMeetingRealtimeTranscriptionProvider, } from "./realtime-engine-support.js"; - export type MeetingRuntimePlatform = { /** Adapter-owned identity keeps platform names and log prefixes out of core. */ displayName: string; @@ -98,7 +98,6 @@ const MEETING_REALTIME_OUTPUT_MAX_PENDING_MS = 2_000; const MEETING_REALTIME_OUTPUT_MAX_WRITE_MS = 500; const MEETING_REALTIME_OUTPUT_MAX_PENDING_FRAMES = 256; const MEETING_REALTIME_CANCELLATION_RACE_DETAIL = "Cancellation failed: no active response found"; - export async function startMeetingRealtimeEngine(params: { config: MeetingRealtimeEngineConfig; fullConfig: OpenClawConfig; @@ -135,9 +134,11 @@ export async function startMeetingRealtimeEngine(params: { let outputPendingBytes = 0; let outputPendingFrames = 0; let outputGenerationActive = false; + let continuityResetActive = false; let outputClearTail = Promise.resolve(); const outputQueue: Array<{ audio: Buffer; generation: number }> = []; const outputOwner = createMeetingRealtimeOutputOwner(); + const toolContinuity = createMeetingRealtimeToolContinuity(params.handleToolCall); const outputMaxPendingBytes = meetingOutputBytesPerMs(params.config.chrome.audioFormat) * MEETING_REALTIME_OUTPUT_MAX_PENDING_MS; @@ -160,6 +161,7 @@ export async function startMeetingRealtimeEngine(params: { outputOwner.reset(); outputClearAfterActive = false; invalidateOutputQueue(); + toolContinuity.reset("meeting realtime stopped"); } if (stopPromise) { await stopPromise; @@ -549,6 +551,30 @@ export async function startMeetingRealtimeEngine(params: { } }, onEvent: (event) => { + if (event.direction === "server" && event.type === "session.created") { + continuityResetActive = false; + } + if (event.direction === "client" && event.type === "session.continuity.reset") { + if (continuityResetActive) { + return; + } + continuityResetActive = true; + realtimeReady = false; + outputOwner.reset(); + outputGenerationActive = false; + toolContinuity.reset(event.type); + const turnId = harness.talk.activeTurnId; + invalidateOutputPlayback(); + harness.flushOutput(clearOutputPlayback); + harness.finishOutputAudio(event.type); + if (turnId) { + harness.talk.cancelTurn({ + turnId, + payload: { ...outputTalkPayload, reason: event.type }, + }); + } + return; + } outputOwner.noteEvent(event); if (event.type === "input_audio_buffer.speech_started") { harness.ensureTurn(); @@ -600,26 +626,18 @@ export async function startMeetingRealtimeEngine(params: { ); } }, - onToolCall: (event, session) => { - harness.emit({ - type: "tool.call", - turnId: harness.ensureTurn(), - itemId: event.itemId, - callId: event.callId, - payload: { name: event.name, args: event.args }, - }); - const turnId = harness.ensureTurn(); - return params.handleToolCall({ - strategy, + onToolCall: (event, session) => + toolContinuity.run({ session, - event, - meetingSessionId: params.meetingSessionId, - requesterSessionKey: params.requesterSessionKey, - transcript: harness.transcript, - onTalkEvent: (inputLocal) => - harness.emit({ ...inputLocal, turnId: inputLocal.turnId ?? turnId }), - }); - }, + call: { + strategy, + event, + meetingSessionId: params.meetingSessionId, + requesterSessionKey: params.requesterSessionKey, + transcript: harness.transcript, + }, + harness, + }), onError: (error) => { harness.emit({ type: "session.error", @@ -644,6 +662,7 @@ export async function startMeetingRealtimeEngine(params: { }, onReady: () => { realtimeReady = true; + continuityResetActive = false; harness.emit({ type: "session.ready", payload: outputTalkPayload, diff --git a/src/meeting-bot/realtime-tool-continuity.ts b/src/meeting-bot/realtime-tool-continuity.ts new file mode 100644 index 000000000000..8c97ed36a951 --- /dev/null +++ b/src/meeting-bot/realtime-tool-continuity.ts @@ -0,0 +1,82 @@ +import type { RealtimeVoiceToolCallEvent } from "../talk/provider-types.js"; +import type { RealtimeVoiceSessionHarness } from "../talk/realtime-session-harness.js"; +import type { RealtimeVoiceBridgeSession } from "../talk/session-runtime.js"; +import type { TalkEventInput } from "../talk/talk-events.js"; + +type MeetingRealtimeToolContinuityCall = { + session: RealtimeVoiceBridgeSession; + event: RealtimeVoiceToolCallEvent; + onTalkEvent: (event: TalkEventInput) => void; +}; + +const meetingRealtimeToolAbortSignals = new WeakMap(); + +export function readMeetingRealtimeToolAbortSignal( + session: RealtimeVoiceBridgeSession, +): AbortSignal | undefined { + return meetingRealtimeToolAbortSignals.get(session); +} + +export function createMeetingRealtimeToolContinuity< + TCall extends MeetingRealtimeToolContinuityCall, +>(handleToolCall: (call: TCall) => Promise) { + let epoch = 0; + const activeControllers = new Set(); + + // Reset invalidates provider submissions and Talk events together so late + // async completions cannot leak into the replacement provider generation. + const reset = (reason: string) => { + epoch += 1; + for (const controller of activeControllers) { + controller.abort(reason); + } + activeControllers.clear(); + }; + + const run = (params: { + session: RealtimeVoiceBridgeSession; + call: Omit; + harness: Pick; + }): Promise => { + const callEpoch = epoch; + const controller = new AbortController(); + activeControllers.add(controller); + const isActive = () => !controller.signal.aborted && callEpoch === epoch; + const turnId = params.harness.ensureTurn(); + params.harness.emit({ + type: "tool.call", + turnId, + itemId: params.call.event.itemId, + callId: params.call.event.callId, + payload: { name: params.call.event.name, args: params.call.event.args }, + }); + const guardedSession = Object.create(params.session) as RealtimeVoiceBridgeSession; + meetingRealtimeToolAbortSignals.set(guardedSession, controller.signal); + guardedSession.submitToolResult = (callId, result, options) => { + if (!isActive()) { + return; + } + return params.session.submitToolResult(callId, result, options); + }; + return handleToolCall({ + ...params.call, + session: guardedSession, + onTalkEvent: (event) => { + if (isActive()) { + params.harness.emit({ ...event, turnId: event.turnId ?? turnId }); + } + }, + } as TCall) + .catch((error: unknown) => { + if (isActive()) { + throw error; + } + }) + .finally(() => { + meetingRealtimeToolAbortSignals.delete(guardedSession); + activeControllers.delete(controller); + }); + }; + + return { reset, run }; +}