merge(voice): preserve realtime reconnect continuity (#117599)

This commit is contained in:
Vincent Koc
2026-08-02 05:03:59 +08:00
17 changed files with 1603 additions and 117 deletions

View File

@@ -44,7 +44,8 @@ export type TestRealtimeBridgeParams = {
cfg?: unknown;
instructions?: string;
interruptResponseOnInputAudio?: boolean;
onEvent?: (event: { detail?: string; direction: "server"; type: string }) => void;
onEvent?: (event: { detail?: string; direction: "client" | "server"; type: string }) => void;
onReady?: () => void;
onToolCall?: (
event: { args: unknown; callId: string; itemId: string; name: string },
session: unknown,

View File

@@ -3263,6 +3263,98 @@ describe("DiscordVoiceManager", () => {
expect(player.stop).toHaveBeenCalledTimes(stopCallsAfterControl + 1);
});
it("drops stale active-run control after provider continuity reset", async () => {
let resolveOldControl: ((result: RealtimeVoiceAgentControlResult) => void) | undefined;
controlRealtimeVoiceAgentRunMock
.mockReturnValueOnce(
new Promise((resolve) => {
resolveOldControl = resolve;
}),
)
.mockResolvedValueOnce({
ok: true,
mode: "cancel",
sessionKey: "discord:g1:c1",
sessionId: "embedded-fresh",
active: true,
aborted: true,
message: "Fresh control result.",
speak: true,
show: true,
suppress: false,
});
const manager = createAgentProxyManager();
await manager.join({ guildId: "g1", channelId: "1001" });
const bridgeParams = lastRealtimeBridgeParams();
bridgeParams?.onTranscript?.("user", "cancel that", true);
await vi.waitFor(() => expect(controlRealtimeVoiceAgentRunMock).toHaveBeenCalledTimes(1));
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
resolveOldControl?.({
ok: true,
mode: "cancel",
sessionKey: "discord:g1:c1",
sessionId: "embedded-old",
active: true,
aborted: true,
message: "Stale control result.",
speak: true,
show: true,
suppress: false,
});
await Promise.resolve();
await Promise.resolve();
expectUserMessageNotIncludes("Stale control result.");
expect(realtimeSessionMock.handleBargeIn).not.toHaveBeenCalled();
bridgeParams?.onReady?.();
bridgeParams?.onTranscript?.("user", "stop that", true);
await vi.waitFor(() => expectUserMessageIncludes("Fresh control result."));
expect(realtimeSessionMock.handleBargeIn).toHaveBeenCalledTimes(1);
});
it("replaces stale talkback work across provider continuity reset", async () => {
let resolveOldTalkback: ((result: { payloads: Array<{ text: string }> }) => void) | undefined;
agentCommandMock
.mockReturnValueOnce(
new Promise((resolve) => {
resolveOldTalkback = resolve;
}),
)
.mockResolvedValueOnce({ payloads: [{ text: "fresh talkback" }] });
const manager = createManager({
groupPolicy: "open",
voice: {
enabled: true,
mode: "agent-proxy",
realtime: { provider: "openai", debounceMs: 1, toolPolicy: "none" },
},
});
await manager.join({ guildId: "g1", channelId: "1001" });
const entry = getSessionEntry(manager);
const bridgeParams = lastRealtimeBridgeParams();
beginSpeakerTurn(entry);
await emitFinalRealtimeUserTranscript(bridgeParams, "old question");
await vi.waitFor(() => expect(agentCommandMock).toHaveBeenCalledTimes(1));
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onReady?.();
beginSpeakerTurn(entry);
await emitFinalRealtimeUserTranscript(bridgeParams, "fresh question");
await vi.waitFor(() => expect(agentCommandMock).toHaveBeenCalledTimes(2));
await vi.waitFor(() => expectUserMessageIncludes("fresh talkback"));
resolveOldTalkback?.({ payloads: [{ text: "stale talkback" }] });
await Promise.resolve();
await Promise.resolve();
expectUserMessageNotIncludes("stale talkback");
});
it("preserves realtime forced consults when no active run accepts steering", async () => {
agentCommandMock.mockResolvedValueOnce({ payloads: [{ text: "normal answer" }] });
const manager = createAgentProxyManager();
@@ -3418,6 +3510,203 @@ describe("DiscordVoiceManager", () => {
expectUserMessageIncludes("wake answer");
});
it("does not carry partial wake-name state across provider continuity resets", async () => {
const { entry, bridgeParams } = await createWakeNameFixture();
const wakeAckCount = () =>
sentUserMessages().filter((message) => message.includes('Answer: "Yeah."')).length;
beginSpeakerTurn(entry);
bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" });
bridgeParams?.onTranscript?.("user", "Hey, Mol", false);
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onTranscript?.("user", "ty", false);
expect(wakeAckCount()).toBe(0);
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onReady?.();
bridgeParams?.onTranscript?.("user", "Hey, Molty", false);
expect(wakeAckCount()).toBe(1);
bridgeParams?.onEvent?.({ direction: "server", type: "response.done" });
});
it("preserves the wake-name acknowledgement across provider continuity resets", async () => {
const { entry, bridgeParams } = await createWakeNameFixture();
const wakeAckCount = () =>
sentUserMessages().filter((message) => message.includes('Answer: "')).length;
beginSpeakerTurn(entry);
bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" });
bridgeParams?.onTranscript?.("user", "Hey, Molty", false);
expect(wakeAckCount()).toBe(1);
bridgeParams?.onEvent?.({ direction: "server", type: "response.done" });
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onReady?.();
bridgeParams?.onTranscript?.("user", "Hey, Molty", false);
expect(wakeAckCount()).toBe(1);
bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" });
bridgeParams?.onTranscript?.("user", "Hey, Molty", false);
expect(wakeAckCount()).toBe(2);
});
it("replays zero-audio exact speech once after provider continuity reset", async () => {
agentCommandMock
.mockResolvedValueOnce({ payloads: [{ text: "first answer" }] })
.mockResolvedValueOnce({ payloads: [{ text: "second answer" }] });
const manager = createAgentProxyManager();
await manager.join({ guildId: "g1", channelId: "1001" });
const entry = getSessionEntry(manager);
const player = getLastAudioPlayer();
const bridgeParams = lastRealtimeBridgeParams();
beginSpeakerTurn(entry);
await emitFinalRealtimeUserTranscript(bridgeParams, "first question");
await vi.waitFor(() => expectUserMessageIncludes("first answer"));
beginSpeakerTurn(entry);
await emitFinalRealtimeUserTranscript(bridgeParams, "second question");
expectUserMessageNotIncludes("second answer");
const stopCallsBeforeReset = player.stop.mock.calls.length;
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
expectUserMessageNotIncludes("second answer");
expect(player.stop).toHaveBeenCalledTimes(stopCallsBeforeReset + 1);
expect(realtimeSessionMock.handleBargeIn).not.toHaveBeenCalled();
expect(realtimeSessionMock.close).not.toHaveBeenCalled();
bridgeParams?.onReady?.();
expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength(
2,
);
expectUserMessageNotIncludes("second answer");
bridgeParams?.onEvent?.({ direction: "server", type: "response.done" });
expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength(
1,
);
});
it("replays exact speech buffered below playback preroll after continuity reset", async () => {
agentCommandMock
.mockResolvedValueOnce({ payloads: [{ text: "first answer" }] })
.mockResolvedValueOnce({ payloads: [{ text: "second answer" }] });
const manager = createAgentProxyManager();
await manager.join({ guildId: "g1", channelId: "1001" });
const entry = getSessionEntry(manager);
const player = getLastAudioPlayer();
const bridgeParams = lastRealtimeBridgeParams();
beginSpeakerTurn(entry);
await emitFinalRealtimeUserTranscript(bridgeParams, "first question");
await vi.waitFor(() => expectUserMessageIncludes("first answer"));
bridgeParams?.audioSink?.sendAudio(Buffer.alloc(480));
expect(player.play).not.toHaveBeenCalled();
beginSpeakerTurn(entry);
await emitFinalRealtimeUserTranscript(bridgeParams, "second question");
expectUserMessageNotIncludes("second answer");
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onReady?.();
expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength(
2,
);
expectUserMessageNotIncludes("second answer");
bridgeParams?.onEvent?.({ direction: "server", type: "response.done" });
expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength(
1,
);
});
it("does not replay exact speech after Discord playback starts", async () => {
agentCommandMock
.mockResolvedValueOnce({ payloads: [{ text: "first answer" }] })
.mockResolvedValueOnce({ payloads: [{ text: "second answer" }] });
const manager = createAgentProxyManager();
await manager.join({ guildId: "g1", channelId: "1001" });
const entry = getSessionEntry(manager);
const player = getLastAudioPlayer();
const bridgeParams = lastRealtimeBridgeParams();
beginSpeakerTurn(entry);
await emitFinalRealtimeUserTranscript(bridgeParams, "first question");
await vi.waitFor(() => expectUserMessageIncludes("first answer"));
for (let index = 0; index < 50; index += 1) {
bridgeParams?.audioSink?.sendAudio(Buffer.alloc(480));
}
expect(player.play).toHaveBeenCalledOnce();
beginSpeakerTurn(entry);
await emitFinalRealtimeUserTranscript(bridgeParams, "second question");
expectUserMessageNotIncludes("second answer");
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
bridgeParams?.onReady?.();
expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength(
1,
);
expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength(
1,
);
});
it("drops stale native consult delivery after provider continuity reset", async () => {
let resolveOld: ((result: { payloads: Array<{ text: string }> }) => void) | undefined;
agentCommandMock
.mockReturnValueOnce(
new Promise((resolve) => {
resolveOld = resolve;
}),
)
.mockResolvedValueOnce({ payloads: [{ text: "fresh answer" }] });
const manager = createAgentProxyManager();
await manager.join({ guildId: "g1", channelId: "1001" });
const entry = getSessionEntry(manager);
const bridgeParams = lastRealtimeBridgeParams();
beginSpeakerTurn(entry);
const oldSubmission = bridgeParams?.onToolCall?.(
{
itemId: "item-old",
callId: "call-old",
name: "openclaw_agent_consult",
args: { question: "same question" },
},
realtimeSessionMock,
);
await Promise.resolve();
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
resolveOld?.({ payloads: [{ text: "stale answer" }] });
await oldSubmission;
expect(
realtimeSessionMock.submitToolResult.mock.calls.some(([callId]) => callId === "call-old"),
).toBe(false);
bridgeParams?.onReady?.();
beginSpeakerTurn(entry);
await bridgeParams?.onToolCall?.(
{
itemId: "item-fresh",
callId: "call-fresh",
name: "openclaw_agent_consult",
args: { question: "same question" },
},
realtimeSessionMock,
);
expect(realtimeSessionMock.submitToolResult).toHaveBeenCalledWith("call-fresh", {
text: "fresh answer",
});
});
it("treats a bare wake name as an activation for the next realtime transcript", async () => {
agentCommandMock.mockResolvedValueOnce({ payloads: [{ text: "follow-up answer" }] });
const onUtterance = vi.fn();

View File

@@ -10,6 +10,7 @@ import {
buildRealtimeVoiceAgentConsultPolicyInstructions,
classifySkippableRealtimeVoiceConsultTranscript,
controlRealtimeVoiceAgentRun,
createRealtimeVoiceAgentTalkbackQueue,
createRealtimeVoiceSessionHarness,
createRealtimeVoiceTurnContextTracker,
matchRealtimeVoiceActivationName,
@@ -26,6 +27,7 @@ import {
type RealtimeVoiceBridgeEvent,
type RealtimeVoiceAgentConsultToolPolicy,
type RealtimeVoiceAgentControlResult,
type RealtimeVoiceAgentTalkbackQueue,
type RealtimeVoiceBridgeSession,
type RealtimeVoiceProviderConfig,
type RealtimeVoiceToolCallEvent,
@@ -133,8 +135,10 @@ type RecentAgentProxyConsultResult =
type AgentProxyConsultState = {
speaker: DiscordRealtimeSpeakerContext;
providerEpoch: number;
handledByForcedPlayback?: boolean;
providerDelivery?: Promise<boolean>;
settleProviderDelivery?: (accepted: boolean) => void;
promise?: Promise<string>;
result?: RecentAgentProxyConsultResult;
};
@@ -324,6 +328,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
private bridge: RealtimeVoiceBridgeSession | null = null;
private outputStream: PassThrough | null = null;
private readonly harness: RealtimeVoiceSessionHarness<AgentProxyConsultState>;
private talkback: RealtimeVoiceAgentTalkbackQueue;
private stopped = false;
private consultToolPolicy: RealtimeVoiceAgentConsultToolPolicy = "safe-read-only";
private consultToolsAllow: string[] | undefined;
@@ -347,6 +352,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
private queuedExactSpeechMessages: string[] = [];
private exactSpeechResponseActive = false;
private exactSpeechAudioStarted = false;
private activeExactSpeechMessage: string | undefined;
private bridgeReady = false;
private providerGenerationObserved = false;
private providerContinuityEpoch = 0;
private partialUserTranscript = "";
private wakeNameAckedForTurn = false;
private wakeNameAckIndex = 0;
@@ -397,34 +406,40 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
outputAudioDelta: discordRealtimeTalkPayload,
outputAudioDone: discordRealtimeTalkPayload,
},
talkback: {
debounceMs: this.realtimeConfig?.debounceMs ?? DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS,
logger,
logPrefix: "[discord] realtime agent",
responseStyle: "Brief, natural spoken answer for a Discord voice channel.",
fallbackText: DISCORD_REALTIME_FALLBACK_TEXT,
consult: async ({ question, responseStyle, metadata }) => {
const context = isDiscordRealtimeSpeakerContext(metadata) ? metadata : undefined;
return {
text: await this.runAgentTurn({
context,
message: formatVoiceIngressPrompt(
[question, responseStyle ? `Spoken style: ${responseStyle}` : undefined]
.filter(Boolean)
.join("\n\n"),
context?.speakerLabel ?? "Discord voice speaker",
),
}),
};
},
deliver: (text) => this.enqueueExactSpeechMessage(text),
},
forcedConsults: {
limit: DISCORD_REALTIME_RECENT_AGENT_PROXY_CONSULT_LIMIT,
nativeDedupeMs: DISCORD_REALTIME_RECENT_AGENT_PROXY_CONSULT_TTL_MS,
questionsMatch: matchRealtimeVoiceConsultQuestions,
},
});
this.talkback = this.createTalkbackQueue();
}
private createTalkbackQueue(): RealtimeVoiceAgentTalkbackQueue {
const providerEpoch = this.providerContinuityEpoch;
return createRealtimeVoiceAgentTalkbackQueue({
debounceMs: this.realtimeConfig?.debounceMs ?? DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS,
isStopped: () => this.stopped || providerEpoch !== this.providerContinuityEpoch,
logger,
logPrefix: "[discord] realtime agent",
responseStyle: "Brief, natural spoken answer for a Discord voice channel.",
fallbackText: DISCORD_REALTIME_FALLBACK_TEXT,
consult: async ({ question, responseStyle, metadata }) => {
const context = isDiscordRealtimeSpeakerContext(metadata) ? metadata : undefined;
return {
text: await this.runAgentTurn({
context,
message: formatVoiceIngressPrompt(
[question, responseStyle ? `Spoken style: ${responseStyle}` : undefined]
.filter(Boolean)
.join("\n\n"),
context?.speakerLabel ?? "Discord voice speaker",
),
}),
};
},
deliver: (text) => this.enqueueExactSpeechMessage(text),
});
}
async connect(): Promise<void> {
@@ -493,10 +508,14 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
audioSink: {
isOpen: () => !this.stopped,
sendAudio: (audio) => this.sendOutputAudio(audio),
clearAudio: () =>
this.harness.flushOutput(() => this.clearOutputAudio("provider-clear-audio")),
clearAudio: () => {
this.markProviderGenerationObserved();
this.harness.flushOutput(() => this.clearOutputAudio("provider-clear-audio"));
},
},
onTranscript: (role, text, isFinal) => {
this.markProviderGenerationObserved();
const providerEpoch = this.providerContinuityEpoch;
if (isFinal && text.trim()) {
logger.info(
`discord voice: realtime ${role} transcript (${text.length} chars): ${formatVoiceLogPreview(text)}`,
@@ -512,11 +531,28 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
this.handlePartialUserTranscript(text);
return;
}
void this.handleFinalUserTranscript(text, { usesRealtimeAgentHandoff });
void this.handleFinalUserTranscript(text, {
providerEpoch,
usesRealtimeAgentHandoff,
});
},
onToolCall: (event, session) => {
this.markProviderGenerationObserved();
return this.handleToolCall(event, session);
},
onReady: () => {
this.markProviderGenerationObserved();
this.bridgeReady = true;
this.drainQueuedExactSpeechMessages("provider-ready");
},
onToolCall: (event, session) => this.handleToolCall(event, session),
onEvent: (event) => {
if (!(event.direction === "client" && event.type === "session.continuity.reset")) {
this.markProviderGenerationObserved();
}
const detail = event.detail ? ` ${event.detail}` : "";
if (event.direction === "client" && event.type === "session.continuity.reset") {
this.resetProviderContinuity(event.type);
}
if (event.direction === "server" && event.type === "input_audio_buffer.speech_started") {
this.resetPartialWakeNameTracking();
}
@@ -571,6 +607,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
const voiceSdk = loadDiscordVoiceSdk();
this.params.entry.player.on(voiceSdk.AudioPlayerStatus.Idle, this.playerIdleHandler);
await this.bridge.connect();
// Some provider/test bridges do not expose an explicit ready callback.
this.markProviderGenerationObserved();
this.bridgeReady = true;
this.drainQueuedExactSpeechMessages("provider-connected");
logger.info(
`discord voice: realtime bridge ready mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"}`,
);
@@ -578,13 +618,18 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
close(): void {
this.stopped = true;
this.bridgeReady = false;
this.providerContinuityEpoch += 1;
this.outputBackpressure = undefined;
this.flushSuppressedRealtimeErrors();
this.clearProviderConsultState();
this.talkback.close();
this.harness.close();
this.speakerTurns.clear();
this.queuedExactSpeechMessages = [];
this.exactSpeechResponseActive = false;
this.exactSpeechAudioStarted = false;
this.activeExactSpeechMessage = undefined;
this.resetPartialWakeNameTracking();
this.pendingWakeNameFollowup = undefined;
this.clearOutputAudio("session-close");
@@ -733,6 +778,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
}
private sendOutputAudio(realtimePcm24kMono: Buffer): void {
this.markProviderGenerationObserved();
if (this.stopped || this.outputBackpressure) {
return;
}
@@ -948,7 +994,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
if (this.stopped || !text.trim()) {
return;
}
if (this.exactSpeechResponseActive || this.hasInterruptibleOutputAudio()) {
if (!this.bridgeReady || this.exactSpeechResponseActive || this.hasInterruptibleOutputAudio()) {
this.queuedExactSpeechMessages.push(text);
logger.info(
`discord voice: realtime exact speech queued guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()}`,
@@ -964,6 +1010,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
}
this.exactSpeechResponseActive = true;
this.exactSpeechAudioStarted = false;
this.activeExactSpeechMessage = text;
this.bridge?.sendUserMessage(buildDiscordSpeakExactUserMessage(text));
}
@@ -983,7 +1030,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
logger.info(
`discord voice: realtime wake-name ack canonical=${result.activationName} heard=${result.heardName} match=${result.match} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`,
);
this.sendExactSpeechMessage(ack ?? "Yeah.");
this.enqueueExactSpeechMessage(ack ?? "Yeah.");
}
private speakControlResult(text: string): void {
@@ -1001,7 +1048,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
sentAt: Date.now(),
assistantTranscriptCount: 0,
};
this.sendExactSpeechMessage(trimmed);
this.enqueueExactSpeechMessage(trimmed);
}
private suppressDuplicateControlSpeech(text: string): void {
@@ -1034,6 +1081,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
}
this.exactSpeechResponseActive = false;
this.exactSpeechAudioStarted = false;
this.activeExactSpeechMessage = undefined;
if (options?.drain === false) {
return;
}
@@ -1043,6 +1091,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
private drainQueuedExactSpeechMessages(reason: string): void {
if (
this.stopped ||
!this.bridgeReady ||
this.exactSpeechResponseActive ||
this.queuedExactSpeechMessages.length === 0 ||
this.hasInterruptibleOutputAudio()
@@ -1123,9 +1172,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
event: RealtimeVoiceToolCallEvent,
session: RealtimeVoiceBridgeSession,
): Promise<void> {
const providerEpoch = this.providerContinuityEpoch;
const callId = event.callId || event.itemId || "unknown";
if (event.name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) {
await this.handleAgentControlToolCall(event, session, callId);
await this.handleAgentControlToolCall(event, session, callId, providerEpoch);
return;
}
if (event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
@@ -1223,11 +1273,17 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
try {
text = await promise;
} catch (error) {
if (providerEpoch !== this.providerContinuityEpoch) {
return;
}
const message = formatErrorMessage(error);
logger.warn(`discord voice: realtime consult failed call=${callId || "unknown"}: ${message}`);
await session.submitToolResult(callId, { error: message });
return;
}
if (providerEpoch !== this.providerContinuityEpoch) {
return;
}
logger.info(
`discord voice: realtime consult answer (${text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(text)}`,
);
@@ -1238,6 +1294,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
event: RealtimeVoiceToolCallEvent,
session: RealtimeVoiceBridgeSession,
callId: string,
providerEpoch: number,
): Promise<void> {
let result: RealtimeVoiceAgentControlResult;
try {
@@ -1248,9 +1305,15 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
mode: parsed.mode,
});
} catch (error) {
if (providerEpoch !== this.providerContinuityEpoch) {
return;
}
await session.submitToolResult(callId, { error: formatErrorMessage(error) });
return;
}
if (providerEpoch !== this.providerContinuityEpoch) {
return;
}
this.logAgentControlResult(result);
await session.submitToolResult(callId, result);
}
@@ -1273,7 +1336,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
private async handleFinalUserTranscript(
text: string,
params: { usesRealtimeAgentHandoff: boolean },
params: { providerEpoch: number; usesRealtimeAgentHandoff: boolean },
): Promise<void> {
const trimmed = text.trim();
if (!trimmed) {
@@ -1316,15 +1379,24 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
usesAgentProxy && params.usesRealtimeAgentHandoff
? this.prepareForcedAgentProxyConsult(acceptedText, forcedSpeakerContext)
: undefined;
const control = await maybeControlDiscordVoiceAgentRun({
entry: this.params.entry,
text: acceptedText,
}).catch((error: unknown) => {
let control: Awaited<ReturnType<typeof maybeControlDiscordVoiceAgentRun>> | undefined;
try {
control = await maybeControlDiscordVoiceAgentRun({
entry: this.params.entry,
text: acceptedText,
});
} catch (error) {
if (params.providerEpoch !== this.providerContinuityEpoch) {
return;
}
logger.warn(
`discord voice: realtime active-run control failed; falling back to normal transcript handling: ${formatErrorMessage(error)}`,
);
return undefined;
});
control = undefined;
}
if (params.providerEpoch !== this.providerContinuityEpoch) {
return;
}
if (control?.handled) {
if (pendingForcedConsult) {
this.harness.forcedConsults.remove(pendingForcedConsult);
@@ -1345,7 +1417,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
}
return;
}
this.harness.talkback?.enqueue(
this.talkback.enqueue(
acceptedText,
forcedSpeakerContext ?? this.consumePendingSpeakerContext(),
);
@@ -1372,6 +1444,52 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
this.wakeNameAckedForTurn = false;
}
private markProviderGenerationObserved(): void {
this.providerGenerationObserved = true;
}
private resetProviderContinuity(reason: string): void {
if (!this.providerGenerationObserved) {
return;
}
this.providerGenerationObserved = false;
this.bridgeReady = false;
this.providerContinuityEpoch += 1;
this.talkback.close();
this.talkback = this.createTalkbackQueue();
this.outputBackpressure = undefined;
this.partialUserTranscript = "";
this.pendingWakeNameFollowup = undefined;
this.lastControlSpeech = undefined;
this.clearProviderConsultState();
const replayExactSpeech =
this.exactSpeechResponseActive && !this.harness.outputActivity.snapshot().playbackStarted
? this.activeExactSpeechMessage
: undefined;
this.exactSpeechResponseActive = false;
this.exactSpeechAudioStarted = false;
this.activeExactSpeechMessage = undefined;
if (replayExactSpeech) {
this.queuedExactSpeechMessages.unshift(replayExactSpeech);
}
this.harness.flushOutput(() => this.clearOutputAudio(reason));
this.harness.finishOutputAudio(reason);
}
private clearProviderConsultState(): void {
for (const handle of this.harness.forcedConsults.handles()) {
const state = handle.context;
if (!state) {
continue;
}
state.handledByForcedPlayback = false;
state.settleProviderDelivery?.(false);
state.settleProviderDelivery = undefined;
state.providerDelivery = undefined;
}
this.harness.forcedConsults.clear();
}
private resolveWakeNameTranscript(
text: string,
requireWakeName: boolean,
@@ -1477,7 +1595,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
return undefined;
}
return this.harness.forcedConsults.prepare(question, {
context: { speaker: context },
context: { speaker: context, providerEpoch: this.providerContinuityEpoch },
});
}
@@ -1498,7 +1616,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
}
const context = state.speaker;
const { question } = pending;
if (this.stopped) {
if (this.stopped || state.providerEpoch !== this.providerContinuityEpoch) {
this.harness.forcedConsults.markCancelled(pending);
return;
}
@@ -1523,6 +1641,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
this.setRecentAgentProxyConsultPromise(pending, promise);
const text = await promise;
await state.providerDelivery;
if (state.providerEpoch !== this.providerContinuityEpoch) {
return;
}
logger.info(
`discord voice: realtime forced agent consult answer (${text.length} chars) elapsedMs=${Date.now() - startedAt} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}: ${formatVoiceLogPreview(text)}`,
);
@@ -1531,6 +1652,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
}
} catch (error) {
await state.providerDelivery;
if (state.providerEpoch !== this.providerContinuityEpoch) {
return;
}
logger.warn(
`discord voice: realtime forced agent consult failed elapsedMs=${Date.now() - startedAt}: ${formatErrorMessage(error)}`,
);
@@ -1612,7 +1736,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
options: { id?: string; started?: boolean } = {},
): AgentProxyConsultHandle {
const handle = this.harness.forcedConsults.prepare(question, {
context: { speaker: context },
context: { speaker: context, providerEpoch: this.providerContinuityEpoch },
...(options.id ? { id: options.id } : {}),
});
if (!handle) {
@@ -1636,10 +1760,16 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
state.promise = promise;
void promise
.then((text) => {
if (state.providerEpoch !== this.providerContinuityEpoch) {
return;
}
state.result = { status: "fulfilled", text };
this.harness.forcedConsults.markDelivered(recent);
})
.catch((error: unknown) => {
if (state.providerEpoch !== this.providerContinuityEpoch) {
return;
}
state.result = { status: "rejected", error: formatErrorMessage(error) };
this.harness.forcedConsults.markDelivered(recent);
});
@@ -1674,6 +1804,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
if (!state) {
return false;
}
if (state.providerEpoch !== this.providerContinuityEpoch) {
return true;
}
const providerOwnsDelivery = Boolean(
state.handledByForcedPlayback &&
state.promise &&
@@ -1686,15 +1819,22 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
// the local success/fallback path instead of losing the answer entirely.
state.providerDelivery = new Promise<boolean>((resolve) => {
resolveProviderDelivery = resolve;
state.settleProviderDelivery = resolve;
});
}
const submitAlreadyDelivered = async (): Promise<void> => {
if (state.providerEpoch !== this.providerContinuityEpoch) {
return;
}
await this.submitTerminalRealtimeToolResult(callId, session, {
status: "already_delivered",
message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it.",
});
};
const submitResult = async (result: RecentAgentProxyConsultResult): Promise<void> => {
if (state.providerEpoch !== this.providerContinuityEpoch) {
return;
}
if (state.handledByForcedPlayback && !providerOwnsDelivery) {
await submitAlreadyDelivered();
return;
@@ -1720,6 +1860,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
);
if (state.handledByForcedPlayback && !providerOwnsDelivery) {
await state.promise.catch(() => undefined);
if (state.providerEpoch !== this.providerContinuityEpoch) {
return true;
}
await submitAlreadyDelivered();
return true;
}
@@ -1729,13 +1872,18 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
} catch (error) {
result = { status: "rejected", error: formatErrorMessage(error) };
}
if (state.providerEpoch !== this.providerContinuityEpoch) {
return true;
}
try {
await submitResult(result);
if (providerOwnsDelivery) {
state.handledByForcedPlayback = false;
state.settleProviderDelivery = undefined;
resolveProviderDelivery?.(true);
}
} catch (error) {
state.settleProviderDelivery = undefined;
resolveProviderDelivery?.(false);
throw error;
}

View File

@@ -815,6 +815,9 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
serverContent: { inputTranscription: { text: "Before " } },
});
firstSession.onmessage({
sessionResumptionUpdate: { newHandle: "unconfirmed-handle" },
});
firstSession.onclose({ code: 1011, reason: "temporary" });
await vi.advanceTimersByTimeAsync(250);
lastConnectParams().callbacks.onmessage({
@@ -827,6 +830,46 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
]);
});
it("preserves continuity when resumability recovers before reconnect", async () => {
vi.useFakeTimers();
const provider = buildGoogleRealtimeVoiceProvider();
const onEvent = vi.fn();
const onTranscript = vi.fn();
const bridge = provider.createBridge({
providerConfig: { apiKey: "gemini-key" },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
onEvent,
onTranscript,
});
await bridge.connect();
const firstSession = lastConnectParams().callbacks;
firstSession.onmessage({
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
serverContent: { inputTranscription: { text: "Before " } },
});
firstSession.onmessage({
sessionResumptionUpdate: { resumable: false },
});
firstSession.onmessage({
sessionResumptionUpdate: { resumable: true, newHandle: "resume-2" },
});
expect(onEvent).not.toHaveBeenCalled();
firstSession.onclose({ code: 1011, reason: "temporary" });
await vi.advanceTimersByTimeAsync(250);
lastConnectParams().callbacks.onmessage({
serverContent: { inputTranscription: { text: "after", finished: true } },
});
expect(lastConnectParams().config.sessionResumption).toEqual({ handle: "resume-2" });
expect(onEvent).not.toHaveBeenCalled();
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
["user", "Before after", true],
]);
});
it("drops unfinished hypotheses when a new session has no continuity", async () => {
vi.useFakeTimers();
const provider = buildGoogleRealtimeVoiceProvider();
@@ -854,28 +897,37 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
]);
});
it("reconnects unexpected Google Live closes with the latest resumption handle", async () => {
it.each([undefined, "invalidated-handle"])("invalidates resume handles %#", async (newHandle) => {
vi.useFakeTimers();
try {
const provider = buildGoogleRealtimeVoiceProvider();
const onClose = vi.fn();
const onError = vi.fn();
const onEvent = vi.fn();
const onTranscript = vi.fn();
const bridge = provider.createBridge({
providerConfig: { apiKey: "gemini-key" },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
onClose,
onError,
onEvent,
onTranscript,
});
await bridge.connect();
lastConnectParams().callbacks.onmessage({
setupComplete: { sessionId: "session-1" },
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
serverContent: {
inputTranscription: { text: "Previous caller " },
outputTranscription: { text: "Previous assistant " },
},
});
lastConnectParams().callbacks.onmessage({
sessionResumptionUpdate: { resumable: false },
sessionResumptionUpdate: { resumable: false, ...(newHandle ? { newHandle } : {}) },
});
expect(onEvent).not.toHaveBeenCalled();
lastConnectParams().callbacks.onclose({
code: 1011,
reason: "temporary upstream close",
@@ -883,13 +935,38 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
});
expect(onClose).not.toHaveBeenCalled();
expect(onEvent).toHaveBeenCalledOnce();
expect(onEvent).toHaveBeenCalledWith({
direction: "client",
type: "session.continuity.reset",
});
const error = requireFirstError(onError);
expect(error.message).toContain("reconnecting 1/3");
await vi.advanceTimersByTimeAsync(250);
expect(connectMock).toHaveBeenCalledTimes(2);
expect(lastConnectParams().config.sessionResumption).toEqual({ handle: "resume-1" });
expect(lastConnectParams().config.sessionResumption).toEqual({});
expect(onEvent).toHaveBeenCalledOnce();
const resetOrder = onEvent.mock.invocationCallOrder[0];
const reconnectOrder = connectMock.mock.invocationCallOrder[1];
if (resetOrder === undefined || reconnectOrder === undefined) {
throw new Error("expected continuity reset before reconnect");
}
expect(resetOrder).toBeLessThan(reconnectOrder);
lastConnectParams().callbacks.onmessage({
setupComplete: { sessionId: "session-2" },
serverContent: {
inputTranscription: { text: "Fresh caller", finished: true },
outputTranscription: { text: "Fresh assistant", finished: true },
},
});
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
["user", "Fresh caller", true],
["assistant", "Fresh assistant", true],
]);
} finally {
vi.useRealTimers();
}
@@ -995,6 +1072,103 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
);
});
it("emits one continuity reset across failed fresh reconnect attempts", async () => {
vi.useFakeTimers();
const provider = buildGoogleRealtimeVoiceProvider();
const onClose = vi.fn();
const onEvent = vi.fn();
const bridge = provider.createBridge({
providerConfig: { apiKey: "gemini-key", sessionResumption: false },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
onClose,
onEvent,
});
await bridge.connect();
const firstSession = lastConnectParams().callbacks;
firstSession.onmessage({ setupComplete: {} });
connectMock
.mockRejectedValueOnce(new Error("connect failed 1"))
.mockRejectedValueOnce(new Error("connect failed 2"))
.mockRejectedValueOnce(new Error("connect failed 3"));
firstSession.onclose({ code: 1011, reason: "temporary" });
await vi.advanceTimersByTimeAsync(1_750);
expect(connectMock).toHaveBeenCalledTimes(4);
expect(onEvent.mock.calls).toEqual([
[{ direction: "client", type: "session.continuity.reset" }],
]);
expect(onClose).toHaveBeenCalledWith("error");
});
it("rearms continuity reset after pre-return setup selects a fresh session", async () => {
vi.useFakeTimers();
const pendingSession = createDeferred<MockGoogleLiveSession>();
const freshSession = createMockGoogleLiveSession();
connectMock
.mockReturnValueOnce(Promise.resolve(session))
.mockReturnValueOnce(pendingSession.promise);
const provider = buildGoogleRealtimeVoiceProvider();
const onEvent = vi.fn();
const onReady = vi.fn();
const onTranscript = vi.fn();
const bridge = provider.createBridge({
providerConfig: { apiKey: "gemini-key", sessionResumption: false },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
onEvent,
onReady,
onTranscript,
});
await bridge.connect();
const firstCallbacks = lastConnectParams().callbacks;
firstCallbacks.onopen();
firstCallbacks.onmessage({ setupComplete: {} });
firstCallbacks.onclose({ code: 1011, reason: "temporary" });
await vi.advanceTimersByTimeAsync(250);
const freshCallbacks = lastConnectParams().callbacks;
expect(onEvent).toHaveBeenCalledTimes(1);
freshCallbacks.onopen();
freshCallbacks.onmessage({
setupComplete: {},
serverContent: { inputTranscription: { text: "Fresh partial " } },
});
expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([
"session.continuity.reset",
"session.created",
]);
expect(onReady).toHaveBeenCalledTimes(1);
pendingSession.resolve(freshSession);
await vi.waitFor(() => {
expect(onReady).toHaveBeenCalledTimes(2);
});
const sessionCreatedOrder = onEvent.mock.invocationCallOrder[1];
const freshReadyOrder = onReady.mock.invocationCallOrder[1];
if (sessionCreatedOrder === undefined || freshReadyOrder === undefined) {
throw new Error("expected fresh session creation before readiness");
}
expect(sessionCreatedOrder).toBeLessThan(freshReadyOrder);
freshCallbacks.onclose({ code: 1011, reason: "temporary again" });
await vi.advanceTimersByTimeAsync(250);
expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([
"session.continuity.reset",
"session.created",
"session.continuity.reset",
]);
lastConnectParams().callbacks.onmessage({
serverContent: { inputTranscription: { text: "Next", finished: true } },
});
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
["user", "Next", true],
]);
});
it("waits for the returned session after setup completion before activating", async () => {
const pendingSession = createDeferred<MockGoogleLiveSession>();
const connectedSession = createMockGoogleLiveSession();

View File

@@ -482,6 +482,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
private reconnectAttempts = 0;
private reconnectTimer: ReturnType<typeof setTimeout> | undefined;
private hasConnectedSession = false;
private continuityResetEmitted = false;
private terminalError: Error | undefined;
private closeNotified = false;
private connectionOwner: GoogleLiveConnectionAttempt | undefined;
@@ -530,13 +531,6 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
}
private async connectOwned(attempt: GoogleLiveConnectionAttempt): Promise<void> {
const canResumeSession =
this.config.sessionResumption !== false && Boolean(this.resumptionHandle);
if (this.hasConnectedSession && !canResumeSession) {
// An unfinished recognition hypothesis cannot cross into a fresh server session.
// Dropping it avoids treating a transport break as a completed user utterance.
this.resetPendingTranscripts();
}
this.intentionallyClosed = false;
this.closeNotified = false;
this.setupCompleteReceived = false;
@@ -852,7 +846,9 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
sessionResumptionUpdate?: { newHandle?: string; resumable?: boolean };
};
const update = raw.sessionResumptionUpdate;
if (update?.resumable && update.newHandle) {
if (update?.resumable === false) {
this.resumptionHandle = undefined;
} else if (update?.resumable && update.newHandle) {
this.resumptionHandle = update.newHandle;
}
if (raw.goAway?.timeLeft) {
@@ -861,6 +857,14 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
}
private handleSetupComplete(): void {
if (!this.setupCompleteReceived) {
// setupComplete proves Google selected a new server session. A later
// continuity loss therefore owns a new reset generation.
if (this.continuityResetEmitted) {
this.config.onEvent?.({ direction: "server", type: "session.created" });
}
this.continuityResetEmitted = false;
}
this.setupCompleteReceived = true;
this.maybeActivateSession();
}
@@ -1046,6 +1050,18 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
if (this.reconnectAttempts >= GOOGLE_REALTIME_RECONNECT_MAX_ATTEMPTS) {
return false;
}
const canResumeSession =
this.config.sessionResumption !== false && Boolean(this.resumptionHandle);
if (this.hasConnectedSession && !canResumeSession && !this.continuityResetEmitted) {
// A non-resumable close ends the provider generation immediately. Reset
// consumers before backoff so stale work cannot finish into the replacement.
this.continuityResetEmitted = true;
this.resetPendingTranscripts();
this.config.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
}
const attempt = ++this.reconnectAttempts;
const delayMs = Math.min(
GOOGLE_REALTIME_RECONNECT_MAX_DELAY_MS,

View File

@@ -254,6 +254,7 @@ async function withBargeInHarness(
createBridge: ReturnType<typeof vi.fn>;
handleBargeIn: ReturnType<typeof vi.fn>;
outboundMessages: Array<Record<string, unknown>>;
processEvent: ReturnType<typeof vi.fn>;
sendAudio: ReturnType<typeof vi.fn>;
ws: WebSocket;
}) => Promise<void>,
@@ -261,6 +262,7 @@ async function withBargeInHarness(
let callbacks: RealtimeBridgeRequest | undefined;
const sendAudio = vi.fn();
const handleBargeIn = vi.fn();
const processEvent = vi.fn();
const call = makeCallRecord(params.providerCallId);
const createBridge = vi.fn((request: RealtimeBridgeRequest) => {
callbacks = request;
@@ -278,6 +280,7 @@ async function withBargeInHarness(
const handler = makeHandler(undefined, {
manager: {
getCallByProviderCallId: vi.fn((): CallRecord => call),
processEvent,
},
providerConfig: {
apiKey: "test-key",
@@ -313,6 +316,7 @@ async function withBargeInHarness(
createBridge,
handleBargeIn,
outboundMessages,
processEvent,
sendAudio,
ws,
});
@@ -948,6 +952,63 @@ describe("RealtimeCallHandler path routing", () => {
);
});
it("starts fresh transcript and Talk state after provider continuity resets", async () => {
await withBargeInHarness(
{ providerCallId: "CA-continuity-reset" },
async ({ callbacks, call, outboundMessages, processEvent }) => {
callbacks.onTranscript?.("user", "Old caller ", false);
callbacks.onTranscript?.("assistant", "Old assistant ", false);
callbacks.onAudio?.(Buffer.alloc(320, 0xff));
const oldTurnId = recentTalkEvents(call).findLast(
(event) => event.type === "turn.started",
)?.turnId;
expect(oldTurnId).toBeTruthy();
callbacks.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
callbacks.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
await waitForRealtimeTest(() => {
expect(outboundMessages.some((message) => message.event === "clear")).toBe(true);
});
expect(requireCancelledTurn(call).turnId).toBe(oldTurnId);
const resetEvents = recentTalkEvents(call);
expect(resetEvents.filter((event) => event.type === "turn.cancelled")).toHaveLength(1);
expect(resetEvents.findIndex((event) => event.type === "output.audio.done")).toBeLessThan(
resetEvents.findIndex((event) => event.type === "turn.cancelled"),
);
callbacks.onTranscript?.("user", "Fresh caller", true);
callbacks.onTranscript?.("assistant", "Fresh assistant", true);
callbacks.onEvent?.({ direction: "server", type: "response.done" });
const processedEvents = processEvent.mock.calls.map(([event]) => event as NormalizedEvent);
expect(
processedEvents
.filter((event) => event.type === "call.speech")
.map((event) => (event.type === "call.speech" ? event.transcript : undefined)),
).toEqual(["Fresh caller"]);
expect(
processedEvents
.filter((event) => event.type === "call.assistant-speech")
.map((event) =>
event.type === "call.assistant-speech" ? event.transcript : undefined,
),
).toEqual(["Fresh assistant"]);
const startedTurns = recentTalkEvents(call).filter(
(event) => event.type === "turn.started",
);
expect(startedTurns).toHaveLength(2);
expect(startedTurns[1]?.turnId).not.toBe(oldTurnId);
},
);
});
it("passes the disabled input-interruption policy without cancelling speech-start", async () => {
await withBargeInHarness(
{
@@ -1492,6 +1553,87 @@ describe("RealtimeCallHandler path routing", () => {
}
});
it("clears cancelled consult dedupe for a fresh provider session", async () => {
let callbacks: RealtimeBridgeRequest | undefined;
let sessionHarness: RealtimeVoiceSessionHarness | undefined;
realtimeVoiceHarnessTestHooks.onCreate = (harness) => {
sessionHarness = harness;
};
const submitToolResult = vi.fn();
const createBridge = vi.fn((request: RealtimeBridgeRequest) => {
callbacks = request;
return makeBridge({ submitToolResult });
});
const handler = makeHandler(undefined, {
manager: {
getCallByProviderCallId: vi.fn((providerCallId: string) => makeCallRecord(providerCallId)),
},
realtimeProvider: makeRealtimeProvider(createBridge),
});
const consult = vi.fn(async () => ({ text: "fresh consult answer" }));
handler.registerToolHandler("openclaw_agent_consult", consult);
const server = await startRealtimeServer(handler);
try {
const ws = await connectWs(server.url);
try {
ws.send(
JSON.stringify({
event: "start",
start: { streamSid: "MZ-continuity-consult", callSid: "CA-continuity-consult" },
}),
);
await waitForRealtimeTest(() => {
expect(callbacks).toBeDefined();
expect(sessionHarness).toBeDefined();
});
const coordinator = expectDefined(
sessionHarness,
"voice-call realtime session harness",
).forcedConsults;
const cancelled = expectDefined(
coordinator.prepare("same question"),
"cancelled forced consult",
);
coordinator.markStarted(cancelled);
coordinator.markCancelled(cancelled);
callbacks?.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
callbacks?.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
expect(coordinator.handles()).toEqual([]);
callbacks?.onToolCall?.({
itemId: "item-fresh",
callId: "native-fresh",
name: "openclaw_agent_consult",
args: { question: "same question" },
});
await waitForRealtimeTest(() => {
expect(consult).toHaveBeenCalledTimes(1);
expect(submitToolResult).toHaveBeenCalledWith(
"native-fresh",
{ text: "fresh consult answer" },
undefined,
);
});
} finally {
if (ws.readyState !== WebSocket.CLOSED && ws.readyState !== WebSocket.CLOSING) {
ws.close();
}
}
} finally {
await server.close();
}
});
it("does not deliver a forced consult after its realtime session closes", async () => {
let callbacks:
| {
@@ -1727,6 +1869,125 @@ describe("RealtimeCallHandler path routing", () => {
}
});
it("ignores late continuity reset and close from a replaced bridge", async () => {
const callbacks: RealtimeBridgeRequest[] = [];
const oldCloseBridge = vi.fn();
const replacementCloseBridge = vi.fn();
const bridges = [
makeBridge({ close: oldCloseBridge }),
makeBridge({ close: replacementCloseBridge }),
];
const createBridge = vi.fn((request: RealtimeBridgeRequest) => {
callbacks.push(request);
const bridge = bridges[callbacks.length - 1];
if (!bridge) {
throw new Error("unexpected replacement bridge");
}
return bridge;
});
const processEvent = vi.fn();
const hangupCall = vi.fn(async () => {});
const sharedCallSid = "CA-continuity-shared";
const call = makeCallRecord(sharedCallSid);
const handler = makeHandler(undefined, {
manager: {
getCallByProviderCallId: vi.fn(() => call),
processEvent,
},
provider: { hangupCall },
realtimeProvider: makeRealtimeProvider(createBridge),
});
const oldServer = await startRealtimeServer(handler);
let replacementServer: Awaited<ReturnType<typeof startRealtimeServer>> | undefined;
let oldWs: WebSocket | undefined;
try {
oldWs = await connectWs(oldServer.url);
oldWs.send(
JSON.stringify({
event: "start",
start: { streamSid: "MZ-continuity-old", callSid: sharedCallSid },
}),
);
await waitForRealtimeTest(() => {
expect(callbacks).toHaveLength(1);
});
replacementServer = await startRealtimeServer(handler);
const replacementWs = await connectWs(replacementServer.url);
try {
replacementWs.send(
JSON.stringify({
event: "start",
start: {
streamSid: "MZ-continuity-replacement",
callSid: sharedCallSid,
},
}),
);
await waitForRealtimeTest(() => {
expect(callbacks).toHaveLength(2);
});
callbacks[1]?.onTranscript?.("user", "Fresh ", false);
callbacks[0]?.onTranscript?.("user", "stale partial", false);
callbacks[0]?.onTranscript?.("user", "stale final", true);
callbacks[0]?.onTranscript?.("assistant", "stale assistant", true);
callbacks[0]?.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
callbacks[0]?.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
const oldClosed = waitForClose(oldWs);
callbacks[0]?.onClose?.("error");
await oldClosed;
expect(oldCloseBridge).toHaveBeenCalledOnce();
expect(replacementCloseBridge).not.toHaveBeenCalled();
expect(hangupCall).not.toHaveBeenCalled();
expect(
processEvent.mock.calls
.map(([event]) => event as NormalizedEvent)
.filter((event) => event.type === "call.ended"),
).toHaveLength(0);
callbacks[1]?.onTranscript?.("user", "caller", true);
await waitForRealtimeTest(() => {
expect(
processEvent.mock.calls
.map(([event]) => event as NormalizedEvent)
.filter((event) => event.type === "call.speech")
.map((event) => (event.type === "call.speech" ? event.transcript : undefined)),
).toEqual(["Fresh caller"]);
});
expect(
processEvent.mock.calls
.map(([event]) => event as NormalizedEvent)
.filter((event) => event.type === "call.assistant-speech"),
).toHaveLength(0);
} finally {
if (
replacementWs.readyState !== WebSocket.CLOSED &&
replacementWs.readyState !== WebSocket.CLOSING
) {
replacementWs.close();
}
}
} finally {
if (
oldWs &&
oldWs.readyState !== WebSocket.CLOSED &&
oldWs.readyState !== WebSocket.CLOSING
) {
oldWs.close();
}
await replacementServer?.close();
await oldServer.close();
}
});
it("does not share a native consult with a replacement realtime session", async () => {
const callbacks: RealtimeBridgeRequest[] = [];
const oldSubmitToolResult = vi.fn();

View File

@@ -339,7 +339,10 @@ export class RealtimeCallHandler {
private readonly activeBridgesByCallId = new Map<string, ActiveRealtimeVoiceBridge>();
private readonly activeTelephonyClosersByCallId = new Map<
string,
(reason: TelephonyCloseReason) => void
{
owner: ActiveRealtimeVoiceBridge;
close: (reason: TelephonyCloseReason) => void;
}
>();
private readonly partialUserTranscriptsByCallId = new Map<string, string>();
private readonly rawPartialUserTranscriptsByCallId = new Map<string, string>();
@@ -797,6 +800,10 @@ export class RealtimeCallHandler {
},
},
onTranscript: (role, text, isFinal) => {
const owner = nativeConsultOwner.current;
if (owner && !this.isActiveBridgeOwner(callId, owner)) {
return;
}
const turnId = harness.ensureTurn();
const eventType =
role === "assistant"
@@ -898,6 +905,27 @@ export class RealtimeCallHandler {
);
},
onEvent: (event) => {
if (event.direction === "client" && event.type === "session.continuity.reset") {
// A fresh provider session cannot complete the prior session's text,
// audio, tool work, or Talk turn.
const turnId = harness.talk.activeTurnId;
const owner = nativeConsultOwner.current;
if (owner && this.isActiveBridgeOwner(callId, owner)) {
this.clearUserTranscriptState(callId);
this.resetConsultSessionForContinuity(callId, owner);
}
harness.flushOutput(() => {
audioPacer.clearAudio();
harness.finishOutputAudio(event.type);
});
if (turnId) {
harness.talk.cancelTurn({
turnId,
payload: { callId, providerCallId: callSid, reason: event.type },
});
}
return;
}
if (event.type === "input_audio_buffer.speech_started") {
harness.ensureTurn();
return;
@@ -943,11 +971,15 @@ export class RealtimeCallHandler {
});
},
onClose: (reason) => {
if (nativeConsultOwner.current) {
this.clearActiveBridgeMappings(callId, callSid, nativeConsultOwner.current);
this.cancelConsultSession(callId, nativeConsultOwner.current);
const owner = nativeConsultOwner.current;
const ownsCallState = owner ? this.isActiveBridgeOwner(callId, owner) : false;
if (owner) {
this.clearActiveBridgeMappings(callId, callSid, owner);
this.cancelConsultSession(callId, owner);
}
if (ownsCallState) {
this.clearUserTranscriptState(callId);
}
this.clearUserTranscriptState(callId);
harness.finishOutputAudio(reason);
harness.emit({
type: "session.closed",
@@ -957,10 +989,13 @@ export class RealtimeCallHandler {
if (reason !== "error") {
return;
}
emitCallEnd("error");
if (ws.readyState === WebSocket.OPEN) {
ws.close(1011, "Bridge disconnected");
}
if (owner && !ownsCallState) {
return;
}
emitCallEnd("error");
void this.provider
.hangupCall({ callId, providerCallId: callSid, reason: "error" })
.catch((error: unknown) => {
@@ -992,8 +1027,9 @@ export class RealtimeCallHandler {
});
this.activeBridgesByCallId.set(callId, session);
this.activeBridgesByCallId.set(callSid, session);
this.activeTelephonyClosersByCallId.set(callId, closeTelephony);
this.activeTelephonyClosersByCallId.set(callSid, closeTelephony);
const telephonyCloser = { owner: session, close: closeTelephony };
this.activeTelephonyClosersByCallId.set(callId, telephonyCloser);
this.activeTelephonyClosersByCallId.set(callSid, telephonyCloser);
const sendAudioToSession = session.sendAudio.bind(session);
session.sendAudio = (audio) => {
if (speechDetector.accept({ rms: calculateMulawRms(audio), peak: 0 })) {
@@ -1021,9 +1057,12 @@ export class RealtimeCallHandler {
try {
closeSession();
} finally {
const ownsCallState = this.isActiveBridgeOwner(callId, session);
this.clearActiveBridgeMappings(callId, callSid, session);
this.cancelConsultSession(callId, session);
this.clearUserTranscriptState(callId);
if (ownsCallState) {
this.clearUserTranscriptState(callId);
}
harness.close();
audioPacer.close();
}
@@ -1031,8 +1070,11 @@ export class RealtimeCallHandler {
session.connect().catch((error: unknown) => {
console.error("[voice-call] Failed to connect realtime bridge:", error);
const ownsCallState = this.isActiveBridgeOwner(callId, session);
session.close();
emitCallEnd("error");
if (ownsCallState) {
emitCallEnd("error");
}
ws.close(1011, "Failed to connect");
});
@@ -1105,22 +1147,43 @@ export class RealtimeCallHandler {
this.forcedConsultsByCallId.delete(callId);
}
private cancelConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge | undefined): void {
if (!owner) {
return;
}
private resetConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge): boolean {
const session = this.consultSessionsByCallId.get(callId);
if (!session || session.owner !== owner) {
return;
return false;
}
// Forced and native consults share bridge ownership. Replacement or close
// must invalidate both before a newer bridge can observe call-scoped state.
session.coordinator.clearPending();
this.cancelForcedConsult(callId, owner);
this.cancelNativeConsult(callId, owner);
return true;
}
private resetConsultSessionForContinuity(
callId: string,
owner: ActiveRealtimeVoiceBridge,
): boolean {
const session = this.consultSessionsByCallId.get(callId);
if (!session || session.owner !== owner) {
return false;
}
this.cancelForcedConsult(callId, owner);
this.cancelNativeConsult(callId, owner);
// A fresh provider session must not inherit cancelled/recent consult dedupe.
session.coordinator.clear();
return true;
}
private cancelConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge | undefined): void {
if (!owner || !this.resetConsultSession(callId, owner)) {
return;
}
this.consultSessionsByCallId.delete(callId);
}
private isActiveBridgeOwner(callId: string, owner: ActiveRealtimeVoiceBridge): boolean {
return this.activeBridgesByCallId.get(callId) === owner;
}
private clearActiveBridgeMappings(
callId: string,
callSid: string,
@@ -1197,8 +1260,8 @@ export class RealtimeCallHandler {
reason: TelephonyCloseReason,
): void {
const closer = this.activeTelephonyClosersByCallId.get(callIdOrSid);
if (closer) {
closer(reason);
if (closer && closer.owner === bridge) {
closer.close(reason);
return;
}
bridge?.close();

View File

@@ -22,6 +22,7 @@ import {
ensureRelayTurn,
noFallbackRelayOutputFlush,
relaySessions,
resolveRelayProviderToolCallId,
type ForcedTerminalProviderResult,
type RelayAgentControlProviderSubmission,
type RelaySession,
@@ -275,7 +276,7 @@ export function submitRealtimeAgentConsultWorkingResponse(
}
const epoch = session.toolResultEpoch;
const submission = session.bridge.submitToolResult(
callId,
resolveRelayProviderToolCallId(session, callId),
buildRealtimeVoiceAgentConsultWorkingResponse("person"),
{ willContinue: true },
);

View File

@@ -5,6 +5,7 @@ import {
} from "../talk/agent-run-control.js";
import { registerClientVoiceConsultRun } from "../talk/client-voice-session.js";
import type { RealtimeVoiceToolResultOptions } from "../talk/provider-types.js";
import type { TalkEvent } from "../talk/talk-session-controller.js";
import { abortChatRunById } from "./chat-abort.js";
import { formatError } from "./server-utils.js";
import {
@@ -30,6 +31,7 @@ import {
ensureRelayTurn,
noFallbackRelayOutputFlush,
relaySessions,
resolveRelayProviderToolCallId,
type RelaySession,
} from "./talk-realtime-relay-state.js";
import {
@@ -298,7 +300,11 @@ export function submitTalkRealtimeRelayToolResult(params: {
return trackAgentFinalToolResult(session, params.callId, completion);
}
const submit = () =>
session.bridge.submitToolResult(params.callId, params.result, params.options);
session.bridge.submitToolResult(
resolveRelayProviderToolCallId(session, params.callId),
params.result,
params.options,
);
const pendingWorking = session.pendingWorkingToolResults.get(params.callId);
if (pendingWorking) {
const submission = pendingWorking.then(async () => {
@@ -451,6 +457,54 @@ export function cancelTalkRealtimeRelayTurn(params: {
});
}
/** Drops one provider generation without sending cancellation into its replacement. */
export function resetTalkRealtimeRelayContinuity(
session: RelaySession,
reason = "session.continuity.reset",
): TalkEvent | undefined {
session.toolResultEpoch += 1;
const retiredCallIds = new Set<string>([
...session.activeAgentToolCalls.keys(),
...session.cancelledAgentToolCalls.keys(),
...session.providerToolCallIds.keys(),
...session.providerToolCallIds.values(),
...session.pendingFinalToolResults.keys(),
...session.pendingProviderToolResults.keys(),
...session.pendingWorkingToolResults.keys(),
...session.forcedTerminalProviderResults.keys(),
]);
for (const handle of session.harness.forcedConsults.handles()) {
retiredCallIds.add(handle.id);
for (const nativeCallId of session.harness.forcedConsults.nativeCallIds(handle)) {
retiredCallIds.add(nativeCallId);
}
}
for (const callId of retiredCallIds) {
session.completedAgentToolCalls.add(callId);
}
session.cancelledAgentToolCalls.clear();
session.providerToolCallIds.clear();
session.relayToolCallIdsByProviderId.clear();
session.pendingFinalToolResults.clear();
session.completedProviderToolResults.clear();
session.pendingProviderToolResults.clear();
session.pendingWorkingToolResults.clear();
session.forcedTerminalProviderResults.clear();
session.harness.forcedConsults.clear();
abortRelayAgentRuns(session, reason);
const turnId = session.harness.talk.activeTurnId;
session.harness.flushOutput(noFallbackRelayOutputFlush);
session.harness.finishOutputAudio(reason);
if (!turnId) {
return undefined;
}
const cancelled = session.harness.talk.cancelTurn({
turnId,
payload: { reason },
});
return cancelled.ok ? cancelled.event : undefined;
}
/** Closes a realtime relay session owned by the current connection. */
export function stopTalkRealtimeRelaySession(params: {
relaySessionId: string;

View File

@@ -1,6 +1,11 @@
import { buildRealtimeVoiceAgentCancelProviderResult } from "../talk/agent-run-control-shared.js";
import type { RealtimeVoiceToolResultOptions } from "../talk/provider-types.js";
import { broadcastToOwner, relaySessions, type RelaySession } from "./talk-realtime-relay-state.js";
import {
broadcastToOwner,
relaySessions,
resolveRelayProviderToolCallId,
type RelaySession,
} from "./talk-realtime-relay-state.js";
export function suppressedToolResultOptions(
session: RelaySession,
@@ -63,8 +68,13 @@ export function submitFinalProviderToolResult(params: {
options?: RealtimeVoiceToolResultOptions;
onAccepted?: () => void;
}): void | Promise<void> {
if (params.session.completedProviderToolResults.has(params.callId)) {
if (relaySessions.get(params.session.id) === params.session) {
const epoch = params.session.toolResultEpoch;
const providerCallId = resolveRelayProviderToolCallId(params.session, params.callId);
if (params.session.completedProviderToolResults.has(providerCallId)) {
if (
relaySessions.get(params.session.id) === params.session &&
params.session.toolResultEpoch === epoch
) {
params.onAccepted?.();
}
return;
@@ -74,9 +84,8 @@ export function submitFinalProviderToolResult(params: {
return pending;
}
const submit = () =>
params.session.bridge.submitToolResult(params.callId, params.result, params.options);
params.session.bridge.submitToolResult(providerCallId, params.result, params.options);
const working = params.session.pendingWorkingToolResults.get(params.callId);
const epoch = params.session.toolResultEpoch;
const submitAfterWorking = async () => {
if (relaySessions.get(params.session.id) !== params.session) {
return false;
@@ -89,13 +98,13 @@ export function submitFinalProviderToolResult(params: {
// the provider's working-result acknowledgement. Finish the cancelled call
// here so the provider is not left waiting for a terminal result.
await params.session.bridge.submitToolResult(
params.callId,
providerCallId,
buildRealtimeVoiceAgentCancelProviderResult(
"OpenClaw cancelled this consult before completion. Do not restart it.",
),
suppressedToolResultOptions(params.session),
);
params.session.completedProviderToolResults.add(params.callId);
params.session.completedProviderToolResults.add(providerCallId);
params.session.cancelledAgentToolCalls.delete(params.callId);
params.session.completedAgentToolCalls.add(params.callId);
return false;
@@ -105,7 +114,10 @@ export function submitFinalProviderToolResult(params: {
};
const submission = working ? working.then(submitAfterWorking, submitAfterWorking) : submit();
const accept = () => {
params.session.completedProviderToolResults.add(params.callId);
if (params.session.toolResultEpoch !== epoch) {
return;
}
params.session.completedProviderToolResults.add(providerCallId);
if (relaySessions.get(params.session.id) === params.session) {
params.onAccepted?.();
}

View File

@@ -35,12 +35,14 @@ import {
enforceRelaySessionLimits,
pruneInactiveRelayAgentRuns,
registerTalkRealtimeRelayAgentRun,
resetTalkRealtimeRelayContinuity,
steerTalkRealtimeRelayAgentRun,
} from "./talk-realtime-relay-operations.js";
import { suppressedToolResultOptions } from "./talk-realtime-relay-provider-results.js";
import {
RELAY_SESSION_TTL_MS,
RELAY_TRANSCRIPT_ECHO_LOOKBACK_MS,
adoptRelayProviderToolCallId,
broadcastToOwner,
ensureRelayTurn,
relayEventDeliveryOptions,
@@ -105,6 +107,7 @@ export function createTalkRealtimeRelaySession(
let currentOutputItemId: string | undefined;
let currentOutputResponseId: string | undefined;
let ready = false;
let continuityResetActive = false;
let failureEmitted = false;
let transcriptPersistenceFailed = false;
const constructionTerminal: {
@@ -253,12 +256,34 @@ export function createTalkRealtimeRelaySession(
},
},
onEvent: (event) => {
if (!getActiveRelay()) {
const relay = getActiveRelay();
if (!relay) {
return;
}
if (event.direction === "client" && event.type === "session.continuity.reset") {
if (continuityResetActive) {
return;
}
continuityResetActive = true;
ready = false;
currentOutputItemId = undefined;
currentOutputResponseId = undefined;
const talkEvent = resetTalkRealtimeRelayContinuity(relay, event.type);
const clearEvent = { relaySessionId, type: "clear" as const };
broadcastToOwner(
params.context,
params.connId,
{ ...clearEvent, ...(talkEvent ? { talkEvent } : {}) },
relayEventDeliveryOptions(clearEvent),
);
return;
}
if (event.direction !== "server") {
return;
}
if (event.type === "session.created") {
continuityResetActive = false;
}
if (
event.type === "conversation.output_audio.delta" ||
event.type === "response.audio.delta" ||
@@ -365,11 +390,13 @@ export function createTalkRealtimeRelaySession(
if (!relay) {
return;
}
const providerCallId = toolCall.callId;
const relayCallId = adoptRelayProviderToolCallId(relay, providerCallId);
let shouldSubmitWorkingResult = false;
if (toolCall.name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
const forcedConsult = relay.harness.forcedConsults.recordNativeConsult(
toolCall.args,
toolCall.callId,
providerCallId,
);
if (forcedConsult.kind === "in_flight" || forcedConsult.kind === "already_delivered") {
if (forcedConsult.kind === "already_delivered") {
@@ -380,7 +407,7 @@ export function createTalkRealtimeRelaySession(
: buildAlreadyDeliveredToolResult();
return submitForcedConsultProviderResult(
relay,
toolCall.callId,
providerCallId,
result,
suppressedToolResultOptions(relay),
);
@@ -388,7 +415,7 @@ export function createTalkRealtimeRelaySession(
if (relay.forcedTerminalProviderResults.has(forcedConsult.handle.id)) {
return relay.pendingFinalToolResults.get(forcedConsult.handle.id);
}
return submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId);
return submitRealtimeAgentConsultWorkingResponse(relay, relayCallId);
}
shouldSubmitWorkingResult = true;
}
@@ -398,20 +425,20 @@ export function createTalkRealtimeRelaySession(
relaySessionId,
type: "toolCall",
itemId: toolCall.itemId,
callId: toolCall.callId,
callId: relayCallId,
name: toolCall.name,
args: toolCall.args,
},
{
type: "tool.call",
itemId: toolCall.itemId,
callId: toolCall.callId,
callId: relayCallId,
turnId,
payload: { name: toolCall.name, args: toolCall.args },
},
);
if (shouldSubmitWorkingResult) {
return submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId, turnId);
return submitRealtimeAgentConsultWorkingResponse(relay, relayCallId, turnId);
}
},
onReady: () => {
@@ -419,6 +446,7 @@ export function createTalkRealtimeRelaySession(
return;
}
ready = true;
continuityResetActive = false;
emit({ relaySessionId, type: "ready" }, { type: "session.ready", payload: null });
},
onError: (error) => {
@@ -536,6 +564,8 @@ export function createTalkRealtimeRelaySession(
provider: params.provider.id,
activeAgentToolCalls: new Map(),
completedAgentToolCalls: new Set(),
providerToolCallIds: new Map(),
relayToolCallIdsByProviderId: new Map(),
cancelledAgentToolCalls: new Map(),
pendingFinalToolResults: new Map(),
completedProviderToolResults: new Set(),

View File

@@ -1,3 +1,4 @@
import { randomUUID } from "node:crypto";
import type { OpenClawConfig } from "../config/types.js";
import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js";
import type { BoundedSerialQueue } from "../shared/bounded-serial-queue.js";
@@ -94,6 +95,8 @@ export type RelaySession = {
provider: string;
activeAgentToolCalls: Map<string, string>;
completedAgentToolCalls: Set<string>;
providerToolCallIds: Map<string, string>;
relayToolCallIdsByProviderId: Map<string, string>;
// Cancelled calls retain their original turn long enough to terminally satisfy
// late browser results without creating a replacement turn or owner success event.
cancelledAgentToolCalls: Map<string, string>;
@@ -150,6 +153,28 @@ export const relaySessions = new Map<string, RelaySession>();
// until durable close settles. Session limits count both maps.
export const drainingRelaySessions = new Set<RelaySession>();
export function adoptRelayProviderToolCallId(
session: RelaySession,
providerCallId: string,
): string {
const current = session.relayToolCallIdsByProviderId.get(providerCallId);
if (current) {
return current;
}
const relayCallId = session.completedAgentToolCalls.has(providerCallId)
? `relay-${randomUUID()}`
: providerCallId;
session.completedProviderToolResults.delete(providerCallId);
session.completedAgentToolCalls.delete(relayCallId);
session.providerToolCallIds.set(relayCallId, providerCallId);
session.relayToolCallIdsByProviderId.set(providerCallId, relayCallId);
return relayCallId;
}
export function resolveRelayProviderToolCallId(session: RelaySession, relayCallId: string): string {
return session.providerToolCallIds.get(relayCallId) ?? relayCallId;
}
export function broadcastToOwner(
context: GatewayRequestContext,
connId: string,

View File

@@ -989,6 +989,165 @@ describe("talk realtime gateway relay", () => {
expectRecordFields(mockCallArg(mock, 0, 2), { runId: "run-1", state: "aborted" });
}
it("resets provider continuity without cancelling the replacement provider", async () => {
let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined;
const pendingWorking = createDeferredVoid();
const handleBargeIn = vi.fn();
const submitToolResult = vi
.fn<RealtimeVoiceBridge["submitToolResult"]>()
.mockReturnValueOnce(pendingWorking.promise);
const bridge = {
supportsToolResultContinuation: true,
connect: vi.fn(async () => undefined),
sendAudio: vi.fn(),
setMediaTimestamp: vi.fn(),
sendUserMessage: vi.fn(),
handleBargeIn,
submitToolResult,
acknowledgeMark: vi.fn(),
close: vi.fn(),
isConnected: vi.fn(() => true),
};
const provider: RealtimeVoiceProviderPlugin = {
id: "relay-test",
label: "Relay Test",
isConfigured: () => true,
createBridge: (request) => {
bridgeRequest = request;
return bridge;
},
};
const fixture = createAbortableRelayRunFixture(provider);
const relay = relaySessions.get(fixture.session.relaySessionId);
if (!relay || !bridgeRequest) {
throw new Error("expected active relay continuity fixture");
}
const request = bridgeRequest;
request.onReady?.();
request.onEvent?.({
direction: "server",
type: "response.audio.delta",
itemId: "old-item",
responseId: "old-response",
});
request.onAudio(Buffer.from("old audio"));
request.onToolCall?.({
itemId: "old-item",
callId: "call-1",
name: "custom_tool",
args: {},
});
const working = submitTalkRealtimeRelayToolResult({
relaySessionId: fixture.session.relaySessionId,
connId: "conn-1",
callId: "call-1",
result: { status: "working" },
options: { willContinue: true },
});
request.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
request.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
expect(fixture.abortController.signal.aborted).toBe(true);
expect(handleBargeIn).not.toHaveBeenCalled();
expect(bridge.close).not.toHaveBeenCalled();
expect(submitToolResult).toHaveBeenCalledTimes(1);
expect(relay.pendingWorkingToolResults.size).toBe(0);
expect(relay.activeAgentRuns.size).toBe(0);
expect(relay.activeAgentToolCalls.size).toBe(0);
const payloads = fixture.broadcastToConnIds.mock.calls.map(([, payload]) => payload);
const clears = payloads.filter(
(payload): payload is Record<string, unknown> =>
typeof payload === "object" &&
payload !== null &&
(payload as Record<string, unknown>).type === "clear",
);
expect(clears).toHaveLength(1);
expectRecordFields(clears[0]?.talkEvent, {
type: "turn.cancelled",
payload: { reason: "session.continuity.reset" },
final: true,
});
pendingWorking.resolve();
await working;
expect(submitToolResult).toHaveBeenCalledTimes(1);
request.onEvent?.({ direction: "server", type: "session.created" });
request.onAudio(Buffer.from("fresh audio"));
const freshAudio = fixture.broadcastToConnIds.mock.calls
.map(([, payload]) => payload)
.findLast(
(payload): payload is Record<string, unknown> =>
typeof payload === "object" &&
payload !== null &&
(payload as Record<string, unknown>).type === "audio",
);
expectRecordFields(freshAudio, {
type: "audio",
audioBase64: Buffer.from("fresh audio").toString("base64"),
});
expect(freshAudio).not.toHaveProperty("itemId");
expect(freshAudio).not.toHaveProperty("responseId");
request.onToolCall?.({
itemId: "fresh-item",
callId: "call-1",
name: "custom_tool",
args: {},
});
const freshToolCall = fixture.broadcastToConnIds.mock.calls
.map(([, payload]) => payload)
.findLast(
(payload): payload is Record<string, unknown> =>
typeof payload === "object" &&
payload !== null &&
(payload as Record<string, unknown>).type === "toolCall",
);
const freshCallId = freshToolCall?.callId;
expect(typeof freshCallId).toBe("string");
expect(freshCallId).not.toBe("call-1");
await submitTalkRealtimeRelayToolResult({
relaySessionId: fixture.session.relaySessionId,
connId: "conn-1",
callId: "call-1",
result: { stale: true },
});
expect(submitToolResult).toHaveBeenCalledTimes(1);
await submitTalkRealtimeRelayToolResult({
relaySessionId: fixture.session.relaySessionId,
connId: "conn-1",
callId: String(freshCallId),
result: { ok: true },
});
expect(submitToolResult).toHaveBeenLastCalledWith("call-1", { ok: true }, undefined);
request.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
request.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
expect(
fixture.broadcastToConnIds.mock.calls
.map(([, payload]) => payload)
.filter(
(payload) =>
typeof payload === "object" &&
payload !== null &&
(payload as Record<string, unknown>).type === "clear",
),
).toHaveLength(2);
});
it("bridges browser audio, transcripts, and tool results through a backend provider", async () => {
let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined;
const bridge = {

View File

@@ -24,6 +24,7 @@ import type {
MeetingRealtimeToolCallParams,
MeetingRuntimePlatform,
} from "./realtime-engine.js";
import { readMeetingRealtimeToolAbortSignal } from "./realtime-tool-continuity.js";
function resolveMeetingRealtimeTools(
policy: RealtimeVoiceAgentConsultToolPolicy,
@@ -77,7 +78,8 @@ export function createMeetingRealtimeEngineBindings(params: {
...consult,
}),
tools: resolveMeetingRealtimeTools(params.config.realtime.toolPolicy),
handleToolCall: async (call) =>
handleToolCall: async (call) => {
const abortSignal = readMeetingRealtimeToolAbortSignal(call.session);
await handleMeetingRealtimeConsultToolCall({
surface,
config: params.fullConfig,
@@ -85,17 +87,20 @@ export function createMeetingRealtimeEngineBindings(params: {
logger: params.logger,
agentId: params.config.realtime.agentId,
toolPolicy: params.config.realtime.toolPolicy,
abortSignal,
...call,
}),
});
},
};
}
async function submitMeetingConsultWorkingResponse(params: {
session: RealtimeVoiceBridgeSession;
abortSignal?: AbortSignal;
callId: string;
label: string;
}): Promise<void> {
if (!params.session.bridge.supportsToolResultContinuation) {
if (params.abortSignal?.aborted || !params.session.bridge.supportsToolResultContinuation) {
return;
}
await params.session.submitToolResult(
@@ -116,6 +121,7 @@ async function consultMeetingAgent(params: {
requesterSessionKey?: string;
args: unknown;
transcript: Array<{ role: "user" | "assistant"; text: string }>;
abortSignal?: AbortSignal;
}): Promise<{ text: string }> {
const agentId = params.agentId
? normalizeAgentId(params.agentId)
@@ -142,6 +148,7 @@ async function consultMeetingAgent(params: {
questionSourceLabel: params.surface.questionSourceLabel,
toolsAllow: resolveRealtimeVoiceAgentConsultToolsAllow(params.toolPolicy),
extraSystemPrompt: params.surface.extraSystemPrompt,
abortSignal: params.abortSignal,
});
}
@@ -158,12 +165,19 @@ async function handleMeetingRealtimeConsultToolCall(params: {
meetingSessionId: string;
requesterSessionKey?: string;
transcript: Array<{ role: "user" | "assistant"; text: string }>;
abortSignal?: AbortSignal;
onTalkEvent?: (event: TalkEventInput) => void;
}): Promise<void> {
const callId = params.event.callId || params.event.itemId;
if (params.abortSignal?.aborted) {
return;
}
if (params.strategy !== "bidi") {
const error = `Tool "${params.event.name}" is only available in bidi realtime strategy`;
await params.session.submitToolResult(callId, { error });
if (params.abortSignal?.aborted) {
return;
}
params.onTalkEvent?.({
type: "tool.error",
callId,
@@ -175,6 +189,9 @@ async function handleMeetingRealtimeConsultToolCall(params: {
if (params.event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
const error = `Tool "${params.event.name}" not available`;
await params.session.submitToolResult(callId, { error });
if (params.abortSignal?.aborted) {
return;
}
params.onTalkEvent?.({
type: "tool.error",
callId,
@@ -185,9 +202,13 @@ async function handleMeetingRealtimeConsultToolCall(params: {
}
await submitMeetingConsultWorkingResponse({
session: params.session,
abortSignal: params.abortSignal,
callId,
label: params.surface.workingResponseLabel,
});
if (params.abortSignal?.aborted) {
return;
}
params.onTalkEvent?.({
type: "tool.progress",
callId,
@@ -206,10 +227,17 @@ async function handleMeetingRealtimeConsultToolCall(params: {
requesterSessionKey: params.requesterSessionKey,
args: params.event.args,
transcript: params.transcript,
abortSignal: params.abortSignal,
});
} catch (error) {
if (params.abortSignal?.aborted) {
return;
}
const message = formatErrorMessage(error);
await params.session.submitToolResult(callId, { error: message });
if (params.abortSignal?.aborted) {
return;
}
params.onTalkEvent?.({
type: "tool.error",
callId,
@@ -218,7 +246,13 @@ async function handleMeetingRealtimeConsultToolCall(params: {
});
return;
}
if (params.abortSignal?.aborted) {
return;
}
await params.session.submitToolResult(callId, result);
if (params.abortSignal?.aborted) {
return;
}
params.onTalkEvent?.({
type: "tool.result",
callId,

View File

@@ -5,16 +5,22 @@ import type {
RealtimeVoiceBridgeCreateRequest,
} from "../talk/provider-types.js";
import type { MeetingRealtimeAudioTransport } from "./realtime-audio-transport.js";
import { startMeetingRealtimeEngine } from "./realtime-engine.js";
import {
startMeetingRealtimeEngine,
type MeetingRealtimeToolCallParams,
} from "./realtime-engine.js";
type PendingWrite = {
resolve: () => void;
};
async function createEngineFixture() {
async function createEngineFixture(options?: {
handleToolCall?: (params: MeetingRealtimeToolCallParams) => Promise<void>;
}) {
let callbacks: RealtimeVoiceBridgeCreateRequest | undefined;
let onHumanBargeIn: ((audio: Buffer) => boolean) | undefined;
const handleBargeIn = vi.fn();
const submitToolResult = vi.fn();
const bridge: RealtimeVoiceBridge = {
acknowledgeMark: vi.fn(),
close: vi.fn(),
@@ -23,7 +29,7 @@ async function createEngineFixture() {
isConnected: vi.fn(() => true),
sendAudio: vi.fn(),
setMediaTimestamp: vi.fn(),
submitToolResult: vi.fn(),
submitToolResult,
};
const provider: RealtimeVoiceProviderPlugin = {
id: "test",
@@ -66,7 +72,7 @@ async function createEngineFixture() {
},
consultAgent: vi.fn(async () => ({ text: "unused" })),
fullConfig: {} as never,
handleToolCall: vi.fn(async () => {}),
handleToolCall: options?.handleToolCall ?? vi.fn(async () => {}),
logger: {
debug: vi.fn(),
error: vi.fn(),
@@ -94,6 +100,7 @@ async function createEngineFixture() {
clearOutput,
handle,
handleBargeIn,
submitToolResult,
releaseWrite(index: number) {
const pending = pendingWrites[index];
if (!pending) {
@@ -127,6 +134,117 @@ async function createEngineFixture() {
}
describe("meeting realtime engine output ownership", () => {
it("rearms continuity reset when the provider creates a fresh session before ready", async () => {
const fixture = await createEngineFixture();
try {
fixture.callbacks.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
fixture.callbacks.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
await vi.waitFor(() => {
expect(fixture.clearOutput).toHaveBeenCalledOnce();
});
fixture.callbacks.onEvent?.({
direction: "server",
type: "session.created",
});
fixture.callbacks.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
await vi.waitFor(() => {
expect(fixture.clearOutput).toHaveBeenCalledTimes(2);
});
} finally {
await fixture.handle.stop();
}
});
it("resets provider continuity without replaying old output or tool work", async () => {
let releaseTool: (() => void) | undefined;
const toolGate = new Promise<void>((resolve) => {
releaseTool = resolve;
});
const fixture = await createEngineFixture({
handleToolCall: async ({ session, event, onTalkEvent }) => {
await toolGate;
await session.submitToolResult(event.callId, { text: "stale result" });
onTalkEvent({
type: "tool.result",
callId: event.callId,
payload: { name: event.name },
final: true,
});
},
});
try {
const active = Buffer.from([1]);
const stale = Buffer.from([2]);
const fresh = Buffer.from([3]);
fixture.callbacks.onReady?.();
fixture.callbacks.onTranscript?.("user", "old turn", true);
fixture.sendOutputAudio(active, "response-1");
await vi.waitFor(() => {
expect(fixture.writeOutput).toHaveBeenCalledOnce();
});
fixture.sendOutputAudio(stale, "response-1");
fixture.callbacks.onToolCall?.({
itemId: "item-old",
callId: "call-old",
name: "openclaw_agent_consult",
args: { question: "old work" },
});
fixture.callbacks.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
fixture.callbacks.onEvent?.({
direction: "client",
type: "session.continuity.reset",
});
await vi.waitFor(() => {
expect(fixture.clearOutput).toHaveBeenCalledOnce();
});
expect(fixture.handleBargeIn).not.toHaveBeenCalled();
expect(
fixture.handle
.getHealth()
.recentTalkEvents.filter((event) => event.type === "turn.cancelled"),
).toHaveLength(1);
releaseTool?.();
await Promise.resolve();
await Promise.resolve();
expect(fixture.submitToolResult).not.toHaveBeenCalled();
expect(
fixture.handle.getHealth().recentTalkEvents.some((event) => event.type === "tool.result"),
).toBe(false);
fixture.releaseWrite(0);
await vi.waitFor(() => {
expect(fixture.clearOutput).toHaveBeenCalledTimes(2);
});
expect(fixture.writeOutput).not.toHaveBeenCalledWith(stale);
fixture.callbacks.onReady?.();
fixture.sendOutputAudio(fresh, "response-1");
await vi.waitFor(() => {
expect(fixture.writeOutput).toHaveBeenCalledTimes(2);
});
expect(fixture.writeOutput).toHaveBeenLastCalledWith(fresh);
fixture.releaseWrite(1);
} finally {
await fixture.handle.stop();
}
});
it("serializes transport writes and coalesces queued 20 ms frames", async () => {
const fixture = await createEngineFixture();
try {

View File

@@ -26,6 +26,7 @@ import {
resolveMeetingRealtimeProvider,
} from "./realtime-engine-support.js";
import { createMeetingRealtimeOutputOwner } from "./realtime-output-owner.js";
import { createMeetingRealtimeToolContinuity } from "./realtime-tool-continuity.js";
export {
formatMeetingAgentAudioModelLog,
@@ -35,7 +36,6 @@ export {
normalizeMeetingTtsPromptText,
resolveMeetingRealtimeTranscriptionProvider,
} from "./realtime-engine-support.js";
export type MeetingRuntimePlatform = {
/** Adapter-owned identity keeps platform names and log prefixes out of core. */
displayName: string;
@@ -98,7 +98,6 @@ const MEETING_REALTIME_OUTPUT_MAX_PENDING_MS = 2_000;
const MEETING_REALTIME_OUTPUT_MAX_WRITE_MS = 500;
const MEETING_REALTIME_OUTPUT_MAX_PENDING_FRAMES = 256;
const MEETING_REALTIME_CANCELLATION_RACE_DETAIL = "Cancellation failed: no active response found";
export async function startMeetingRealtimeEngine(params: {
config: MeetingRealtimeEngineConfig;
fullConfig: OpenClawConfig;
@@ -135,9 +134,11 @@ export async function startMeetingRealtimeEngine(params: {
let outputPendingBytes = 0;
let outputPendingFrames = 0;
let outputGenerationActive = false;
let continuityResetActive = false;
let outputClearTail = Promise.resolve();
const outputQueue: Array<{ audio: Buffer; generation: number }> = [];
const outputOwner = createMeetingRealtimeOutputOwner();
const toolContinuity = createMeetingRealtimeToolContinuity(params.handleToolCall);
const outputMaxPendingBytes =
meetingOutputBytesPerMs(params.config.chrome.audioFormat) *
MEETING_REALTIME_OUTPUT_MAX_PENDING_MS;
@@ -160,6 +161,7 @@ export async function startMeetingRealtimeEngine(params: {
outputOwner.reset();
outputClearAfterActive = false;
invalidateOutputQueue();
toolContinuity.reset("meeting realtime stopped");
}
if (stopPromise) {
await stopPromise;
@@ -549,6 +551,30 @@ export async function startMeetingRealtimeEngine(params: {
}
},
onEvent: (event) => {
if (event.direction === "server" && event.type === "session.created") {
continuityResetActive = false;
}
if (event.direction === "client" && event.type === "session.continuity.reset") {
if (continuityResetActive) {
return;
}
continuityResetActive = true;
realtimeReady = false;
outputOwner.reset();
outputGenerationActive = false;
toolContinuity.reset(event.type);
const turnId = harness.talk.activeTurnId;
invalidateOutputPlayback();
harness.flushOutput(clearOutputPlayback);
harness.finishOutputAudio(event.type);
if (turnId) {
harness.talk.cancelTurn({
turnId,
payload: { ...outputTalkPayload, reason: event.type },
});
}
return;
}
outputOwner.noteEvent(event);
if (event.type === "input_audio_buffer.speech_started") {
harness.ensureTurn();
@@ -600,26 +626,18 @@ export async function startMeetingRealtimeEngine(params: {
);
}
},
onToolCall: (event, session) => {
harness.emit({
type: "tool.call",
turnId: harness.ensureTurn(),
itemId: event.itemId,
callId: event.callId,
payload: { name: event.name, args: event.args },
});
const turnId = harness.ensureTurn();
return params.handleToolCall({
strategy,
onToolCall: (event, session) =>
toolContinuity.run({
session,
event,
meetingSessionId: params.meetingSessionId,
requesterSessionKey: params.requesterSessionKey,
transcript: harness.transcript,
onTalkEvent: (inputLocal) =>
harness.emit({ ...inputLocal, turnId: inputLocal.turnId ?? turnId }),
});
},
call: {
strategy,
event,
meetingSessionId: params.meetingSessionId,
requesterSessionKey: params.requesterSessionKey,
transcript: harness.transcript,
},
harness,
}),
onError: (error) => {
harness.emit({
type: "session.error",
@@ -644,6 +662,7 @@ export async function startMeetingRealtimeEngine(params: {
},
onReady: () => {
realtimeReady = true;
continuityResetActive = false;
harness.emit({
type: "session.ready",
payload: outputTalkPayload,

View File

@@ -0,0 +1,82 @@
import type { RealtimeVoiceToolCallEvent } from "../talk/provider-types.js";
import type { RealtimeVoiceSessionHarness } from "../talk/realtime-session-harness.js";
import type { RealtimeVoiceBridgeSession } from "../talk/session-runtime.js";
import type { TalkEventInput } from "../talk/talk-events.js";
type MeetingRealtimeToolContinuityCall = {
session: RealtimeVoiceBridgeSession;
event: RealtimeVoiceToolCallEvent;
onTalkEvent: (event: TalkEventInput) => void;
};
const meetingRealtimeToolAbortSignals = new WeakMap<RealtimeVoiceBridgeSession, AbortSignal>();
export function readMeetingRealtimeToolAbortSignal(
session: RealtimeVoiceBridgeSession,
): AbortSignal | undefined {
return meetingRealtimeToolAbortSignals.get(session);
}
export function createMeetingRealtimeToolContinuity<
TCall extends MeetingRealtimeToolContinuityCall,
>(handleToolCall: (call: TCall) => Promise<void>) {
let epoch = 0;
const activeControllers = new Set<AbortController>();
// Reset invalidates provider submissions and Talk events together so late
// async completions cannot leak into the replacement provider generation.
const reset = (reason: string) => {
epoch += 1;
for (const controller of activeControllers) {
controller.abort(reason);
}
activeControllers.clear();
};
const run = (params: {
session: RealtimeVoiceBridgeSession;
call: Omit<TCall, "session" | "onTalkEvent">;
harness: Pick<RealtimeVoiceSessionHarness, "emit" | "ensureTurn">;
}): Promise<void> => {
const callEpoch = epoch;
const controller = new AbortController();
activeControllers.add(controller);
const isActive = () => !controller.signal.aborted && callEpoch === epoch;
const turnId = params.harness.ensureTurn();
params.harness.emit({
type: "tool.call",
turnId,
itemId: params.call.event.itemId,
callId: params.call.event.callId,
payload: { name: params.call.event.name, args: params.call.event.args },
});
const guardedSession = Object.create(params.session) as RealtimeVoiceBridgeSession;
meetingRealtimeToolAbortSignals.set(guardedSession, controller.signal);
guardedSession.submitToolResult = (callId, result, options) => {
if (!isActive()) {
return;
}
return params.session.submitToolResult(callId, result, options);
};
return handleToolCall({
...params.call,
session: guardedSession,
onTalkEvent: (event) => {
if (isActive()) {
params.harness.emit({ ...event, turnId: event.turnId ?? turnId });
}
},
} as TCall)
.catch((error: unknown) => {
if (isActive()) {
throw error;
}
})
.finally(() => {
meetingRealtimeToolAbortSignals.delete(guardedSession);
activeControllers.delete(controller);
});
};
return { reset, run };
}