mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-02 00:31:36 +00:00
merge(voice): preserve realtime reconnect continuity (#117599)
This commit is contained in:
@@ -44,7 +44,8 @@ export type TestRealtimeBridgeParams = {
|
||||
cfg?: unknown;
|
||||
instructions?: string;
|
||||
interruptResponseOnInputAudio?: boolean;
|
||||
onEvent?: (event: { detail?: string; direction: "server"; type: string }) => void;
|
||||
onEvent?: (event: { detail?: string; direction: "client" | "server"; type: string }) => void;
|
||||
onReady?: () => void;
|
||||
onToolCall?: (
|
||||
event: { args: unknown; callId: string; itemId: string; name: string },
|
||||
session: unknown,
|
||||
|
||||
@@ -3263,6 +3263,98 @@ describe("DiscordVoiceManager", () => {
|
||||
expect(player.stop).toHaveBeenCalledTimes(stopCallsAfterControl + 1);
|
||||
});
|
||||
|
||||
it("drops stale active-run control after provider continuity reset", async () => {
|
||||
let resolveOldControl: ((result: RealtimeVoiceAgentControlResult) => void) | undefined;
|
||||
controlRealtimeVoiceAgentRunMock
|
||||
.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
resolveOldControl = resolve;
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
mode: "cancel",
|
||||
sessionKey: "discord:g1:c1",
|
||||
sessionId: "embedded-fresh",
|
||||
active: true,
|
||||
aborted: true,
|
||||
message: "Fresh control result.",
|
||||
speak: true,
|
||||
show: true,
|
||||
suppress: false,
|
||||
});
|
||||
const manager = createAgentProxyManager();
|
||||
|
||||
await manager.join({ guildId: "g1", channelId: "1001" });
|
||||
const bridgeParams = lastRealtimeBridgeParams();
|
||||
bridgeParams?.onTranscript?.("user", "cancel that", true);
|
||||
await vi.waitFor(() => expect(controlRealtimeVoiceAgentRunMock).toHaveBeenCalledTimes(1));
|
||||
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
resolveOldControl?.({
|
||||
ok: true,
|
||||
mode: "cancel",
|
||||
sessionKey: "discord:g1:c1",
|
||||
sessionId: "embedded-old",
|
||||
active: true,
|
||||
aborted: true,
|
||||
message: "Stale control result.",
|
||||
speak: true,
|
||||
show: true,
|
||||
suppress: false,
|
||||
});
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
expectUserMessageNotIncludes("Stale control result.");
|
||||
expect(realtimeSessionMock.handleBargeIn).not.toHaveBeenCalled();
|
||||
|
||||
bridgeParams?.onReady?.();
|
||||
bridgeParams?.onTranscript?.("user", "stop that", true);
|
||||
await vi.waitFor(() => expectUserMessageIncludes("Fresh control result."));
|
||||
expect(realtimeSessionMock.handleBargeIn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("replaces stale talkback work across provider continuity reset", async () => {
|
||||
let resolveOldTalkback: ((result: { payloads: Array<{ text: string }> }) => void) | undefined;
|
||||
agentCommandMock
|
||||
.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
resolveOldTalkback = resolve;
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce({ payloads: [{ text: "fresh talkback" }] });
|
||||
const manager = createManager({
|
||||
groupPolicy: "open",
|
||||
voice: {
|
||||
enabled: true,
|
||||
mode: "agent-proxy",
|
||||
realtime: { provider: "openai", debounceMs: 1, toolPolicy: "none" },
|
||||
},
|
||||
});
|
||||
|
||||
await manager.join({ guildId: "g1", channelId: "1001" });
|
||||
const entry = getSessionEntry(manager);
|
||||
const bridgeParams = lastRealtimeBridgeParams();
|
||||
beginSpeakerTurn(entry);
|
||||
await emitFinalRealtimeUserTranscript(bridgeParams, "old question");
|
||||
await vi.waitFor(() => expect(agentCommandMock).toHaveBeenCalledTimes(1));
|
||||
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onReady?.();
|
||||
beginSpeakerTurn(entry);
|
||||
await emitFinalRealtimeUserTranscript(bridgeParams, "fresh question");
|
||||
|
||||
await vi.waitFor(() => expect(agentCommandMock).toHaveBeenCalledTimes(2));
|
||||
await vi.waitFor(() => expectUserMessageIncludes("fresh talkback"));
|
||||
resolveOldTalkback?.({ payloads: [{ text: "stale talkback" }] });
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expectUserMessageNotIncludes("stale talkback");
|
||||
});
|
||||
|
||||
it("preserves realtime forced consults when no active run accepts steering", async () => {
|
||||
agentCommandMock.mockResolvedValueOnce({ payloads: [{ text: "normal answer" }] });
|
||||
const manager = createAgentProxyManager();
|
||||
@@ -3418,6 +3510,203 @@ describe("DiscordVoiceManager", () => {
|
||||
expectUserMessageIncludes("wake answer");
|
||||
});
|
||||
|
||||
it("does not carry partial wake-name state across provider continuity resets", async () => {
|
||||
const { entry, bridgeParams } = await createWakeNameFixture();
|
||||
const wakeAckCount = () =>
|
||||
sentUserMessages().filter((message) => message.includes('Answer: "Yeah."')).length;
|
||||
|
||||
beginSpeakerTurn(entry);
|
||||
bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" });
|
||||
bridgeParams?.onTranscript?.("user", "Hey, Mol", false);
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onTranscript?.("user", "ty", false);
|
||||
|
||||
expect(wakeAckCount()).toBe(0);
|
||||
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onReady?.();
|
||||
bridgeParams?.onTranscript?.("user", "Hey, Molty", false);
|
||||
expect(wakeAckCount()).toBe(1);
|
||||
bridgeParams?.onEvent?.({ direction: "server", type: "response.done" });
|
||||
});
|
||||
|
||||
it("preserves the wake-name acknowledgement across provider continuity resets", async () => {
|
||||
const { entry, bridgeParams } = await createWakeNameFixture();
|
||||
const wakeAckCount = () =>
|
||||
sentUserMessages().filter((message) => message.includes('Answer: "')).length;
|
||||
|
||||
beginSpeakerTurn(entry);
|
||||
bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" });
|
||||
bridgeParams?.onTranscript?.("user", "Hey, Molty", false);
|
||||
expect(wakeAckCount()).toBe(1);
|
||||
bridgeParams?.onEvent?.({ direction: "server", type: "response.done" });
|
||||
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onReady?.();
|
||||
bridgeParams?.onTranscript?.("user", "Hey, Molty", false);
|
||||
expect(wakeAckCount()).toBe(1);
|
||||
|
||||
bridgeParams?.onEvent?.({ direction: "server", type: "input_audio_buffer.speech_started" });
|
||||
bridgeParams?.onTranscript?.("user", "Hey, Molty", false);
|
||||
expect(wakeAckCount()).toBe(2);
|
||||
});
|
||||
|
||||
it("replays zero-audio exact speech once after provider continuity reset", async () => {
|
||||
agentCommandMock
|
||||
.mockResolvedValueOnce({ payloads: [{ text: "first answer" }] })
|
||||
.mockResolvedValueOnce({ payloads: [{ text: "second answer" }] });
|
||||
const manager = createAgentProxyManager();
|
||||
|
||||
await manager.join({ guildId: "g1", channelId: "1001" });
|
||||
const entry = getSessionEntry(manager);
|
||||
const player = getLastAudioPlayer();
|
||||
const bridgeParams = lastRealtimeBridgeParams();
|
||||
|
||||
beginSpeakerTurn(entry);
|
||||
await emitFinalRealtimeUserTranscript(bridgeParams, "first question");
|
||||
await vi.waitFor(() => expectUserMessageIncludes("first answer"));
|
||||
beginSpeakerTurn(entry);
|
||||
await emitFinalRealtimeUserTranscript(bridgeParams, "second question");
|
||||
expectUserMessageNotIncludes("second answer");
|
||||
|
||||
const stopCallsBeforeReset = player.stop.mock.calls.length;
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
expectUserMessageNotIncludes("second answer");
|
||||
expect(player.stop).toHaveBeenCalledTimes(stopCallsBeforeReset + 1);
|
||||
expect(realtimeSessionMock.handleBargeIn).not.toHaveBeenCalled();
|
||||
expect(realtimeSessionMock.close).not.toHaveBeenCalled();
|
||||
|
||||
bridgeParams?.onReady?.();
|
||||
expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength(
|
||||
2,
|
||||
);
|
||||
expectUserMessageNotIncludes("second answer");
|
||||
bridgeParams?.onEvent?.({ direction: "server", type: "response.done" });
|
||||
expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength(
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it("replays exact speech buffered below playback preroll after continuity reset", async () => {
|
||||
agentCommandMock
|
||||
.mockResolvedValueOnce({ payloads: [{ text: "first answer" }] })
|
||||
.mockResolvedValueOnce({ payloads: [{ text: "second answer" }] });
|
||||
const manager = createAgentProxyManager();
|
||||
|
||||
await manager.join({ guildId: "g1", channelId: "1001" });
|
||||
const entry = getSessionEntry(manager);
|
||||
const player = getLastAudioPlayer();
|
||||
const bridgeParams = lastRealtimeBridgeParams();
|
||||
|
||||
beginSpeakerTurn(entry);
|
||||
await emitFinalRealtimeUserTranscript(bridgeParams, "first question");
|
||||
await vi.waitFor(() => expectUserMessageIncludes("first answer"));
|
||||
bridgeParams?.audioSink?.sendAudio(Buffer.alloc(480));
|
||||
expect(player.play).not.toHaveBeenCalled();
|
||||
beginSpeakerTurn(entry);
|
||||
await emitFinalRealtimeUserTranscript(bridgeParams, "second question");
|
||||
expectUserMessageNotIncludes("second answer");
|
||||
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onReady?.();
|
||||
|
||||
expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength(
|
||||
2,
|
||||
);
|
||||
expectUserMessageNotIncludes("second answer");
|
||||
bridgeParams?.onEvent?.({ direction: "server", type: "response.done" });
|
||||
expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength(
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not replay exact speech after Discord playback starts", async () => {
|
||||
agentCommandMock
|
||||
.mockResolvedValueOnce({ payloads: [{ text: "first answer" }] })
|
||||
.mockResolvedValueOnce({ payloads: [{ text: "second answer" }] });
|
||||
const manager = createAgentProxyManager();
|
||||
|
||||
await manager.join({ guildId: "g1", channelId: "1001" });
|
||||
const entry = getSessionEntry(manager);
|
||||
const player = getLastAudioPlayer();
|
||||
const bridgeParams = lastRealtimeBridgeParams();
|
||||
|
||||
beginSpeakerTurn(entry);
|
||||
await emitFinalRealtimeUserTranscript(bridgeParams, "first question");
|
||||
await vi.waitFor(() => expectUserMessageIncludes("first answer"));
|
||||
for (let index = 0; index < 50; index += 1) {
|
||||
bridgeParams?.audioSink?.sendAudio(Buffer.alloc(480));
|
||||
}
|
||||
expect(player.play).toHaveBeenCalledOnce();
|
||||
beginSpeakerTurn(entry);
|
||||
await emitFinalRealtimeUserTranscript(bridgeParams, "second question");
|
||||
expectUserMessageNotIncludes("second answer");
|
||||
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
bridgeParams?.onReady?.();
|
||||
|
||||
expect(sentUserMessages().filter((message) => message.includes("first answer"))).toHaveLength(
|
||||
1,
|
||||
);
|
||||
expect(sentUserMessages().filter((message) => message.includes("second answer"))).toHaveLength(
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it("drops stale native consult delivery after provider continuity reset", async () => {
|
||||
let resolveOld: ((result: { payloads: Array<{ text: string }> }) => void) | undefined;
|
||||
agentCommandMock
|
||||
.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
resolveOld = resolve;
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce({ payloads: [{ text: "fresh answer" }] });
|
||||
const manager = createAgentProxyManager();
|
||||
|
||||
await manager.join({ guildId: "g1", channelId: "1001" });
|
||||
const entry = getSessionEntry(manager);
|
||||
const bridgeParams = lastRealtimeBridgeParams();
|
||||
beginSpeakerTurn(entry);
|
||||
const oldSubmission = bridgeParams?.onToolCall?.(
|
||||
{
|
||||
itemId: "item-old",
|
||||
callId: "call-old",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "same question" },
|
||||
},
|
||||
realtimeSessionMock,
|
||||
);
|
||||
await Promise.resolve();
|
||||
|
||||
bridgeParams?.onEvent?.({ direction: "client", type: "session.continuity.reset" });
|
||||
resolveOld?.({ payloads: [{ text: "stale answer" }] });
|
||||
await oldSubmission;
|
||||
expect(
|
||||
realtimeSessionMock.submitToolResult.mock.calls.some(([callId]) => callId === "call-old"),
|
||||
).toBe(false);
|
||||
|
||||
bridgeParams?.onReady?.();
|
||||
beginSpeakerTurn(entry);
|
||||
await bridgeParams?.onToolCall?.(
|
||||
{
|
||||
itemId: "item-fresh",
|
||||
callId: "call-fresh",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "same question" },
|
||||
},
|
||||
realtimeSessionMock,
|
||||
);
|
||||
expect(realtimeSessionMock.submitToolResult).toHaveBeenCalledWith("call-fresh", {
|
||||
text: "fresh answer",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats a bare wake name as an activation for the next realtime transcript", async () => {
|
||||
agentCommandMock.mockResolvedValueOnce({ payloads: [{ text: "follow-up answer" }] });
|
||||
const onUtterance = vi.fn();
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
buildRealtimeVoiceAgentConsultPolicyInstructions,
|
||||
classifySkippableRealtimeVoiceConsultTranscript,
|
||||
controlRealtimeVoiceAgentRun,
|
||||
createRealtimeVoiceAgentTalkbackQueue,
|
||||
createRealtimeVoiceSessionHarness,
|
||||
createRealtimeVoiceTurnContextTracker,
|
||||
matchRealtimeVoiceActivationName,
|
||||
@@ -26,6 +27,7 @@ import {
|
||||
type RealtimeVoiceBridgeEvent,
|
||||
type RealtimeVoiceAgentConsultToolPolicy,
|
||||
type RealtimeVoiceAgentControlResult,
|
||||
type RealtimeVoiceAgentTalkbackQueue,
|
||||
type RealtimeVoiceBridgeSession,
|
||||
type RealtimeVoiceProviderConfig,
|
||||
type RealtimeVoiceToolCallEvent,
|
||||
@@ -133,8 +135,10 @@ type RecentAgentProxyConsultResult =
|
||||
|
||||
type AgentProxyConsultState = {
|
||||
speaker: DiscordRealtimeSpeakerContext;
|
||||
providerEpoch: number;
|
||||
handledByForcedPlayback?: boolean;
|
||||
providerDelivery?: Promise<boolean>;
|
||||
settleProviderDelivery?: (accepted: boolean) => void;
|
||||
promise?: Promise<string>;
|
||||
result?: RecentAgentProxyConsultResult;
|
||||
};
|
||||
@@ -324,6 +328,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
private bridge: RealtimeVoiceBridgeSession | null = null;
|
||||
private outputStream: PassThrough | null = null;
|
||||
private readonly harness: RealtimeVoiceSessionHarness<AgentProxyConsultState>;
|
||||
private talkback: RealtimeVoiceAgentTalkbackQueue;
|
||||
private stopped = false;
|
||||
private consultToolPolicy: RealtimeVoiceAgentConsultToolPolicy = "safe-read-only";
|
||||
private consultToolsAllow: string[] | undefined;
|
||||
@@ -347,6 +352,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
private queuedExactSpeechMessages: string[] = [];
|
||||
private exactSpeechResponseActive = false;
|
||||
private exactSpeechAudioStarted = false;
|
||||
private activeExactSpeechMessage: string | undefined;
|
||||
private bridgeReady = false;
|
||||
private providerGenerationObserved = false;
|
||||
private providerContinuityEpoch = 0;
|
||||
private partialUserTranscript = "";
|
||||
private wakeNameAckedForTurn = false;
|
||||
private wakeNameAckIndex = 0;
|
||||
@@ -397,34 +406,40 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
outputAudioDelta: discordRealtimeTalkPayload,
|
||||
outputAudioDone: discordRealtimeTalkPayload,
|
||||
},
|
||||
talkback: {
|
||||
debounceMs: this.realtimeConfig?.debounceMs ?? DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS,
|
||||
logger,
|
||||
logPrefix: "[discord] realtime agent",
|
||||
responseStyle: "Brief, natural spoken answer for a Discord voice channel.",
|
||||
fallbackText: DISCORD_REALTIME_FALLBACK_TEXT,
|
||||
consult: async ({ question, responseStyle, metadata }) => {
|
||||
const context = isDiscordRealtimeSpeakerContext(metadata) ? metadata : undefined;
|
||||
return {
|
||||
text: await this.runAgentTurn({
|
||||
context,
|
||||
message: formatVoiceIngressPrompt(
|
||||
[question, responseStyle ? `Spoken style: ${responseStyle}` : undefined]
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
context?.speakerLabel ?? "Discord voice speaker",
|
||||
),
|
||||
}),
|
||||
};
|
||||
},
|
||||
deliver: (text) => this.enqueueExactSpeechMessage(text),
|
||||
},
|
||||
forcedConsults: {
|
||||
limit: DISCORD_REALTIME_RECENT_AGENT_PROXY_CONSULT_LIMIT,
|
||||
nativeDedupeMs: DISCORD_REALTIME_RECENT_AGENT_PROXY_CONSULT_TTL_MS,
|
||||
questionsMatch: matchRealtimeVoiceConsultQuestions,
|
||||
},
|
||||
});
|
||||
this.talkback = this.createTalkbackQueue();
|
||||
}
|
||||
|
||||
private createTalkbackQueue(): RealtimeVoiceAgentTalkbackQueue {
|
||||
const providerEpoch = this.providerContinuityEpoch;
|
||||
return createRealtimeVoiceAgentTalkbackQueue({
|
||||
debounceMs: this.realtimeConfig?.debounceMs ?? DISCORD_REALTIME_TALKBACK_DEBOUNCE_MS,
|
||||
isStopped: () => this.stopped || providerEpoch !== this.providerContinuityEpoch,
|
||||
logger,
|
||||
logPrefix: "[discord] realtime agent",
|
||||
responseStyle: "Brief, natural spoken answer for a Discord voice channel.",
|
||||
fallbackText: DISCORD_REALTIME_FALLBACK_TEXT,
|
||||
consult: async ({ question, responseStyle, metadata }) => {
|
||||
const context = isDiscordRealtimeSpeakerContext(metadata) ? metadata : undefined;
|
||||
return {
|
||||
text: await this.runAgentTurn({
|
||||
context,
|
||||
message: formatVoiceIngressPrompt(
|
||||
[question, responseStyle ? `Spoken style: ${responseStyle}` : undefined]
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
context?.speakerLabel ?? "Discord voice speaker",
|
||||
),
|
||||
}),
|
||||
};
|
||||
},
|
||||
deliver: (text) => this.enqueueExactSpeechMessage(text),
|
||||
});
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
@@ -493,10 +508,14 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
audioSink: {
|
||||
isOpen: () => !this.stopped,
|
||||
sendAudio: (audio) => this.sendOutputAudio(audio),
|
||||
clearAudio: () =>
|
||||
this.harness.flushOutput(() => this.clearOutputAudio("provider-clear-audio")),
|
||||
clearAudio: () => {
|
||||
this.markProviderGenerationObserved();
|
||||
this.harness.flushOutput(() => this.clearOutputAudio("provider-clear-audio"));
|
||||
},
|
||||
},
|
||||
onTranscript: (role, text, isFinal) => {
|
||||
this.markProviderGenerationObserved();
|
||||
const providerEpoch = this.providerContinuityEpoch;
|
||||
if (isFinal && text.trim()) {
|
||||
logger.info(
|
||||
`discord voice: realtime ${role} transcript (${text.length} chars): ${formatVoiceLogPreview(text)}`,
|
||||
@@ -512,11 +531,28 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
this.handlePartialUserTranscript(text);
|
||||
return;
|
||||
}
|
||||
void this.handleFinalUserTranscript(text, { usesRealtimeAgentHandoff });
|
||||
void this.handleFinalUserTranscript(text, {
|
||||
providerEpoch,
|
||||
usesRealtimeAgentHandoff,
|
||||
});
|
||||
},
|
||||
onToolCall: (event, session) => {
|
||||
this.markProviderGenerationObserved();
|
||||
return this.handleToolCall(event, session);
|
||||
},
|
||||
onReady: () => {
|
||||
this.markProviderGenerationObserved();
|
||||
this.bridgeReady = true;
|
||||
this.drainQueuedExactSpeechMessages("provider-ready");
|
||||
},
|
||||
onToolCall: (event, session) => this.handleToolCall(event, session),
|
||||
onEvent: (event) => {
|
||||
if (!(event.direction === "client" && event.type === "session.continuity.reset")) {
|
||||
this.markProviderGenerationObserved();
|
||||
}
|
||||
const detail = event.detail ? ` ${event.detail}` : "";
|
||||
if (event.direction === "client" && event.type === "session.continuity.reset") {
|
||||
this.resetProviderContinuity(event.type);
|
||||
}
|
||||
if (event.direction === "server" && event.type === "input_audio_buffer.speech_started") {
|
||||
this.resetPartialWakeNameTracking();
|
||||
}
|
||||
@@ -571,6 +607,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
const voiceSdk = loadDiscordVoiceSdk();
|
||||
this.params.entry.player.on(voiceSdk.AudioPlayerStatus.Idle, this.playerIdleHandler);
|
||||
await this.bridge.connect();
|
||||
// Some provider/test bridges do not expose an explicit ready callback.
|
||||
this.markProviderGenerationObserved();
|
||||
this.bridgeReady = true;
|
||||
this.drainQueuedExactSpeechMessages("provider-connected");
|
||||
logger.info(
|
||||
`discord voice: realtime bridge ready mode=${this.params.mode} provider=${resolved.provider.id} model=${resolvedModel ?? "default"} voice=${resolvedVoice ?? "default"}`,
|
||||
);
|
||||
@@ -578,13 +618,18 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
|
||||
close(): void {
|
||||
this.stopped = true;
|
||||
this.bridgeReady = false;
|
||||
this.providerContinuityEpoch += 1;
|
||||
this.outputBackpressure = undefined;
|
||||
this.flushSuppressedRealtimeErrors();
|
||||
this.clearProviderConsultState();
|
||||
this.talkback.close();
|
||||
this.harness.close();
|
||||
this.speakerTurns.clear();
|
||||
this.queuedExactSpeechMessages = [];
|
||||
this.exactSpeechResponseActive = false;
|
||||
this.exactSpeechAudioStarted = false;
|
||||
this.activeExactSpeechMessage = undefined;
|
||||
this.resetPartialWakeNameTracking();
|
||||
this.pendingWakeNameFollowup = undefined;
|
||||
this.clearOutputAudio("session-close");
|
||||
@@ -733,6 +778,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
}
|
||||
|
||||
private sendOutputAudio(realtimePcm24kMono: Buffer): void {
|
||||
this.markProviderGenerationObserved();
|
||||
if (this.stopped || this.outputBackpressure) {
|
||||
return;
|
||||
}
|
||||
@@ -948,7 +994,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
if (this.stopped || !text.trim()) {
|
||||
return;
|
||||
}
|
||||
if (this.exactSpeechResponseActive || this.hasInterruptibleOutputAudio()) {
|
||||
if (!this.bridgeReady || this.exactSpeechResponseActive || this.hasInterruptibleOutputAudio()) {
|
||||
this.queuedExactSpeechMessages.push(text);
|
||||
logger.info(
|
||||
`discord voice: realtime exact speech queued guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()}`,
|
||||
@@ -964,6 +1010,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
}
|
||||
this.exactSpeechResponseActive = true;
|
||||
this.exactSpeechAudioStarted = false;
|
||||
this.activeExactSpeechMessage = text;
|
||||
this.bridge?.sendUserMessage(buildDiscordSpeakExactUserMessage(text));
|
||||
}
|
||||
|
||||
@@ -983,7 +1030,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
logger.info(
|
||||
`discord voice: realtime wake-name ack canonical=${result.activationName} heard=${result.heardName} match=${result.match} voiceSession=${this.params.entry.voiceSessionKey} agent=${this.params.entry.route.agentId}`,
|
||||
);
|
||||
this.sendExactSpeechMessage(ack ?? "Yeah.");
|
||||
this.enqueueExactSpeechMessage(ack ?? "Yeah.");
|
||||
}
|
||||
|
||||
private speakControlResult(text: string): void {
|
||||
@@ -1001,7 +1048,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
sentAt: Date.now(),
|
||||
assistantTranscriptCount: 0,
|
||||
};
|
||||
this.sendExactSpeechMessage(trimmed);
|
||||
this.enqueueExactSpeechMessage(trimmed);
|
||||
}
|
||||
|
||||
private suppressDuplicateControlSpeech(text: string): void {
|
||||
@@ -1034,6 +1081,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
}
|
||||
this.exactSpeechResponseActive = false;
|
||||
this.exactSpeechAudioStarted = false;
|
||||
this.activeExactSpeechMessage = undefined;
|
||||
if (options?.drain === false) {
|
||||
return;
|
||||
}
|
||||
@@ -1043,6 +1091,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
private drainQueuedExactSpeechMessages(reason: string): void {
|
||||
if (
|
||||
this.stopped ||
|
||||
!this.bridgeReady ||
|
||||
this.exactSpeechResponseActive ||
|
||||
this.queuedExactSpeechMessages.length === 0 ||
|
||||
this.hasInterruptibleOutputAudio()
|
||||
@@ -1123,9 +1172,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
event: RealtimeVoiceToolCallEvent,
|
||||
session: RealtimeVoiceBridgeSession,
|
||||
): Promise<void> {
|
||||
const providerEpoch = this.providerContinuityEpoch;
|
||||
const callId = event.callId || event.itemId || "unknown";
|
||||
if (event.name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) {
|
||||
await this.handleAgentControlToolCall(event, session, callId);
|
||||
await this.handleAgentControlToolCall(event, session, callId, providerEpoch);
|
||||
return;
|
||||
}
|
||||
if (event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
|
||||
@@ -1223,11 +1273,17 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
try {
|
||||
text = await promise;
|
||||
} catch (error) {
|
||||
if (providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
const message = formatErrorMessage(error);
|
||||
logger.warn(`discord voice: realtime consult failed call=${callId || "unknown"}: ${message}`);
|
||||
await session.submitToolResult(callId, { error: message });
|
||||
return;
|
||||
}
|
||||
if (providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
logger.info(
|
||||
`discord voice: realtime consult answer (${text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(text)}`,
|
||||
);
|
||||
@@ -1238,6 +1294,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
event: RealtimeVoiceToolCallEvent,
|
||||
session: RealtimeVoiceBridgeSession,
|
||||
callId: string,
|
||||
providerEpoch: number,
|
||||
): Promise<void> {
|
||||
let result: RealtimeVoiceAgentControlResult;
|
||||
try {
|
||||
@@ -1248,9 +1305,15 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
mode: parsed.mode,
|
||||
});
|
||||
} catch (error) {
|
||||
if (providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
await session.submitToolResult(callId, { error: formatErrorMessage(error) });
|
||||
return;
|
||||
}
|
||||
if (providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
this.logAgentControlResult(result);
|
||||
await session.submitToolResult(callId, result);
|
||||
}
|
||||
@@ -1273,7 +1336,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
|
||||
private async handleFinalUserTranscript(
|
||||
text: string,
|
||||
params: { usesRealtimeAgentHandoff: boolean },
|
||||
params: { providerEpoch: number; usesRealtimeAgentHandoff: boolean },
|
||||
): Promise<void> {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) {
|
||||
@@ -1316,15 +1379,24 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
usesAgentProxy && params.usesRealtimeAgentHandoff
|
||||
? this.prepareForcedAgentProxyConsult(acceptedText, forcedSpeakerContext)
|
||||
: undefined;
|
||||
const control = await maybeControlDiscordVoiceAgentRun({
|
||||
entry: this.params.entry,
|
||||
text: acceptedText,
|
||||
}).catch((error: unknown) => {
|
||||
let control: Awaited<ReturnType<typeof maybeControlDiscordVoiceAgentRun>> | undefined;
|
||||
try {
|
||||
control = await maybeControlDiscordVoiceAgentRun({
|
||||
entry: this.params.entry,
|
||||
text: acceptedText,
|
||||
});
|
||||
} catch (error) {
|
||||
if (params.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
logger.warn(
|
||||
`discord voice: realtime active-run control failed; falling back to normal transcript handling: ${formatErrorMessage(error)}`,
|
||||
);
|
||||
return undefined;
|
||||
});
|
||||
control = undefined;
|
||||
}
|
||||
if (params.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
if (control?.handled) {
|
||||
if (pendingForcedConsult) {
|
||||
this.harness.forcedConsults.remove(pendingForcedConsult);
|
||||
@@ -1345,7 +1417,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
}
|
||||
return;
|
||||
}
|
||||
this.harness.talkback?.enqueue(
|
||||
this.talkback.enqueue(
|
||||
acceptedText,
|
||||
forcedSpeakerContext ?? this.consumePendingSpeakerContext(),
|
||||
);
|
||||
@@ -1372,6 +1444,52 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
this.wakeNameAckedForTurn = false;
|
||||
}
|
||||
|
||||
private markProviderGenerationObserved(): void {
|
||||
this.providerGenerationObserved = true;
|
||||
}
|
||||
|
||||
private resetProviderContinuity(reason: string): void {
|
||||
if (!this.providerGenerationObserved) {
|
||||
return;
|
||||
}
|
||||
this.providerGenerationObserved = false;
|
||||
this.bridgeReady = false;
|
||||
this.providerContinuityEpoch += 1;
|
||||
this.talkback.close();
|
||||
this.talkback = this.createTalkbackQueue();
|
||||
this.outputBackpressure = undefined;
|
||||
this.partialUserTranscript = "";
|
||||
this.pendingWakeNameFollowup = undefined;
|
||||
this.lastControlSpeech = undefined;
|
||||
this.clearProviderConsultState();
|
||||
const replayExactSpeech =
|
||||
this.exactSpeechResponseActive && !this.harness.outputActivity.snapshot().playbackStarted
|
||||
? this.activeExactSpeechMessage
|
||||
: undefined;
|
||||
this.exactSpeechResponseActive = false;
|
||||
this.exactSpeechAudioStarted = false;
|
||||
this.activeExactSpeechMessage = undefined;
|
||||
if (replayExactSpeech) {
|
||||
this.queuedExactSpeechMessages.unshift(replayExactSpeech);
|
||||
}
|
||||
this.harness.flushOutput(() => this.clearOutputAudio(reason));
|
||||
this.harness.finishOutputAudio(reason);
|
||||
}
|
||||
|
||||
private clearProviderConsultState(): void {
|
||||
for (const handle of this.harness.forcedConsults.handles()) {
|
||||
const state = handle.context;
|
||||
if (!state) {
|
||||
continue;
|
||||
}
|
||||
state.handledByForcedPlayback = false;
|
||||
state.settleProviderDelivery?.(false);
|
||||
state.settleProviderDelivery = undefined;
|
||||
state.providerDelivery = undefined;
|
||||
}
|
||||
this.harness.forcedConsults.clear();
|
||||
}
|
||||
|
||||
private resolveWakeNameTranscript(
|
||||
text: string,
|
||||
requireWakeName: boolean,
|
||||
@@ -1477,7 +1595,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
return undefined;
|
||||
}
|
||||
return this.harness.forcedConsults.prepare(question, {
|
||||
context: { speaker: context },
|
||||
context: { speaker: context, providerEpoch: this.providerContinuityEpoch },
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1498,7 +1616,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
}
|
||||
const context = state.speaker;
|
||||
const { question } = pending;
|
||||
if (this.stopped) {
|
||||
if (this.stopped || state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
this.harness.forcedConsults.markCancelled(pending);
|
||||
return;
|
||||
}
|
||||
@@ -1523,6 +1641,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
this.setRecentAgentProxyConsultPromise(pending, promise);
|
||||
const text = await promise;
|
||||
await state.providerDelivery;
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
logger.info(
|
||||
`discord voice: realtime forced agent consult answer (${text.length} chars) elapsedMs=${Date.now() - startedAt} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}: ${formatVoiceLogPreview(text)}`,
|
||||
);
|
||||
@@ -1531,6 +1652,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
}
|
||||
} catch (error) {
|
||||
await state.providerDelivery;
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
logger.warn(
|
||||
`discord voice: realtime forced agent consult failed elapsedMs=${Date.now() - startedAt}: ${formatErrorMessage(error)}`,
|
||||
);
|
||||
@@ -1612,7 +1736,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
options: { id?: string; started?: boolean } = {},
|
||||
): AgentProxyConsultHandle {
|
||||
const handle = this.harness.forcedConsults.prepare(question, {
|
||||
context: { speaker: context },
|
||||
context: { speaker: context, providerEpoch: this.providerContinuityEpoch },
|
||||
...(options.id ? { id: options.id } : {}),
|
||||
});
|
||||
if (!handle) {
|
||||
@@ -1636,10 +1760,16 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
state.promise = promise;
|
||||
void promise
|
||||
.then((text) => {
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
state.result = { status: "fulfilled", text };
|
||||
this.harness.forcedConsults.markDelivered(recent);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
state.result = { status: "rejected", error: formatErrorMessage(error) };
|
||||
this.harness.forcedConsults.markDelivered(recent);
|
||||
});
|
||||
@@ -1674,6 +1804,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
if (!state) {
|
||||
return false;
|
||||
}
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return true;
|
||||
}
|
||||
const providerOwnsDelivery = Boolean(
|
||||
state.handledByForcedPlayback &&
|
||||
state.promise &&
|
||||
@@ -1686,15 +1819,22 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
// the local success/fallback path instead of losing the answer entirely.
|
||||
state.providerDelivery = new Promise<boolean>((resolve) => {
|
||||
resolveProviderDelivery = resolve;
|
||||
state.settleProviderDelivery = resolve;
|
||||
});
|
||||
}
|
||||
const submitAlreadyDelivered = async (): Promise<void> => {
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
await this.submitTerminalRealtimeToolResult(callId, session, {
|
||||
status: "already_delivered",
|
||||
message: "OpenClaw already delivered this answer to Discord voice. Do not repeat it.",
|
||||
});
|
||||
};
|
||||
const submitResult = async (result: RecentAgentProxyConsultResult): Promise<void> => {
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return;
|
||||
}
|
||||
if (state.handledByForcedPlayback && !providerOwnsDelivery) {
|
||||
await submitAlreadyDelivered();
|
||||
return;
|
||||
@@ -1720,6 +1860,9 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
);
|
||||
if (state.handledByForcedPlayback && !providerOwnsDelivery) {
|
||||
await state.promise.catch(() => undefined);
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return true;
|
||||
}
|
||||
await submitAlreadyDelivered();
|
||||
return true;
|
||||
}
|
||||
@@ -1729,13 +1872,18 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
|
||||
} catch (error) {
|
||||
result = { status: "rejected", error: formatErrorMessage(error) };
|
||||
}
|
||||
if (state.providerEpoch !== this.providerContinuityEpoch) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
await submitResult(result);
|
||||
if (providerOwnsDelivery) {
|
||||
state.handledByForcedPlayback = false;
|
||||
state.settleProviderDelivery = undefined;
|
||||
resolveProviderDelivery?.(true);
|
||||
}
|
||||
} catch (error) {
|
||||
state.settleProviderDelivery = undefined;
|
||||
resolveProviderDelivery?.(false);
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -815,6 +815,9 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
||||
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
|
||||
serverContent: { inputTranscription: { text: "Before " } },
|
||||
});
|
||||
firstSession.onmessage({
|
||||
sessionResumptionUpdate: { newHandle: "unconfirmed-handle" },
|
||||
});
|
||||
firstSession.onclose({ code: 1011, reason: "temporary" });
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
lastConnectParams().callbacks.onmessage({
|
||||
@@ -827,6 +830,46 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves continuity when resumability recovers before reconnect", async () => {
|
||||
vi.useFakeTimers();
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const onEvent = vi.fn();
|
||||
const onTranscript = vi.fn();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: { apiKey: "gemini-key" },
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
onEvent,
|
||||
onTranscript,
|
||||
});
|
||||
|
||||
await bridge.connect();
|
||||
const firstSession = lastConnectParams().callbacks;
|
||||
firstSession.onmessage({
|
||||
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
|
||||
serverContent: { inputTranscription: { text: "Before " } },
|
||||
});
|
||||
firstSession.onmessage({
|
||||
sessionResumptionUpdate: { resumable: false },
|
||||
});
|
||||
firstSession.onmessage({
|
||||
sessionResumptionUpdate: { resumable: true, newHandle: "resume-2" },
|
||||
});
|
||||
expect(onEvent).not.toHaveBeenCalled();
|
||||
|
||||
firstSession.onclose({ code: 1011, reason: "temporary" });
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
lastConnectParams().callbacks.onmessage({
|
||||
serverContent: { inputTranscription: { text: "after", finished: true } },
|
||||
});
|
||||
|
||||
expect(lastConnectParams().config.sessionResumption).toEqual({ handle: "resume-2" });
|
||||
expect(onEvent).not.toHaveBeenCalled();
|
||||
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
|
||||
["user", "Before after", true],
|
||||
]);
|
||||
});
|
||||
|
||||
it("drops unfinished hypotheses when a new session has no continuity", async () => {
|
||||
vi.useFakeTimers();
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
@@ -854,28 +897,37 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("reconnects unexpected Google Live closes with the latest resumption handle", async () => {
|
||||
it.each([undefined, "invalidated-handle"])("invalidates resume handles %#", async (newHandle) => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const onClose = vi.fn();
|
||||
const onError = vi.fn();
|
||||
const onEvent = vi.fn();
|
||||
const onTranscript = vi.fn();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: { apiKey: "gemini-key" },
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
onClose,
|
||||
onError,
|
||||
onEvent,
|
||||
onTranscript,
|
||||
});
|
||||
|
||||
await bridge.connect();
|
||||
lastConnectParams().callbacks.onmessage({
|
||||
setupComplete: { sessionId: "session-1" },
|
||||
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
|
||||
serverContent: {
|
||||
inputTranscription: { text: "Previous caller " },
|
||||
outputTranscription: { text: "Previous assistant " },
|
||||
},
|
||||
});
|
||||
lastConnectParams().callbacks.onmessage({
|
||||
sessionResumptionUpdate: { resumable: false },
|
||||
sessionResumptionUpdate: { resumable: false, ...(newHandle ? { newHandle } : {}) },
|
||||
});
|
||||
expect(onEvent).not.toHaveBeenCalled();
|
||||
lastConnectParams().callbacks.onclose({
|
||||
code: 1011,
|
||||
reason: "temporary upstream close",
|
||||
@@ -883,13 +935,38 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
||||
});
|
||||
|
||||
expect(onClose).not.toHaveBeenCalled();
|
||||
expect(onEvent).toHaveBeenCalledOnce();
|
||||
expect(onEvent).toHaveBeenCalledWith({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
const error = requireFirstError(onError);
|
||||
expect(error.message).toContain("reconnecting 1/3");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
|
||||
expect(connectMock).toHaveBeenCalledTimes(2);
|
||||
expect(lastConnectParams().config.sessionResumption).toEqual({ handle: "resume-1" });
|
||||
expect(lastConnectParams().config.sessionResumption).toEqual({});
|
||||
expect(onEvent).toHaveBeenCalledOnce();
|
||||
const resetOrder = onEvent.mock.invocationCallOrder[0];
|
||||
const reconnectOrder = connectMock.mock.invocationCallOrder[1];
|
||||
if (resetOrder === undefined || reconnectOrder === undefined) {
|
||||
throw new Error("expected continuity reset before reconnect");
|
||||
}
|
||||
expect(resetOrder).toBeLessThan(reconnectOrder);
|
||||
|
||||
lastConnectParams().callbacks.onmessage({
|
||||
setupComplete: { sessionId: "session-2" },
|
||||
serverContent: {
|
||||
inputTranscription: { text: "Fresh caller", finished: true },
|
||||
outputTranscription: { text: "Fresh assistant", finished: true },
|
||||
},
|
||||
});
|
||||
|
||||
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
|
||||
["user", "Fresh caller", true],
|
||||
["assistant", "Fresh assistant", true],
|
||||
]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
@@ -995,6 +1072,103 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("emits one continuity reset across failed fresh reconnect attempts", async () => {
|
||||
vi.useFakeTimers();
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const onClose = vi.fn();
|
||||
const onEvent = vi.fn();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: { apiKey: "gemini-key", sessionResumption: false },
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
onClose,
|
||||
onEvent,
|
||||
});
|
||||
|
||||
await bridge.connect();
|
||||
const firstSession = lastConnectParams().callbacks;
|
||||
firstSession.onmessage({ setupComplete: {} });
|
||||
connectMock
|
||||
.mockRejectedValueOnce(new Error("connect failed 1"))
|
||||
.mockRejectedValueOnce(new Error("connect failed 2"))
|
||||
.mockRejectedValueOnce(new Error("connect failed 3"));
|
||||
firstSession.onclose({ code: 1011, reason: "temporary" });
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_750);
|
||||
|
||||
expect(connectMock).toHaveBeenCalledTimes(4);
|
||||
expect(onEvent.mock.calls).toEqual([
|
||||
[{ direction: "client", type: "session.continuity.reset" }],
|
||||
]);
|
||||
expect(onClose).toHaveBeenCalledWith("error");
|
||||
});
|
||||
|
||||
it("rearms continuity reset after pre-return setup selects a fresh session", async () => {
|
||||
vi.useFakeTimers();
|
||||
const pendingSession = createDeferred<MockGoogleLiveSession>();
|
||||
const freshSession = createMockGoogleLiveSession();
|
||||
connectMock
|
||||
.mockReturnValueOnce(Promise.resolve(session))
|
||||
.mockReturnValueOnce(pendingSession.promise);
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const onEvent = vi.fn();
|
||||
const onReady = vi.fn();
|
||||
const onTranscript = vi.fn();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: { apiKey: "gemini-key", sessionResumption: false },
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
onEvent,
|
||||
onReady,
|
||||
onTranscript,
|
||||
});
|
||||
|
||||
await bridge.connect();
|
||||
const firstCallbacks = lastConnectParams().callbacks;
|
||||
firstCallbacks.onopen();
|
||||
firstCallbacks.onmessage({ setupComplete: {} });
|
||||
firstCallbacks.onclose({ code: 1011, reason: "temporary" });
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
|
||||
const freshCallbacks = lastConnectParams().callbacks;
|
||||
expect(onEvent).toHaveBeenCalledTimes(1);
|
||||
freshCallbacks.onopen();
|
||||
freshCallbacks.onmessage({
|
||||
setupComplete: {},
|
||||
serverContent: { inputTranscription: { text: "Fresh partial " } },
|
||||
});
|
||||
expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([
|
||||
"session.continuity.reset",
|
||||
"session.created",
|
||||
]);
|
||||
expect(onReady).toHaveBeenCalledTimes(1);
|
||||
|
||||
pendingSession.resolve(freshSession);
|
||||
await vi.waitFor(() => {
|
||||
expect(onReady).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
const sessionCreatedOrder = onEvent.mock.invocationCallOrder[1];
|
||||
const freshReadyOrder = onReady.mock.invocationCallOrder[1];
|
||||
if (sessionCreatedOrder === undefined || freshReadyOrder === undefined) {
|
||||
throw new Error("expected fresh session creation before readiness");
|
||||
}
|
||||
expect(sessionCreatedOrder).toBeLessThan(freshReadyOrder);
|
||||
freshCallbacks.onclose({ code: 1011, reason: "temporary again" });
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
|
||||
expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([
|
||||
"session.continuity.reset",
|
||||
"session.created",
|
||||
"session.continuity.reset",
|
||||
]);
|
||||
lastConnectParams().callbacks.onmessage({
|
||||
serverContent: { inputTranscription: { text: "Next", finished: true } },
|
||||
});
|
||||
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
|
||||
["user", "Next", true],
|
||||
]);
|
||||
});
|
||||
|
||||
it("waits for the returned session after setup completion before activating", async () => {
|
||||
const pendingSession = createDeferred<MockGoogleLiveSession>();
|
||||
const connectedSession = createMockGoogleLiveSession();
|
||||
|
||||
@@ -482,6 +482,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
||||
private reconnectAttempts = 0;
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
private hasConnectedSession = false;
|
||||
private continuityResetEmitted = false;
|
||||
private terminalError: Error | undefined;
|
||||
private closeNotified = false;
|
||||
private connectionOwner: GoogleLiveConnectionAttempt | undefined;
|
||||
@@ -530,13 +531,6 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
||||
}
|
||||
|
||||
private async connectOwned(attempt: GoogleLiveConnectionAttempt): Promise<void> {
|
||||
const canResumeSession =
|
||||
this.config.sessionResumption !== false && Boolean(this.resumptionHandle);
|
||||
if (this.hasConnectedSession && !canResumeSession) {
|
||||
// An unfinished recognition hypothesis cannot cross into a fresh server session.
|
||||
// Dropping it avoids treating a transport break as a completed user utterance.
|
||||
this.resetPendingTranscripts();
|
||||
}
|
||||
this.intentionallyClosed = false;
|
||||
this.closeNotified = false;
|
||||
this.setupCompleteReceived = false;
|
||||
@@ -852,7 +846,9 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
||||
sessionResumptionUpdate?: { newHandle?: string; resumable?: boolean };
|
||||
};
|
||||
const update = raw.sessionResumptionUpdate;
|
||||
if (update?.resumable && update.newHandle) {
|
||||
if (update?.resumable === false) {
|
||||
this.resumptionHandle = undefined;
|
||||
} else if (update?.resumable && update.newHandle) {
|
||||
this.resumptionHandle = update.newHandle;
|
||||
}
|
||||
if (raw.goAway?.timeLeft) {
|
||||
@@ -861,6 +857,14 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
||||
}
|
||||
|
||||
private handleSetupComplete(): void {
|
||||
if (!this.setupCompleteReceived) {
|
||||
// setupComplete proves Google selected a new server session. A later
|
||||
// continuity loss therefore owns a new reset generation.
|
||||
if (this.continuityResetEmitted) {
|
||||
this.config.onEvent?.({ direction: "server", type: "session.created" });
|
||||
}
|
||||
this.continuityResetEmitted = false;
|
||||
}
|
||||
this.setupCompleteReceived = true;
|
||||
this.maybeActivateSession();
|
||||
}
|
||||
@@ -1046,6 +1050,18 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
||||
if (this.reconnectAttempts >= GOOGLE_REALTIME_RECONNECT_MAX_ATTEMPTS) {
|
||||
return false;
|
||||
}
|
||||
const canResumeSession =
|
||||
this.config.sessionResumption !== false && Boolean(this.resumptionHandle);
|
||||
if (this.hasConnectedSession && !canResumeSession && !this.continuityResetEmitted) {
|
||||
// A non-resumable close ends the provider generation immediately. Reset
|
||||
// consumers before backoff so stale work cannot finish into the replacement.
|
||||
this.continuityResetEmitted = true;
|
||||
this.resetPendingTranscripts();
|
||||
this.config.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
}
|
||||
const attempt = ++this.reconnectAttempts;
|
||||
const delayMs = Math.min(
|
||||
GOOGLE_REALTIME_RECONNECT_MAX_DELAY_MS,
|
||||
|
||||
@@ -254,6 +254,7 @@ async function withBargeInHarness(
|
||||
createBridge: ReturnType<typeof vi.fn>;
|
||||
handleBargeIn: ReturnType<typeof vi.fn>;
|
||||
outboundMessages: Array<Record<string, unknown>>;
|
||||
processEvent: ReturnType<typeof vi.fn>;
|
||||
sendAudio: ReturnType<typeof vi.fn>;
|
||||
ws: WebSocket;
|
||||
}) => Promise<void>,
|
||||
@@ -261,6 +262,7 @@ async function withBargeInHarness(
|
||||
let callbacks: RealtimeBridgeRequest | undefined;
|
||||
const sendAudio = vi.fn();
|
||||
const handleBargeIn = vi.fn();
|
||||
const processEvent = vi.fn();
|
||||
const call = makeCallRecord(params.providerCallId);
|
||||
const createBridge = vi.fn((request: RealtimeBridgeRequest) => {
|
||||
callbacks = request;
|
||||
@@ -278,6 +280,7 @@ async function withBargeInHarness(
|
||||
const handler = makeHandler(undefined, {
|
||||
manager: {
|
||||
getCallByProviderCallId: vi.fn((): CallRecord => call),
|
||||
processEvent,
|
||||
},
|
||||
providerConfig: {
|
||||
apiKey: "test-key",
|
||||
@@ -313,6 +316,7 @@ async function withBargeInHarness(
|
||||
createBridge,
|
||||
handleBargeIn,
|
||||
outboundMessages,
|
||||
processEvent,
|
||||
sendAudio,
|
||||
ws,
|
||||
});
|
||||
@@ -948,6 +952,63 @@ describe("RealtimeCallHandler path routing", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("starts fresh transcript and Talk state after provider continuity resets", async () => {
|
||||
await withBargeInHarness(
|
||||
{ providerCallId: "CA-continuity-reset" },
|
||||
async ({ callbacks, call, outboundMessages, processEvent }) => {
|
||||
callbacks.onTranscript?.("user", "Old caller ", false);
|
||||
callbacks.onTranscript?.("assistant", "Old assistant ", false);
|
||||
callbacks.onAudio?.(Buffer.alloc(320, 0xff));
|
||||
const oldTurnId = recentTalkEvents(call).findLast(
|
||||
(event) => event.type === "turn.started",
|
||||
)?.turnId;
|
||||
expect(oldTurnId).toBeTruthy();
|
||||
|
||||
callbacks.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
callbacks.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
|
||||
await waitForRealtimeTest(() => {
|
||||
expect(outboundMessages.some((message) => message.event === "clear")).toBe(true);
|
||||
});
|
||||
expect(requireCancelledTurn(call).turnId).toBe(oldTurnId);
|
||||
const resetEvents = recentTalkEvents(call);
|
||||
expect(resetEvents.filter((event) => event.type === "turn.cancelled")).toHaveLength(1);
|
||||
expect(resetEvents.findIndex((event) => event.type === "output.audio.done")).toBeLessThan(
|
||||
resetEvents.findIndex((event) => event.type === "turn.cancelled"),
|
||||
);
|
||||
|
||||
callbacks.onTranscript?.("user", "Fresh caller", true);
|
||||
callbacks.onTranscript?.("assistant", "Fresh assistant", true);
|
||||
callbacks.onEvent?.({ direction: "server", type: "response.done" });
|
||||
|
||||
const processedEvents = processEvent.mock.calls.map(([event]) => event as NormalizedEvent);
|
||||
expect(
|
||||
processedEvents
|
||||
.filter((event) => event.type === "call.speech")
|
||||
.map((event) => (event.type === "call.speech" ? event.transcript : undefined)),
|
||||
).toEqual(["Fresh caller"]);
|
||||
expect(
|
||||
processedEvents
|
||||
.filter((event) => event.type === "call.assistant-speech")
|
||||
.map((event) =>
|
||||
event.type === "call.assistant-speech" ? event.transcript : undefined,
|
||||
),
|
||||
).toEqual(["Fresh assistant"]);
|
||||
const startedTurns = recentTalkEvents(call).filter(
|
||||
(event) => event.type === "turn.started",
|
||||
);
|
||||
expect(startedTurns).toHaveLength(2);
|
||||
expect(startedTurns[1]?.turnId).not.toBe(oldTurnId);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("passes the disabled input-interruption policy without cancelling speech-start", async () => {
|
||||
await withBargeInHarness(
|
||||
{
|
||||
@@ -1492,6 +1553,87 @@ describe("RealtimeCallHandler path routing", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("clears cancelled consult dedupe for a fresh provider session", async () => {
|
||||
let callbacks: RealtimeBridgeRequest | undefined;
|
||||
let sessionHarness: RealtimeVoiceSessionHarness | undefined;
|
||||
realtimeVoiceHarnessTestHooks.onCreate = (harness) => {
|
||||
sessionHarness = harness;
|
||||
};
|
||||
const submitToolResult = vi.fn();
|
||||
const createBridge = vi.fn((request: RealtimeBridgeRequest) => {
|
||||
callbacks = request;
|
||||
return makeBridge({ submitToolResult });
|
||||
});
|
||||
const handler = makeHandler(undefined, {
|
||||
manager: {
|
||||
getCallByProviderCallId: vi.fn((providerCallId: string) => makeCallRecord(providerCallId)),
|
||||
},
|
||||
realtimeProvider: makeRealtimeProvider(createBridge),
|
||||
});
|
||||
const consult = vi.fn(async () => ({ text: "fresh consult answer" }));
|
||||
handler.registerToolHandler("openclaw_agent_consult", consult);
|
||||
const server = await startRealtimeServer(handler);
|
||||
|
||||
try {
|
||||
const ws = await connectWs(server.url);
|
||||
try {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
event: "start",
|
||||
start: { streamSid: "MZ-continuity-consult", callSid: "CA-continuity-consult" },
|
||||
}),
|
||||
);
|
||||
await waitForRealtimeTest(() => {
|
||||
expect(callbacks).toBeDefined();
|
||||
expect(sessionHarness).toBeDefined();
|
||||
});
|
||||
|
||||
const coordinator = expectDefined(
|
||||
sessionHarness,
|
||||
"voice-call realtime session harness",
|
||||
).forcedConsults;
|
||||
const cancelled = expectDefined(
|
||||
coordinator.prepare("same question"),
|
||||
"cancelled forced consult",
|
||||
);
|
||||
coordinator.markStarted(cancelled);
|
||||
coordinator.markCancelled(cancelled);
|
||||
|
||||
callbacks?.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
callbacks?.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
expect(coordinator.handles()).toEqual([]);
|
||||
|
||||
callbacks?.onToolCall?.({
|
||||
itemId: "item-fresh",
|
||||
callId: "native-fresh",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "same question" },
|
||||
});
|
||||
|
||||
await waitForRealtimeTest(() => {
|
||||
expect(consult).toHaveBeenCalledTimes(1);
|
||||
expect(submitToolResult).toHaveBeenCalledWith(
|
||||
"native-fresh",
|
||||
{ text: "fresh consult answer" },
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
if (ws.readyState !== WebSocket.CLOSED && ws.readyState !== WebSocket.CLOSING) {
|
||||
ws.close();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not deliver a forced consult after its realtime session closes", async () => {
|
||||
let callbacks:
|
||||
| {
|
||||
@@ -1727,6 +1869,125 @@ describe("RealtimeCallHandler path routing", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores late continuity reset and close from a replaced bridge", async () => {
|
||||
const callbacks: RealtimeBridgeRequest[] = [];
|
||||
const oldCloseBridge = vi.fn();
|
||||
const replacementCloseBridge = vi.fn();
|
||||
const bridges = [
|
||||
makeBridge({ close: oldCloseBridge }),
|
||||
makeBridge({ close: replacementCloseBridge }),
|
||||
];
|
||||
const createBridge = vi.fn((request: RealtimeBridgeRequest) => {
|
||||
callbacks.push(request);
|
||||
const bridge = bridges[callbacks.length - 1];
|
||||
if (!bridge) {
|
||||
throw new Error("unexpected replacement bridge");
|
||||
}
|
||||
return bridge;
|
||||
});
|
||||
const processEvent = vi.fn();
|
||||
const hangupCall = vi.fn(async () => {});
|
||||
const sharedCallSid = "CA-continuity-shared";
|
||||
const call = makeCallRecord(sharedCallSid);
|
||||
const handler = makeHandler(undefined, {
|
||||
manager: {
|
||||
getCallByProviderCallId: vi.fn(() => call),
|
||||
processEvent,
|
||||
},
|
||||
provider: { hangupCall },
|
||||
realtimeProvider: makeRealtimeProvider(createBridge),
|
||||
});
|
||||
const oldServer = await startRealtimeServer(handler);
|
||||
let replacementServer: Awaited<ReturnType<typeof startRealtimeServer>> | undefined;
|
||||
let oldWs: WebSocket | undefined;
|
||||
|
||||
try {
|
||||
oldWs = await connectWs(oldServer.url);
|
||||
oldWs.send(
|
||||
JSON.stringify({
|
||||
event: "start",
|
||||
start: { streamSid: "MZ-continuity-old", callSid: sharedCallSid },
|
||||
}),
|
||||
);
|
||||
await waitForRealtimeTest(() => {
|
||||
expect(callbacks).toHaveLength(1);
|
||||
});
|
||||
|
||||
replacementServer = await startRealtimeServer(handler);
|
||||
const replacementWs = await connectWs(replacementServer.url);
|
||||
try {
|
||||
replacementWs.send(
|
||||
JSON.stringify({
|
||||
event: "start",
|
||||
start: {
|
||||
streamSid: "MZ-continuity-replacement",
|
||||
callSid: sharedCallSid,
|
||||
},
|
||||
}),
|
||||
);
|
||||
await waitForRealtimeTest(() => {
|
||||
expect(callbacks).toHaveLength(2);
|
||||
});
|
||||
|
||||
callbacks[1]?.onTranscript?.("user", "Fresh ", false);
|
||||
callbacks[0]?.onTranscript?.("user", "stale partial", false);
|
||||
callbacks[0]?.onTranscript?.("user", "stale final", true);
|
||||
callbacks[0]?.onTranscript?.("assistant", "stale assistant", true);
|
||||
callbacks[0]?.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
callbacks[0]?.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
const oldClosed = waitForClose(oldWs);
|
||||
callbacks[0]?.onClose?.("error");
|
||||
await oldClosed;
|
||||
expect(oldCloseBridge).toHaveBeenCalledOnce();
|
||||
expect(replacementCloseBridge).not.toHaveBeenCalled();
|
||||
expect(hangupCall).not.toHaveBeenCalled();
|
||||
expect(
|
||||
processEvent.mock.calls
|
||||
.map(([event]) => event as NormalizedEvent)
|
||||
.filter((event) => event.type === "call.ended"),
|
||||
).toHaveLength(0);
|
||||
callbacks[1]?.onTranscript?.("user", "caller", true);
|
||||
|
||||
await waitForRealtimeTest(() => {
|
||||
expect(
|
||||
processEvent.mock.calls
|
||||
.map(([event]) => event as NormalizedEvent)
|
||||
.filter((event) => event.type === "call.speech")
|
||||
.map((event) => (event.type === "call.speech" ? event.transcript : undefined)),
|
||||
).toEqual(["Fresh caller"]);
|
||||
});
|
||||
expect(
|
||||
processEvent.mock.calls
|
||||
.map(([event]) => event as NormalizedEvent)
|
||||
.filter((event) => event.type === "call.assistant-speech"),
|
||||
).toHaveLength(0);
|
||||
} finally {
|
||||
if (
|
||||
replacementWs.readyState !== WebSocket.CLOSED &&
|
||||
replacementWs.readyState !== WebSocket.CLOSING
|
||||
) {
|
||||
replacementWs.close();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (
|
||||
oldWs &&
|
||||
oldWs.readyState !== WebSocket.CLOSED &&
|
||||
oldWs.readyState !== WebSocket.CLOSING
|
||||
) {
|
||||
oldWs.close();
|
||||
}
|
||||
await replacementServer?.close();
|
||||
await oldServer.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not share a native consult with a replacement realtime session", async () => {
|
||||
const callbacks: RealtimeBridgeRequest[] = [];
|
||||
const oldSubmitToolResult = vi.fn();
|
||||
|
||||
@@ -339,7 +339,10 @@ export class RealtimeCallHandler {
|
||||
private readonly activeBridgesByCallId = new Map<string, ActiveRealtimeVoiceBridge>();
|
||||
private readonly activeTelephonyClosersByCallId = new Map<
|
||||
string,
|
||||
(reason: TelephonyCloseReason) => void
|
||||
{
|
||||
owner: ActiveRealtimeVoiceBridge;
|
||||
close: (reason: TelephonyCloseReason) => void;
|
||||
}
|
||||
>();
|
||||
private readonly partialUserTranscriptsByCallId = new Map<string, string>();
|
||||
private readonly rawPartialUserTranscriptsByCallId = new Map<string, string>();
|
||||
@@ -797,6 +800,10 @@ export class RealtimeCallHandler {
|
||||
},
|
||||
},
|
||||
onTranscript: (role, text, isFinal) => {
|
||||
const owner = nativeConsultOwner.current;
|
||||
if (owner && !this.isActiveBridgeOwner(callId, owner)) {
|
||||
return;
|
||||
}
|
||||
const turnId = harness.ensureTurn();
|
||||
const eventType =
|
||||
role === "assistant"
|
||||
@@ -898,6 +905,27 @@ export class RealtimeCallHandler {
|
||||
);
|
||||
},
|
||||
onEvent: (event) => {
|
||||
if (event.direction === "client" && event.type === "session.continuity.reset") {
|
||||
// A fresh provider session cannot complete the prior session's text,
|
||||
// audio, tool work, or Talk turn.
|
||||
const turnId = harness.talk.activeTurnId;
|
||||
const owner = nativeConsultOwner.current;
|
||||
if (owner && this.isActiveBridgeOwner(callId, owner)) {
|
||||
this.clearUserTranscriptState(callId);
|
||||
this.resetConsultSessionForContinuity(callId, owner);
|
||||
}
|
||||
harness.flushOutput(() => {
|
||||
audioPacer.clearAudio();
|
||||
harness.finishOutputAudio(event.type);
|
||||
});
|
||||
if (turnId) {
|
||||
harness.talk.cancelTurn({
|
||||
turnId,
|
||||
payload: { callId, providerCallId: callSid, reason: event.type },
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (event.type === "input_audio_buffer.speech_started") {
|
||||
harness.ensureTurn();
|
||||
return;
|
||||
@@ -943,11 +971,15 @@ export class RealtimeCallHandler {
|
||||
});
|
||||
},
|
||||
onClose: (reason) => {
|
||||
if (nativeConsultOwner.current) {
|
||||
this.clearActiveBridgeMappings(callId, callSid, nativeConsultOwner.current);
|
||||
this.cancelConsultSession(callId, nativeConsultOwner.current);
|
||||
const owner = nativeConsultOwner.current;
|
||||
const ownsCallState = owner ? this.isActiveBridgeOwner(callId, owner) : false;
|
||||
if (owner) {
|
||||
this.clearActiveBridgeMappings(callId, callSid, owner);
|
||||
this.cancelConsultSession(callId, owner);
|
||||
}
|
||||
if (ownsCallState) {
|
||||
this.clearUserTranscriptState(callId);
|
||||
}
|
||||
this.clearUserTranscriptState(callId);
|
||||
harness.finishOutputAudio(reason);
|
||||
harness.emit({
|
||||
type: "session.closed",
|
||||
@@ -957,10 +989,13 @@ export class RealtimeCallHandler {
|
||||
if (reason !== "error") {
|
||||
return;
|
||||
}
|
||||
emitCallEnd("error");
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.close(1011, "Bridge disconnected");
|
||||
}
|
||||
if (owner && !ownsCallState) {
|
||||
return;
|
||||
}
|
||||
emitCallEnd("error");
|
||||
void this.provider
|
||||
.hangupCall({ callId, providerCallId: callSid, reason: "error" })
|
||||
.catch((error: unknown) => {
|
||||
@@ -992,8 +1027,9 @@ export class RealtimeCallHandler {
|
||||
});
|
||||
this.activeBridgesByCallId.set(callId, session);
|
||||
this.activeBridgesByCallId.set(callSid, session);
|
||||
this.activeTelephonyClosersByCallId.set(callId, closeTelephony);
|
||||
this.activeTelephonyClosersByCallId.set(callSid, closeTelephony);
|
||||
const telephonyCloser = { owner: session, close: closeTelephony };
|
||||
this.activeTelephonyClosersByCallId.set(callId, telephonyCloser);
|
||||
this.activeTelephonyClosersByCallId.set(callSid, telephonyCloser);
|
||||
const sendAudioToSession = session.sendAudio.bind(session);
|
||||
session.sendAudio = (audio) => {
|
||||
if (speechDetector.accept({ rms: calculateMulawRms(audio), peak: 0 })) {
|
||||
@@ -1021,9 +1057,12 @@ export class RealtimeCallHandler {
|
||||
try {
|
||||
closeSession();
|
||||
} finally {
|
||||
const ownsCallState = this.isActiveBridgeOwner(callId, session);
|
||||
this.clearActiveBridgeMappings(callId, callSid, session);
|
||||
this.cancelConsultSession(callId, session);
|
||||
this.clearUserTranscriptState(callId);
|
||||
if (ownsCallState) {
|
||||
this.clearUserTranscriptState(callId);
|
||||
}
|
||||
harness.close();
|
||||
audioPacer.close();
|
||||
}
|
||||
@@ -1031,8 +1070,11 @@ export class RealtimeCallHandler {
|
||||
|
||||
session.connect().catch((error: unknown) => {
|
||||
console.error("[voice-call] Failed to connect realtime bridge:", error);
|
||||
const ownsCallState = this.isActiveBridgeOwner(callId, session);
|
||||
session.close();
|
||||
emitCallEnd("error");
|
||||
if (ownsCallState) {
|
||||
emitCallEnd("error");
|
||||
}
|
||||
ws.close(1011, "Failed to connect");
|
||||
});
|
||||
|
||||
@@ -1105,22 +1147,43 @@ export class RealtimeCallHandler {
|
||||
this.forcedConsultsByCallId.delete(callId);
|
||||
}
|
||||
|
||||
private cancelConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge | undefined): void {
|
||||
if (!owner) {
|
||||
return;
|
||||
}
|
||||
private resetConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge): boolean {
|
||||
const session = this.consultSessionsByCallId.get(callId);
|
||||
if (!session || session.owner !== owner) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
// Forced and native consults share bridge ownership. Replacement or close
|
||||
// must invalidate both before a newer bridge can observe call-scoped state.
|
||||
session.coordinator.clearPending();
|
||||
this.cancelForcedConsult(callId, owner);
|
||||
this.cancelNativeConsult(callId, owner);
|
||||
return true;
|
||||
}
|
||||
|
||||
private resetConsultSessionForContinuity(
|
||||
callId: string,
|
||||
owner: ActiveRealtimeVoiceBridge,
|
||||
): boolean {
|
||||
const session = this.consultSessionsByCallId.get(callId);
|
||||
if (!session || session.owner !== owner) {
|
||||
return false;
|
||||
}
|
||||
this.cancelForcedConsult(callId, owner);
|
||||
this.cancelNativeConsult(callId, owner);
|
||||
// A fresh provider session must not inherit cancelled/recent consult dedupe.
|
||||
session.coordinator.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
private cancelConsultSession(callId: string, owner: ActiveRealtimeVoiceBridge | undefined): void {
|
||||
if (!owner || !this.resetConsultSession(callId, owner)) {
|
||||
return;
|
||||
}
|
||||
this.consultSessionsByCallId.delete(callId);
|
||||
}
|
||||
|
||||
private isActiveBridgeOwner(callId: string, owner: ActiveRealtimeVoiceBridge): boolean {
|
||||
return this.activeBridgesByCallId.get(callId) === owner;
|
||||
}
|
||||
|
||||
private clearActiveBridgeMappings(
|
||||
callId: string,
|
||||
callSid: string,
|
||||
@@ -1197,8 +1260,8 @@ export class RealtimeCallHandler {
|
||||
reason: TelephonyCloseReason,
|
||||
): void {
|
||||
const closer = this.activeTelephonyClosersByCallId.get(callIdOrSid);
|
||||
if (closer) {
|
||||
closer(reason);
|
||||
if (closer && closer.owner === bridge) {
|
||||
closer.close(reason);
|
||||
return;
|
||||
}
|
||||
bridge?.close();
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
ensureRelayTurn,
|
||||
noFallbackRelayOutputFlush,
|
||||
relaySessions,
|
||||
resolveRelayProviderToolCallId,
|
||||
type ForcedTerminalProviderResult,
|
||||
type RelayAgentControlProviderSubmission,
|
||||
type RelaySession,
|
||||
@@ -275,7 +276,7 @@ export function submitRealtimeAgentConsultWorkingResponse(
|
||||
}
|
||||
const epoch = session.toolResultEpoch;
|
||||
const submission = session.bridge.submitToolResult(
|
||||
callId,
|
||||
resolveRelayProviderToolCallId(session, callId),
|
||||
buildRealtimeVoiceAgentConsultWorkingResponse("person"),
|
||||
{ willContinue: true },
|
||||
);
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
} from "../talk/agent-run-control.js";
|
||||
import { registerClientVoiceConsultRun } from "../talk/client-voice-session.js";
|
||||
import type { RealtimeVoiceToolResultOptions } from "../talk/provider-types.js";
|
||||
import type { TalkEvent } from "../talk/talk-session-controller.js";
|
||||
import { abortChatRunById } from "./chat-abort.js";
|
||||
import { formatError } from "./server-utils.js";
|
||||
import {
|
||||
@@ -30,6 +31,7 @@ import {
|
||||
ensureRelayTurn,
|
||||
noFallbackRelayOutputFlush,
|
||||
relaySessions,
|
||||
resolveRelayProviderToolCallId,
|
||||
type RelaySession,
|
||||
} from "./talk-realtime-relay-state.js";
|
||||
import {
|
||||
@@ -298,7 +300,11 @@ export function submitTalkRealtimeRelayToolResult(params: {
|
||||
return trackAgentFinalToolResult(session, params.callId, completion);
|
||||
}
|
||||
const submit = () =>
|
||||
session.bridge.submitToolResult(params.callId, params.result, params.options);
|
||||
session.bridge.submitToolResult(
|
||||
resolveRelayProviderToolCallId(session, params.callId),
|
||||
params.result,
|
||||
params.options,
|
||||
);
|
||||
const pendingWorking = session.pendingWorkingToolResults.get(params.callId);
|
||||
if (pendingWorking) {
|
||||
const submission = pendingWorking.then(async () => {
|
||||
@@ -451,6 +457,54 @@ export function cancelTalkRealtimeRelayTurn(params: {
|
||||
});
|
||||
}
|
||||
|
||||
/** Drops one provider generation without sending cancellation into its replacement. */
|
||||
export function resetTalkRealtimeRelayContinuity(
|
||||
session: RelaySession,
|
||||
reason = "session.continuity.reset",
|
||||
): TalkEvent | undefined {
|
||||
session.toolResultEpoch += 1;
|
||||
const retiredCallIds = new Set<string>([
|
||||
...session.activeAgentToolCalls.keys(),
|
||||
...session.cancelledAgentToolCalls.keys(),
|
||||
...session.providerToolCallIds.keys(),
|
||||
...session.providerToolCallIds.values(),
|
||||
...session.pendingFinalToolResults.keys(),
|
||||
...session.pendingProviderToolResults.keys(),
|
||||
...session.pendingWorkingToolResults.keys(),
|
||||
...session.forcedTerminalProviderResults.keys(),
|
||||
]);
|
||||
for (const handle of session.harness.forcedConsults.handles()) {
|
||||
retiredCallIds.add(handle.id);
|
||||
for (const nativeCallId of session.harness.forcedConsults.nativeCallIds(handle)) {
|
||||
retiredCallIds.add(nativeCallId);
|
||||
}
|
||||
}
|
||||
for (const callId of retiredCallIds) {
|
||||
session.completedAgentToolCalls.add(callId);
|
||||
}
|
||||
session.cancelledAgentToolCalls.clear();
|
||||
session.providerToolCallIds.clear();
|
||||
session.relayToolCallIdsByProviderId.clear();
|
||||
session.pendingFinalToolResults.clear();
|
||||
session.completedProviderToolResults.clear();
|
||||
session.pendingProviderToolResults.clear();
|
||||
session.pendingWorkingToolResults.clear();
|
||||
session.forcedTerminalProviderResults.clear();
|
||||
session.harness.forcedConsults.clear();
|
||||
abortRelayAgentRuns(session, reason);
|
||||
const turnId = session.harness.talk.activeTurnId;
|
||||
session.harness.flushOutput(noFallbackRelayOutputFlush);
|
||||
session.harness.finishOutputAudio(reason);
|
||||
if (!turnId) {
|
||||
return undefined;
|
||||
}
|
||||
const cancelled = session.harness.talk.cancelTurn({
|
||||
turnId,
|
||||
payload: { reason },
|
||||
});
|
||||
return cancelled.ok ? cancelled.event : undefined;
|
||||
}
|
||||
|
||||
/** Closes a realtime relay session owned by the current connection. */
|
||||
export function stopTalkRealtimeRelaySession(params: {
|
||||
relaySessionId: string;
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { buildRealtimeVoiceAgentCancelProviderResult } from "../talk/agent-run-control-shared.js";
|
||||
import type { RealtimeVoiceToolResultOptions } from "../talk/provider-types.js";
|
||||
import { broadcastToOwner, relaySessions, type RelaySession } from "./talk-realtime-relay-state.js";
|
||||
import {
|
||||
broadcastToOwner,
|
||||
relaySessions,
|
||||
resolveRelayProviderToolCallId,
|
||||
type RelaySession,
|
||||
} from "./talk-realtime-relay-state.js";
|
||||
|
||||
export function suppressedToolResultOptions(
|
||||
session: RelaySession,
|
||||
@@ -63,8 +68,13 @@ export function submitFinalProviderToolResult(params: {
|
||||
options?: RealtimeVoiceToolResultOptions;
|
||||
onAccepted?: () => void;
|
||||
}): void | Promise<void> {
|
||||
if (params.session.completedProviderToolResults.has(params.callId)) {
|
||||
if (relaySessions.get(params.session.id) === params.session) {
|
||||
const epoch = params.session.toolResultEpoch;
|
||||
const providerCallId = resolveRelayProviderToolCallId(params.session, params.callId);
|
||||
if (params.session.completedProviderToolResults.has(providerCallId)) {
|
||||
if (
|
||||
relaySessions.get(params.session.id) === params.session &&
|
||||
params.session.toolResultEpoch === epoch
|
||||
) {
|
||||
params.onAccepted?.();
|
||||
}
|
||||
return;
|
||||
@@ -74,9 +84,8 @@ export function submitFinalProviderToolResult(params: {
|
||||
return pending;
|
||||
}
|
||||
const submit = () =>
|
||||
params.session.bridge.submitToolResult(params.callId, params.result, params.options);
|
||||
params.session.bridge.submitToolResult(providerCallId, params.result, params.options);
|
||||
const working = params.session.pendingWorkingToolResults.get(params.callId);
|
||||
const epoch = params.session.toolResultEpoch;
|
||||
const submitAfterWorking = async () => {
|
||||
if (relaySessions.get(params.session.id) !== params.session) {
|
||||
return false;
|
||||
@@ -89,13 +98,13 @@ export function submitFinalProviderToolResult(params: {
|
||||
// the provider's working-result acknowledgement. Finish the cancelled call
|
||||
// here so the provider is not left waiting for a terminal result.
|
||||
await params.session.bridge.submitToolResult(
|
||||
params.callId,
|
||||
providerCallId,
|
||||
buildRealtimeVoiceAgentCancelProviderResult(
|
||||
"OpenClaw cancelled this consult before completion. Do not restart it.",
|
||||
),
|
||||
suppressedToolResultOptions(params.session),
|
||||
);
|
||||
params.session.completedProviderToolResults.add(params.callId);
|
||||
params.session.completedProviderToolResults.add(providerCallId);
|
||||
params.session.cancelledAgentToolCalls.delete(params.callId);
|
||||
params.session.completedAgentToolCalls.add(params.callId);
|
||||
return false;
|
||||
@@ -105,7 +114,10 @@ export function submitFinalProviderToolResult(params: {
|
||||
};
|
||||
const submission = working ? working.then(submitAfterWorking, submitAfterWorking) : submit();
|
||||
const accept = () => {
|
||||
params.session.completedProviderToolResults.add(params.callId);
|
||||
if (params.session.toolResultEpoch !== epoch) {
|
||||
return;
|
||||
}
|
||||
params.session.completedProviderToolResults.add(providerCallId);
|
||||
if (relaySessions.get(params.session.id) === params.session) {
|
||||
params.onAccepted?.();
|
||||
}
|
||||
|
||||
@@ -35,12 +35,14 @@ import {
|
||||
enforceRelaySessionLimits,
|
||||
pruneInactiveRelayAgentRuns,
|
||||
registerTalkRealtimeRelayAgentRun,
|
||||
resetTalkRealtimeRelayContinuity,
|
||||
steerTalkRealtimeRelayAgentRun,
|
||||
} from "./talk-realtime-relay-operations.js";
|
||||
import { suppressedToolResultOptions } from "./talk-realtime-relay-provider-results.js";
|
||||
import {
|
||||
RELAY_SESSION_TTL_MS,
|
||||
RELAY_TRANSCRIPT_ECHO_LOOKBACK_MS,
|
||||
adoptRelayProviderToolCallId,
|
||||
broadcastToOwner,
|
||||
ensureRelayTurn,
|
||||
relayEventDeliveryOptions,
|
||||
@@ -105,6 +107,7 @@ export function createTalkRealtimeRelaySession(
|
||||
let currentOutputItemId: string | undefined;
|
||||
let currentOutputResponseId: string | undefined;
|
||||
let ready = false;
|
||||
let continuityResetActive = false;
|
||||
let failureEmitted = false;
|
||||
let transcriptPersistenceFailed = false;
|
||||
const constructionTerminal: {
|
||||
@@ -253,12 +256,34 @@ export function createTalkRealtimeRelaySession(
|
||||
},
|
||||
},
|
||||
onEvent: (event) => {
|
||||
if (!getActiveRelay()) {
|
||||
const relay = getActiveRelay();
|
||||
if (!relay) {
|
||||
return;
|
||||
}
|
||||
if (event.direction === "client" && event.type === "session.continuity.reset") {
|
||||
if (continuityResetActive) {
|
||||
return;
|
||||
}
|
||||
continuityResetActive = true;
|
||||
ready = false;
|
||||
currentOutputItemId = undefined;
|
||||
currentOutputResponseId = undefined;
|
||||
const talkEvent = resetTalkRealtimeRelayContinuity(relay, event.type);
|
||||
const clearEvent = { relaySessionId, type: "clear" as const };
|
||||
broadcastToOwner(
|
||||
params.context,
|
||||
params.connId,
|
||||
{ ...clearEvent, ...(talkEvent ? { talkEvent } : {}) },
|
||||
relayEventDeliveryOptions(clearEvent),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (event.direction !== "server") {
|
||||
return;
|
||||
}
|
||||
if (event.type === "session.created") {
|
||||
continuityResetActive = false;
|
||||
}
|
||||
if (
|
||||
event.type === "conversation.output_audio.delta" ||
|
||||
event.type === "response.audio.delta" ||
|
||||
@@ -365,11 +390,13 @@ export function createTalkRealtimeRelaySession(
|
||||
if (!relay) {
|
||||
return;
|
||||
}
|
||||
const providerCallId = toolCall.callId;
|
||||
const relayCallId = adoptRelayProviderToolCallId(relay, providerCallId);
|
||||
let shouldSubmitWorkingResult = false;
|
||||
if (toolCall.name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
|
||||
const forcedConsult = relay.harness.forcedConsults.recordNativeConsult(
|
||||
toolCall.args,
|
||||
toolCall.callId,
|
||||
providerCallId,
|
||||
);
|
||||
if (forcedConsult.kind === "in_flight" || forcedConsult.kind === "already_delivered") {
|
||||
if (forcedConsult.kind === "already_delivered") {
|
||||
@@ -380,7 +407,7 @@ export function createTalkRealtimeRelaySession(
|
||||
: buildAlreadyDeliveredToolResult();
|
||||
return submitForcedConsultProviderResult(
|
||||
relay,
|
||||
toolCall.callId,
|
||||
providerCallId,
|
||||
result,
|
||||
suppressedToolResultOptions(relay),
|
||||
);
|
||||
@@ -388,7 +415,7 @@ export function createTalkRealtimeRelaySession(
|
||||
if (relay.forcedTerminalProviderResults.has(forcedConsult.handle.id)) {
|
||||
return relay.pendingFinalToolResults.get(forcedConsult.handle.id);
|
||||
}
|
||||
return submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId);
|
||||
return submitRealtimeAgentConsultWorkingResponse(relay, relayCallId);
|
||||
}
|
||||
shouldSubmitWorkingResult = true;
|
||||
}
|
||||
@@ -398,20 +425,20 @@ export function createTalkRealtimeRelaySession(
|
||||
relaySessionId,
|
||||
type: "toolCall",
|
||||
itemId: toolCall.itemId,
|
||||
callId: toolCall.callId,
|
||||
callId: relayCallId,
|
||||
name: toolCall.name,
|
||||
args: toolCall.args,
|
||||
},
|
||||
{
|
||||
type: "tool.call",
|
||||
itemId: toolCall.itemId,
|
||||
callId: toolCall.callId,
|
||||
callId: relayCallId,
|
||||
turnId,
|
||||
payload: { name: toolCall.name, args: toolCall.args },
|
||||
},
|
||||
);
|
||||
if (shouldSubmitWorkingResult) {
|
||||
return submitRealtimeAgentConsultWorkingResponse(relay, toolCall.callId, turnId);
|
||||
return submitRealtimeAgentConsultWorkingResponse(relay, relayCallId, turnId);
|
||||
}
|
||||
},
|
||||
onReady: () => {
|
||||
@@ -419,6 +446,7 @@ export function createTalkRealtimeRelaySession(
|
||||
return;
|
||||
}
|
||||
ready = true;
|
||||
continuityResetActive = false;
|
||||
emit({ relaySessionId, type: "ready" }, { type: "session.ready", payload: null });
|
||||
},
|
||||
onError: (error) => {
|
||||
@@ -536,6 +564,8 @@ export function createTalkRealtimeRelaySession(
|
||||
provider: params.provider.id,
|
||||
activeAgentToolCalls: new Map(),
|
||||
completedAgentToolCalls: new Set(),
|
||||
providerToolCallIds: new Map(),
|
||||
relayToolCallIdsByProviderId: new Map(),
|
||||
cancelledAgentToolCalls: new Map(),
|
||||
pendingFinalToolResults: new Map(),
|
||||
completedProviderToolResults: new Set(),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js";
|
||||
import type { BoundedSerialQueue } from "../shared/bounded-serial-queue.js";
|
||||
@@ -94,6 +95,8 @@ export type RelaySession = {
|
||||
provider: string;
|
||||
activeAgentToolCalls: Map<string, string>;
|
||||
completedAgentToolCalls: Set<string>;
|
||||
providerToolCallIds: Map<string, string>;
|
||||
relayToolCallIdsByProviderId: Map<string, string>;
|
||||
// Cancelled calls retain their original turn long enough to terminally satisfy
|
||||
// late browser results without creating a replacement turn or owner success event.
|
||||
cancelledAgentToolCalls: Map<string, string>;
|
||||
@@ -150,6 +153,28 @@ export const relaySessions = new Map<string, RelaySession>();
|
||||
// until durable close settles. Session limits count both maps.
|
||||
export const drainingRelaySessions = new Set<RelaySession>();
|
||||
|
||||
export function adoptRelayProviderToolCallId(
|
||||
session: RelaySession,
|
||||
providerCallId: string,
|
||||
): string {
|
||||
const current = session.relayToolCallIdsByProviderId.get(providerCallId);
|
||||
if (current) {
|
||||
return current;
|
||||
}
|
||||
const relayCallId = session.completedAgentToolCalls.has(providerCallId)
|
||||
? `relay-${randomUUID()}`
|
||||
: providerCallId;
|
||||
session.completedProviderToolResults.delete(providerCallId);
|
||||
session.completedAgentToolCalls.delete(relayCallId);
|
||||
session.providerToolCallIds.set(relayCallId, providerCallId);
|
||||
session.relayToolCallIdsByProviderId.set(providerCallId, relayCallId);
|
||||
return relayCallId;
|
||||
}
|
||||
|
||||
export function resolveRelayProviderToolCallId(session: RelaySession, relayCallId: string): string {
|
||||
return session.providerToolCallIds.get(relayCallId) ?? relayCallId;
|
||||
}
|
||||
|
||||
export function broadcastToOwner(
|
||||
context: GatewayRequestContext,
|
||||
connId: string,
|
||||
|
||||
@@ -989,6 +989,165 @@ describe("talk realtime gateway relay", () => {
|
||||
expectRecordFields(mockCallArg(mock, 0, 2), { runId: "run-1", state: "aborted" });
|
||||
}
|
||||
|
||||
it("resets provider continuity without cancelling the replacement provider", async () => {
|
||||
let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined;
|
||||
const pendingWorking = createDeferredVoid();
|
||||
const handleBargeIn = vi.fn();
|
||||
const submitToolResult = vi
|
||||
.fn<RealtimeVoiceBridge["submitToolResult"]>()
|
||||
.mockReturnValueOnce(pendingWorking.promise);
|
||||
const bridge = {
|
||||
supportsToolResultContinuation: true,
|
||||
connect: vi.fn(async () => undefined),
|
||||
sendAudio: vi.fn(),
|
||||
setMediaTimestamp: vi.fn(),
|
||||
sendUserMessage: vi.fn(),
|
||||
handleBargeIn,
|
||||
submitToolResult,
|
||||
acknowledgeMark: vi.fn(),
|
||||
close: vi.fn(),
|
||||
isConnected: vi.fn(() => true),
|
||||
};
|
||||
const provider: RealtimeVoiceProviderPlugin = {
|
||||
id: "relay-test",
|
||||
label: "Relay Test",
|
||||
isConfigured: () => true,
|
||||
createBridge: (request) => {
|
||||
bridgeRequest = request;
|
||||
return bridge;
|
||||
},
|
||||
};
|
||||
const fixture = createAbortableRelayRunFixture(provider);
|
||||
const relay = relaySessions.get(fixture.session.relaySessionId);
|
||||
if (!relay || !bridgeRequest) {
|
||||
throw new Error("expected active relay continuity fixture");
|
||||
}
|
||||
const request = bridgeRequest;
|
||||
request.onReady?.();
|
||||
request.onEvent?.({
|
||||
direction: "server",
|
||||
type: "response.audio.delta",
|
||||
itemId: "old-item",
|
||||
responseId: "old-response",
|
||||
});
|
||||
request.onAudio(Buffer.from("old audio"));
|
||||
request.onToolCall?.({
|
||||
itemId: "old-item",
|
||||
callId: "call-1",
|
||||
name: "custom_tool",
|
||||
args: {},
|
||||
});
|
||||
const working = submitTalkRealtimeRelayToolResult({
|
||||
relaySessionId: fixture.session.relaySessionId,
|
||||
connId: "conn-1",
|
||||
callId: "call-1",
|
||||
result: { status: "working" },
|
||||
options: { willContinue: true },
|
||||
});
|
||||
|
||||
request.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
request.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
|
||||
expect(fixture.abortController.signal.aborted).toBe(true);
|
||||
expect(handleBargeIn).not.toHaveBeenCalled();
|
||||
expect(bridge.close).not.toHaveBeenCalled();
|
||||
expect(submitToolResult).toHaveBeenCalledTimes(1);
|
||||
expect(relay.pendingWorkingToolResults.size).toBe(0);
|
||||
expect(relay.activeAgentRuns.size).toBe(0);
|
||||
expect(relay.activeAgentToolCalls.size).toBe(0);
|
||||
const payloads = fixture.broadcastToConnIds.mock.calls.map(([, payload]) => payload);
|
||||
const clears = payloads.filter(
|
||||
(payload): payload is Record<string, unknown> =>
|
||||
typeof payload === "object" &&
|
||||
payload !== null &&
|
||||
(payload as Record<string, unknown>).type === "clear",
|
||||
);
|
||||
expect(clears).toHaveLength(1);
|
||||
expectRecordFields(clears[0]?.talkEvent, {
|
||||
type: "turn.cancelled",
|
||||
payload: { reason: "session.continuity.reset" },
|
||||
final: true,
|
||||
});
|
||||
|
||||
pendingWorking.resolve();
|
||||
await working;
|
||||
expect(submitToolResult).toHaveBeenCalledTimes(1);
|
||||
|
||||
request.onEvent?.({ direction: "server", type: "session.created" });
|
||||
request.onAudio(Buffer.from("fresh audio"));
|
||||
const freshAudio = fixture.broadcastToConnIds.mock.calls
|
||||
.map(([, payload]) => payload)
|
||||
.findLast(
|
||||
(payload): payload is Record<string, unknown> =>
|
||||
typeof payload === "object" &&
|
||||
payload !== null &&
|
||||
(payload as Record<string, unknown>).type === "audio",
|
||||
);
|
||||
expectRecordFields(freshAudio, {
|
||||
type: "audio",
|
||||
audioBase64: Buffer.from("fresh audio").toString("base64"),
|
||||
});
|
||||
expect(freshAudio).not.toHaveProperty("itemId");
|
||||
expect(freshAudio).not.toHaveProperty("responseId");
|
||||
|
||||
request.onToolCall?.({
|
||||
itemId: "fresh-item",
|
||||
callId: "call-1",
|
||||
name: "custom_tool",
|
||||
args: {},
|
||||
});
|
||||
const freshToolCall = fixture.broadcastToConnIds.mock.calls
|
||||
.map(([, payload]) => payload)
|
||||
.findLast(
|
||||
(payload): payload is Record<string, unknown> =>
|
||||
typeof payload === "object" &&
|
||||
payload !== null &&
|
||||
(payload as Record<string, unknown>).type === "toolCall",
|
||||
);
|
||||
const freshCallId = freshToolCall?.callId;
|
||||
expect(typeof freshCallId).toBe("string");
|
||||
expect(freshCallId).not.toBe("call-1");
|
||||
await submitTalkRealtimeRelayToolResult({
|
||||
relaySessionId: fixture.session.relaySessionId,
|
||||
connId: "conn-1",
|
||||
callId: "call-1",
|
||||
result: { stale: true },
|
||||
});
|
||||
expect(submitToolResult).toHaveBeenCalledTimes(1);
|
||||
await submitTalkRealtimeRelayToolResult({
|
||||
relaySessionId: fixture.session.relaySessionId,
|
||||
connId: "conn-1",
|
||||
callId: String(freshCallId),
|
||||
result: { ok: true },
|
||||
});
|
||||
expect(submitToolResult).toHaveBeenLastCalledWith("call-1", { ok: true }, undefined);
|
||||
|
||||
request.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
request.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
expect(
|
||||
fixture.broadcastToConnIds.mock.calls
|
||||
.map(([, payload]) => payload)
|
||||
.filter(
|
||||
(payload) =>
|
||||
typeof payload === "object" &&
|
||||
payload !== null &&
|
||||
(payload as Record<string, unknown>).type === "clear",
|
||||
),
|
||||
).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("bridges browser audio, transcripts, and tool results through a backend provider", async () => {
|
||||
let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined;
|
||||
const bridge = {
|
||||
|
||||
@@ -24,6 +24,7 @@ import type {
|
||||
MeetingRealtimeToolCallParams,
|
||||
MeetingRuntimePlatform,
|
||||
} from "./realtime-engine.js";
|
||||
import { readMeetingRealtimeToolAbortSignal } from "./realtime-tool-continuity.js";
|
||||
|
||||
function resolveMeetingRealtimeTools(
|
||||
policy: RealtimeVoiceAgentConsultToolPolicy,
|
||||
@@ -77,7 +78,8 @@ export function createMeetingRealtimeEngineBindings(params: {
|
||||
...consult,
|
||||
}),
|
||||
tools: resolveMeetingRealtimeTools(params.config.realtime.toolPolicy),
|
||||
handleToolCall: async (call) =>
|
||||
handleToolCall: async (call) => {
|
||||
const abortSignal = readMeetingRealtimeToolAbortSignal(call.session);
|
||||
await handleMeetingRealtimeConsultToolCall({
|
||||
surface,
|
||||
config: params.fullConfig,
|
||||
@@ -85,17 +87,20 @@ export function createMeetingRealtimeEngineBindings(params: {
|
||||
logger: params.logger,
|
||||
agentId: params.config.realtime.agentId,
|
||||
toolPolicy: params.config.realtime.toolPolicy,
|
||||
abortSignal,
|
||||
...call,
|
||||
}),
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function submitMeetingConsultWorkingResponse(params: {
|
||||
session: RealtimeVoiceBridgeSession;
|
||||
abortSignal?: AbortSignal;
|
||||
callId: string;
|
||||
label: string;
|
||||
}): Promise<void> {
|
||||
if (!params.session.bridge.supportsToolResultContinuation) {
|
||||
if (params.abortSignal?.aborted || !params.session.bridge.supportsToolResultContinuation) {
|
||||
return;
|
||||
}
|
||||
await params.session.submitToolResult(
|
||||
@@ -116,6 +121,7 @@ async function consultMeetingAgent(params: {
|
||||
requesterSessionKey?: string;
|
||||
args: unknown;
|
||||
transcript: Array<{ role: "user" | "assistant"; text: string }>;
|
||||
abortSignal?: AbortSignal;
|
||||
}): Promise<{ text: string }> {
|
||||
const agentId = params.agentId
|
||||
? normalizeAgentId(params.agentId)
|
||||
@@ -142,6 +148,7 @@ async function consultMeetingAgent(params: {
|
||||
questionSourceLabel: params.surface.questionSourceLabel,
|
||||
toolsAllow: resolveRealtimeVoiceAgentConsultToolsAllow(params.toolPolicy),
|
||||
extraSystemPrompt: params.surface.extraSystemPrompt,
|
||||
abortSignal: params.abortSignal,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -158,12 +165,19 @@ async function handleMeetingRealtimeConsultToolCall(params: {
|
||||
meetingSessionId: string;
|
||||
requesterSessionKey?: string;
|
||||
transcript: Array<{ role: "user" | "assistant"; text: string }>;
|
||||
abortSignal?: AbortSignal;
|
||||
onTalkEvent?: (event: TalkEventInput) => void;
|
||||
}): Promise<void> {
|
||||
const callId = params.event.callId || params.event.itemId;
|
||||
if (params.abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
if (params.strategy !== "bidi") {
|
||||
const error = `Tool "${params.event.name}" is only available in bidi realtime strategy`;
|
||||
await params.session.submitToolResult(callId, { error });
|
||||
if (params.abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
params.onTalkEvent?.({
|
||||
type: "tool.error",
|
||||
callId,
|
||||
@@ -175,6 +189,9 @@ async function handleMeetingRealtimeConsultToolCall(params: {
|
||||
if (params.event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
|
||||
const error = `Tool "${params.event.name}" not available`;
|
||||
await params.session.submitToolResult(callId, { error });
|
||||
if (params.abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
params.onTalkEvent?.({
|
||||
type: "tool.error",
|
||||
callId,
|
||||
@@ -185,9 +202,13 @@ async function handleMeetingRealtimeConsultToolCall(params: {
|
||||
}
|
||||
await submitMeetingConsultWorkingResponse({
|
||||
session: params.session,
|
||||
abortSignal: params.abortSignal,
|
||||
callId,
|
||||
label: params.surface.workingResponseLabel,
|
||||
});
|
||||
if (params.abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
params.onTalkEvent?.({
|
||||
type: "tool.progress",
|
||||
callId,
|
||||
@@ -206,10 +227,17 @@ async function handleMeetingRealtimeConsultToolCall(params: {
|
||||
requesterSessionKey: params.requesterSessionKey,
|
||||
args: params.event.args,
|
||||
transcript: params.transcript,
|
||||
abortSignal: params.abortSignal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (params.abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
const message = formatErrorMessage(error);
|
||||
await params.session.submitToolResult(callId, { error: message });
|
||||
if (params.abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
params.onTalkEvent?.({
|
||||
type: "tool.error",
|
||||
callId,
|
||||
@@ -218,7 +246,13 @@ async function handleMeetingRealtimeConsultToolCall(params: {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (params.abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
await params.session.submitToolResult(callId, result);
|
||||
if (params.abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
params.onTalkEvent?.({
|
||||
type: "tool.result",
|
||||
callId,
|
||||
|
||||
@@ -5,16 +5,22 @@ import type {
|
||||
RealtimeVoiceBridgeCreateRequest,
|
||||
} from "../talk/provider-types.js";
|
||||
import type { MeetingRealtimeAudioTransport } from "./realtime-audio-transport.js";
|
||||
import { startMeetingRealtimeEngine } from "./realtime-engine.js";
|
||||
import {
|
||||
startMeetingRealtimeEngine,
|
||||
type MeetingRealtimeToolCallParams,
|
||||
} from "./realtime-engine.js";
|
||||
|
||||
type PendingWrite = {
|
||||
resolve: () => void;
|
||||
};
|
||||
|
||||
async function createEngineFixture() {
|
||||
async function createEngineFixture(options?: {
|
||||
handleToolCall?: (params: MeetingRealtimeToolCallParams) => Promise<void>;
|
||||
}) {
|
||||
let callbacks: RealtimeVoiceBridgeCreateRequest | undefined;
|
||||
let onHumanBargeIn: ((audio: Buffer) => boolean) | undefined;
|
||||
const handleBargeIn = vi.fn();
|
||||
const submitToolResult = vi.fn();
|
||||
const bridge: RealtimeVoiceBridge = {
|
||||
acknowledgeMark: vi.fn(),
|
||||
close: vi.fn(),
|
||||
@@ -23,7 +29,7 @@ async function createEngineFixture() {
|
||||
isConnected: vi.fn(() => true),
|
||||
sendAudio: vi.fn(),
|
||||
setMediaTimestamp: vi.fn(),
|
||||
submitToolResult: vi.fn(),
|
||||
submitToolResult,
|
||||
};
|
||||
const provider: RealtimeVoiceProviderPlugin = {
|
||||
id: "test",
|
||||
@@ -66,7 +72,7 @@ async function createEngineFixture() {
|
||||
},
|
||||
consultAgent: vi.fn(async () => ({ text: "unused" })),
|
||||
fullConfig: {} as never,
|
||||
handleToolCall: vi.fn(async () => {}),
|
||||
handleToolCall: options?.handleToolCall ?? vi.fn(async () => {}),
|
||||
logger: {
|
||||
debug: vi.fn(),
|
||||
error: vi.fn(),
|
||||
@@ -94,6 +100,7 @@ async function createEngineFixture() {
|
||||
clearOutput,
|
||||
handle,
|
||||
handleBargeIn,
|
||||
submitToolResult,
|
||||
releaseWrite(index: number) {
|
||||
const pending = pendingWrites[index];
|
||||
if (!pending) {
|
||||
@@ -127,6 +134,117 @@ async function createEngineFixture() {
|
||||
}
|
||||
|
||||
describe("meeting realtime engine output ownership", () => {
|
||||
it("rearms continuity reset when the provider creates a fresh session before ready", async () => {
|
||||
const fixture = await createEngineFixture();
|
||||
try {
|
||||
fixture.callbacks.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
fixture.callbacks.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(fixture.clearOutput).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
fixture.callbacks.onEvent?.({
|
||||
direction: "server",
|
||||
type: "session.created",
|
||||
});
|
||||
fixture.callbacks.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(fixture.clearOutput).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
} finally {
|
||||
await fixture.handle.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it("resets provider continuity without replaying old output or tool work", async () => {
|
||||
let releaseTool: (() => void) | undefined;
|
||||
const toolGate = new Promise<void>((resolve) => {
|
||||
releaseTool = resolve;
|
||||
});
|
||||
const fixture = await createEngineFixture({
|
||||
handleToolCall: async ({ session, event, onTalkEvent }) => {
|
||||
await toolGate;
|
||||
await session.submitToolResult(event.callId, { text: "stale result" });
|
||||
onTalkEvent({
|
||||
type: "tool.result",
|
||||
callId: event.callId,
|
||||
payload: { name: event.name },
|
||||
final: true,
|
||||
});
|
||||
},
|
||||
});
|
||||
try {
|
||||
const active = Buffer.from([1]);
|
||||
const stale = Buffer.from([2]);
|
||||
const fresh = Buffer.from([3]);
|
||||
fixture.callbacks.onReady?.();
|
||||
fixture.callbacks.onTranscript?.("user", "old turn", true);
|
||||
fixture.sendOutputAudio(active, "response-1");
|
||||
await vi.waitFor(() => {
|
||||
expect(fixture.writeOutput).toHaveBeenCalledOnce();
|
||||
});
|
||||
fixture.sendOutputAudio(stale, "response-1");
|
||||
fixture.callbacks.onToolCall?.({
|
||||
itemId: "item-old",
|
||||
callId: "call-old",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "old work" },
|
||||
});
|
||||
|
||||
fixture.callbacks.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
fixture.callbacks.onEvent?.({
|
||||
direction: "client",
|
||||
type: "session.continuity.reset",
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(fixture.clearOutput).toHaveBeenCalledOnce();
|
||||
});
|
||||
expect(fixture.handleBargeIn).not.toHaveBeenCalled();
|
||||
expect(
|
||||
fixture.handle
|
||||
.getHealth()
|
||||
.recentTalkEvents.filter((event) => event.type === "turn.cancelled"),
|
||||
).toHaveLength(1);
|
||||
|
||||
releaseTool?.();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(fixture.submitToolResult).not.toHaveBeenCalled();
|
||||
expect(
|
||||
fixture.handle.getHealth().recentTalkEvents.some((event) => event.type === "tool.result"),
|
||||
).toBe(false);
|
||||
|
||||
fixture.releaseWrite(0);
|
||||
await vi.waitFor(() => {
|
||||
expect(fixture.clearOutput).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
expect(fixture.writeOutput).not.toHaveBeenCalledWith(stale);
|
||||
|
||||
fixture.callbacks.onReady?.();
|
||||
fixture.sendOutputAudio(fresh, "response-1");
|
||||
await vi.waitFor(() => {
|
||||
expect(fixture.writeOutput).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
expect(fixture.writeOutput).toHaveBeenLastCalledWith(fresh);
|
||||
fixture.releaseWrite(1);
|
||||
} finally {
|
||||
await fixture.handle.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it("serializes transport writes and coalesces queued 20 ms frames", async () => {
|
||||
const fixture = await createEngineFixture();
|
||||
try {
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
resolveMeetingRealtimeProvider,
|
||||
} from "./realtime-engine-support.js";
|
||||
import { createMeetingRealtimeOutputOwner } from "./realtime-output-owner.js";
|
||||
import { createMeetingRealtimeToolContinuity } from "./realtime-tool-continuity.js";
|
||||
|
||||
export {
|
||||
formatMeetingAgentAudioModelLog,
|
||||
@@ -35,7 +36,6 @@ export {
|
||||
normalizeMeetingTtsPromptText,
|
||||
resolveMeetingRealtimeTranscriptionProvider,
|
||||
} from "./realtime-engine-support.js";
|
||||
|
||||
export type MeetingRuntimePlatform = {
|
||||
/** Adapter-owned identity keeps platform names and log prefixes out of core. */
|
||||
displayName: string;
|
||||
@@ -98,7 +98,6 @@ const MEETING_REALTIME_OUTPUT_MAX_PENDING_MS = 2_000;
|
||||
const MEETING_REALTIME_OUTPUT_MAX_WRITE_MS = 500;
|
||||
const MEETING_REALTIME_OUTPUT_MAX_PENDING_FRAMES = 256;
|
||||
const MEETING_REALTIME_CANCELLATION_RACE_DETAIL = "Cancellation failed: no active response found";
|
||||
|
||||
export async function startMeetingRealtimeEngine(params: {
|
||||
config: MeetingRealtimeEngineConfig;
|
||||
fullConfig: OpenClawConfig;
|
||||
@@ -135,9 +134,11 @@ export async function startMeetingRealtimeEngine(params: {
|
||||
let outputPendingBytes = 0;
|
||||
let outputPendingFrames = 0;
|
||||
let outputGenerationActive = false;
|
||||
let continuityResetActive = false;
|
||||
let outputClearTail = Promise.resolve();
|
||||
const outputQueue: Array<{ audio: Buffer; generation: number }> = [];
|
||||
const outputOwner = createMeetingRealtimeOutputOwner();
|
||||
const toolContinuity = createMeetingRealtimeToolContinuity(params.handleToolCall);
|
||||
const outputMaxPendingBytes =
|
||||
meetingOutputBytesPerMs(params.config.chrome.audioFormat) *
|
||||
MEETING_REALTIME_OUTPUT_MAX_PENDING_MS;
|
||||
@@ -160,6 +161,7 @@ export async function startMeetingRealtimeEngine(params: {
|
||||
outputOwner.reset();
|
||||
outputClearAfterActive = false;
|
||||
invalidateOutputQueue();
|
||||
toolContinuity.reset("meeting realtime stopped");
|
||||
}
|
||||
if (stopPromise) {
|
||||
await stopPromise;
|
||||
@@ -549,6 +551,30 @@ export async function startMeetingRealtimeEngine(params: {
|
||||
}
|
||||
},
|
||||
onEvent: (event) => {
|
||||
if (event.direction === "server" && event.type === "session.created") {
|
||||
continuityResetActive = false;
|
||||
}
|
||||
if (event.direction === "client" && event.type === "session.continuity.reset") {
|
||||
if (continuityResetActive) {
|
||||
return;
|
||||
}
|
||||
continuityResetActive = true;
|
||||
realtimeReady = false;
|
||||
outputOwner.reset();
|
||||
outputGenerationActive = false;
|
||||
toolContinuity.reset(event.type);
|
||||
const turnId = harness.talk.activeTurnId;
|
||||
invalidateOutputPlayback();
|
||||
harness.flushOutput(clearOutputPlayback);
|
||||
harness.finishOutputAudio(event.type);
|
||||
if (turnId) {
|
||||
harness.talk.cancelTurn({
|
||||
turnId,
|
||||
payload: { ...outputTalkPayload, reason: event.type },
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
outputOwner.noteEvent(event);
|
||||
if (event.type === "input_audio_buffer.speech_started") {
|
||||
harness.ensureTurn();
|
||||
@@ -600,26 +626,18 @@ export async function startMeetingRealtimeEngine(params: {
|
||||
);
|
||||
}
|
||||
},
|
||||
onToolCall: (event, session) => {
|
||||
harness.emit({
|
||||
type: "tool.call",
|
||||
turnId: harness.ensureTurn(),
|
||||
itemId: event.itemId,
|
||||
callId: event.callId,
|
||||
payload: { name: event.name, args: event.args },
|
||||
});
|
||||
const turnId = harness.ensureTurn();
|
||||
return params.handleToolCall({
|
||||
strategy,
|
||||
onToolCall: (event, session) =>
|
||||
toolContinuity.run({
|
||||
session,
|
||||
event,
|
||||
meetingSessionId: params.meetingSessionId,
|
||||
requesterSessionKey: params.requesterSessionKey,
|
||||
transcript: harness.transcript,
|
||||
onTalkEvent: (inputLocal) =>
|
||||
harness.emit({ ...inputLocal, turnId: inputLocal.turnId ?? turnId }),
|
||||
});
|
||||
},
|
||||
call: {
|
||||
strategy,
|
||||
event,
|
||||
meetingSessionId: params.meetingSessionId,
|
||||
requesterSessionKey: params.requesterSessionKey,
|
||||
transcript: harness.transcript,
|
||||
},
|
||||
harness,
|
||||
}),
|
||||
onError: (error) => {
|
||||
harness.emit({
|
||||
type: "session.error",
|
||||
@@ -644,6 +662,7 @@ export async function startMeetingRealtimeEngine(params: {
|
||||
},
|
||||
onReady: () => {
|
||||
realtimeReady = true;
|
||||
continuityResetActive = false;
|
||||
harness.emit({
|
||||
type: "session.ready",
|
||||
payload: outputTalkPayload,
|
||||
|
||||
82
src/meeting-bot/realtime-tool-continuity.ts
Normal file
82
src/meeting-bot/realtime-tool-continuity.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
import type { RealtimeVoiceToolCallEvent } from "../talk/provider-types.js";
|
||||
import type { RealtimeVoiceSessionHarness } from "../talk/realtime-session-harness.js";
|
||||
import type { RealtimeVoiceBridgeSession } from "../talk/session-runtime.js";
|
||||
import type { TalkEventInput } from "../talk/talk-events.js";
|
||||
|
||||
type MeetingRealtimeToolContinuityCall = {
|
||||
session: RealtimeVoiceBridgeSession;
|
||||
event: RealtimeVoiceToolCallEvent;
|
||||
onTalkEvent: (event: TalkEventInput) => void;
|
||||
};
|
||||
|
||||
const meetingRealtimeToolAbortSignals = new WeakMap<RealtimeVoiceBridgeSession, AbortSignal>();
|
||||
|
||||
export function readMeetingRealtimeToolAbortSignal(
|
||||
session: RealtimeVoiceBridgeSession,
|
||||
): AbortSignal | undefined {
|
||||
return meetingRealtimeToolAbortSignals.get(session);
|
||||
}
|
||||
|
||||
export function createMeetingRealtimeToolContinuity<
|
||||
TCall extends MeetingRealtimeToolContinuityCall,
|
||||
>(handleToolCall: (call: TCall) => Promise<void>) {
|
||||
let epoch = 0;
|
||||
const activeControllers = new Set<AbortController>();
|
||||
|
||||
// Reset invalidates provider submissions and Talk events together so late
|
||||
// async completions cannot leak into the replacement provider generation.
|
||||
const reset = (reason: string) => {
|
||||
epoch += 1;
|
||||
for (const controller of activeControllers) {
|
||||
controller.abort(reason);
|
||||
}
|
||||
activeControllers.clear();
|
||||
};
|
||||
|
||||
const run = (params: {
|
||||
session: RealtimeVoiceBridgeSession;
|
||||
call: Omit<TCall, "session" | "onTalkEvent">;
|
||||
harness: Pick<RealtimeVoiceSessionHarness, "emit" | "ensureTurn">;
|
||||
}): Promise<void> => {
|
||||
const callEpoch = epoch;
|
||||
const controller = new AbortController();
|
||||
activeControllers.add(controller);
|
||||
const isActive = () => !controller.signal.aborted && callEpoch === epoch;
|
||||
const turnId = params.harness.ensureTurn();
|
||||
params.harness.emit({
|
||||
type: "tool.call",
|
||||
turnId,
|
||||
itemId: params.call.event.itemId,
|
||||
callId: params.call.event.callId,
|
||||
payload: { name: params.call.event.name, args: params.call.event.args },
|
||||
});
|
||||
const guardedSession = Object.create(params.session) as RealtimeVoiceBridgeSession;
|
||||
meetingRealtimeToolAbortSignals.set(guardedSession, controller.signal);
|
||||
guardedSession.submitToolResult = (callId, result, options) => {
|
||||
if (!isActive()) {
|
||||
return;
|
||||
}
|
||||
return params.session.submitToolResult(callId, result, options);
|
||||
};
|
||||
return handleToolCall({
|
||||
...params.call,
|
||||
session: guardedSession,
|
||||
onTalkEvent: (event) => {
|
||||
if (isActive()) {
|
||||
params.harness.emit({ ...event, turnId: event.turnId ?? turnId });
|
||||
}
|
||||
},
|
||||
} as TCall)
|
||||
.catch((error: unknown) => {
|
||||
if (isActive()) {
|
||||
throw error;
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
meetingRealtimeToolAbortSignals.delete(guardedSession);
|
||||
activeControllers.delete(controller);
|
||||
});
|
||||
};
|
||||
|
||||
return { reset, run };
|
||||
}
|
||||
Reference in New Issue
Block a user