mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-12 21:06:02 +00:00
* fix: route iOS OpenAI realtime talk through WebRTC * fix(ios): harden native realtime routing * fix(ios): preserve realtime route ownership * fix(ios): balance realtime audio sessions * chore(ios): sync native i18n inventory * fix(ios): preserve Azure realtime relay * fix(ios): harden realtime session recovery * chore(ios): sync native i18n inventory * fix(ios): preserve selected Azure realtime routing * fix(ios): rotate terminal realtime sessions --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
847 lines
32 KiB
Swift
847 lines
32 KiB
Swift
import AVFoundation
|
|
import Foundation
|
|
import OpenClawKit
|
|
import Testing
|
|
@testable import OpenClaw
|
|
|
|
@MainActor
|
|
struct TalkModeManagerTests {
|
|
@Test func `recognizes open AI maximum duration errors as terminal`() throws {
|
|
let event = try JSONDecoder().decode(
|
|
TalkRealtimeServerEvent.self,
|
|
from: Data(#"{"type":"error","error":{"message":"Your session hit the maximum duration of 60 minutes."}}"#
|
|
.utf8))
|
|
|
|
#expect(event.isMaximumDurationError)
|
|
}
|
|
|
|
@Test func `keeps recoverable open AI errors in the current session`() throws {
|
|
let event = try JSONDecoder().decode(
|
|
TalkRealtimeServerEvent.self,
|
|
from: Data(#"{"type":"error","error":{"message":"Cancellation failed: no active response found"}}"#.utf8))
|
|
|
|
#expect(!event.isMaximumDurationError)
|
|
}
|
|
|
|
@Test func `parses open AI realtime provider model and voice`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"provider": "elevenlabs",
|
|
"providers": [
|
|
"elevenlabs": [
|
|
"modelId": "eleven_v3",
|
|
"voiceId": "eleven-voice",
|
|
],
|
|
],
|
|
"resolved": [
|
|
"provider": "elevenlabs",
|
|
"config": [
|
|
"modelId": "eleven_v3",
|
|
"voiceId": "eleven-voice",
|
|
],
|
|
],
|
|
"realtime": [
|
|
"provider": " openai ",
|
|
"model": " gpt-realtime-2 ",
|
|
"voice": " marin ",
|
|
"mode": "realtime",
|
|
"transport": "gateway-relay",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.activeProvider == "elevenlabs")
|
|
#expect(parsed.executionMode == .realtimeRelay)
|
|
#expect(parsed.defaultModelId == "eleven_v3")
|
|
#expect(parsed.defaultVoiceId == "eleven-voice")
|
|
#expect(parsed.realtimeProvider == "openai")
|
|
#expect(parsed.realtimeModelId == "gpt-realtime-2")
|
|
#expect(parsed.realtimeVoiceId == "marin")
|
|
}
|
|
|
|
@Test func `infers realtime provider when provider map has single entry`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"mode": "realtime",
|
|
"transport": "webrtc",
|
|
"providers": [
|
|
"openai": [
|
|
"model": "gpt-realtime-2",
|
|
],
|
|
],
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .realtimeWebRTC)
|
|
#expect(parsed.realtimeProvider == "openai")
|
|
#expect(parsed.realtimeModelId == "gpt-realtime-2")
|
|
}
|
|
|
|
@Test func `formats generic realtime voice mode without native provider fallback`() {
|
|
let descriptor = TalkVoiceModeDescriptorBuilder.build(
|
|
providerId: "realtime",
|
|
providerLabel: "Realtime Voice",
|
|
modelId: "gpt-realtime-2",
|
|
voiceId: nil,
|
|
transport: "webrtc",
|
|
isRealtime: true)
|
|
|
|
#expect(descriptor.title == "Realtime Voice")
|
|
#expect(descriptor.subtitle == "Native WebRTC • gpt-realtime-2")
|
|
}
|
|
|
|
@Test func `defaults open AI realtime model when provider omits model`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "openai",
|
|
"mode": "realtime",
|
|
"transport": "gateway-relay",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .realtimeRelay)
|
|
#expect(parsed.defaultModelId == "eleven_v3")
|
|
#expect(parsed.realtimeModelId == "gpt-realtime-2")
|
|
#expect(parsed.realtimeVoiceId == nil)
|
|
}
|
|
|
|
@Test func `resolves realtime voice picker overrides`() {
|
|
#expect(TalkModeRealtimeVoiceSelection.resolvedOverride(nil) == nil)
|
|
#expect(TalkModeRealtimeVoiceSelection.resolvedOverride("") == nil)
|
|
#expect(TalkModeRealtimeVoiceSelection.resolvedOverride(" Cedar ") == "cedar")
|
|
#expect(TalkModeRealtimeVoiceSelection.resolvedOverride("unknown") == nil)
|
|
}
|
|
|
|
@Test func `formats open AI realtime voice mode`() {
|
|
let descriptor = TalkVoiceModeDescriptorBuilder.build(
|
|
providerId: "openai",
|
|
providerLabel: "OpenAI",
|
|
modelId: "gpt-realtime-2",
|
|
voiceId: "marin",
|
|
transport: "webrtc",
|
|
isRealtime: true)
|
|
|
|
#expect(descriptor.title == "GPT Realtime 2.0")
|
|
#expect(descriptor.subtitle == "Native WebRTC • Marin")
|
|
#expect(descriptor.accessibilityValue == "GPT Realtime 2.0, Native WebRTC • Marin")
|
|
}
|
|
|
|
@Test func `formats gateway relay realtime voice mode`() {
|
|
let descriptor = TalkVoiceModeDescriptorBuilder.build(
|
|
providerId: "google",
|
|
providerLabel: "Google Live Voice",
|
|
modelId: "gemini-live-2.5-flash-preview",
|
|
voiceId: nil,
|
|
transport: "gateway-relay",
|
|
isRealtime: true)
|
|
|
|
#expect(descriptor.title == "Google Live Voice")
|
|
#expect(descriptor.subtitle == "Gateway Relay • gemini-live-2.5-flash-preview")
|
|
}
|
|
|
|
@Test func `formats eleven labs voice mode`() {
|
|
let descriptor = TalkVoiceModeDescriptorBuilder.build(
|
|
providerId: "elevenlabs",
|
|
providerLabel: "ElevenLabs",
|
|
modelId: "eleven_v3",
|
|
voiceId: "voice-id",
|
|
transport: "native",
|
|
isRealtime: false)
|
|
|
|
#expect(descriptor.title == "ElevenLabs")
|
|
#expect(descriptor.subtitle == "Native • eleven_v3 • voice-id")
|
|
}
|
|
|
|
@Test func `formats system voice fallback mode`() {
|
|
let descriptor = TalkVoiceModeDescriptorBuilder.build(
|
|
providerId: "system",
|
|
providerLabel: "iOS System Voice",
|
|
modelId: nil,
|
|
voiceId: "en-US",
|
|
transport: "native",
|
|
isRealtime: false)
|
|
|
|
#expect(descriptor.title == "iOS System Voice")
|
|
#expect(descriptor.subtitle == "Native • en-US")
|
|
}
|
|
|
|
@Test func `open AI realtime selection defaults to native web RTC`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
|
|
manager._test_applyOpenAIRealtimeSelectionDefaults()
|
|
|
|
#expect(manager._test_executionMode() == .realtimeWebRTC)
|
|
#expect(manager._test_realtimeProvider() == "openai")
|
|
#expect(manager._test_realtimeModelId() == "gpt-realtime-2")
|
|
#expect(!manager._test_gatewayTalkUsesRealtimeRelay())
|
|
}
|
|
|
|
@Test func `open AI realtime selection clears stale realtime config`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"model": "gemini-live-2.5-flash-preview",
|
|
"voice": "puck",
|
|
"mode": "realtime",
|
|
"transport": "gateway-relay",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
manager._test_applyLoadedTalkConfig(parsed, providerSelection: .gatewayDefault)
|
|
manager._test_applyOpenAIRealtimeSelectionDefaults()
|
|
|
|
#expect(manager._test_executionMode() == .realtimeWebRTC)
|
|
#expect(manager._test_realtimeProvider() == "openai")
|
|
#expect(manager._test_realtimeModelId() == "gpt-realtime-2")
|
|
#expect(manager.gatewayTalkRealtimeVoiceId == nil)
|
|
#expect(!manager._test_gatewayTalkUsesRealtimeRelay())
|
|
}
|
|
|
|
@Test func `open AI realtime selection keeps explicit open AI voice override`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
let defaults = UserDefaults.standard
|
|
defaults.set(" Cedar ", forKey: TalkModeRealtimeVoiceSelection.storageKey)
|
|
defer { defaults.removeObject(forKey: TalkModeRealtimeVoiceSelection.storageKey) }
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"model": "gemini-live-2.5-flash-preview",
|
|
"voice": "puck",
|
|
"mode": "realtime",
|
|
"transport": "gateway-relay",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
manager._test_applyLoadedTalkConfig(parsed, providerSelection: .openAIRealtime)
|
|
|
|
#expect(manager._test_realtimeProvider() == "openai")
|
|
#expect(manager._test_realtimeModelId() == "gpt-realtime-2")
|
|
#expect(manager.gatewayTalkRealtimeVoiceId == "cedar")
|
|
}
|
|
|
|
@Test func `open AI selection preserves configured voice for case insensitive provider`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": " OpenAI ",
|
|
"voice": "marin",
|
|
"mode": "realtime",
|
|
"transport": "webrtc",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
manager._test_applyLoadedTalkConfig(parsed, providerSelection: .openAIRealtime)
|
|
|
|
#expect(manager._test_realtimeProvider() == "openai")
|
|
#expect(manager.gatewayTalkRealtimeVoiceId == "marin")
|
|
}
|
|
|
|
@Test func `builds generic realtime fallback issue for display`() {
|
|
let issue = TalkRuntimeIssue.realtimeUnavailable(
|
|
message: "OpenAI API key rejected with 401",
|
|
provider: "openai",
|
|
model: "gpt-realtime-2",
|
|
transport: "gateway-relay",
|
|
phase: "start")
|
|
|
|
#expect(issue.code == .realtimeUnavailable)
|
|
#expect(issue.displayMessage == "OpenAI API key rejected with 401")
|
|
#expect(issue.diagnosticSummary.contains("provider: openai"))
|
|
#expect(issue.diagnosticSummary.contains("model: gpt-realtime-2"))
|
|
#expect(issue.fallbackStatusText == "Listening (iOS Speech fallback)")
|
|
#expect(issue.fallbackBannerTitle == "Using iOS Speech fallback")
|
|
#expect(issue.fallbackBannerOwnerLabel == "Fallback active")
|
|
#expect(issue
|
|
.fallbackBannerMessage ==
|
|
"Realtime voice did not start. Talk is running with iOS speech recognition and TTS.")
|
|
#expect(issue.technicalDetails.contains("code: realtime_unavailable"))
|
|
}
|
|
|
|
@Test func `native fallback keeps realtime issue visible`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
let issue = TalkRuntimeIssue(
|
|
code: .realtimeUnavailable,
|
|
message: "Realtime closed before it became ready.",
|
|
provider: "openai",
|
|
model: "gpt-realtime-2",
|
|
transport: "gateway-relay",
|
|
phase: "connect")
|
|
|
|
manager._test_markNativeFallbackActive(after: issue)
|
|
|
|
#expect(manager.statusText == "Listening (iOS Speech fallback)")
|
|
#expect(manager._test_gatewayTalkActiveModeTitle() == "iOS Speech fallback")
|
|
#expect(manager._test_gatewayTalkActiveModeSubtitle() == "Realtime closed before it became ready.")
|
|
#expect(manager._test_gatewayTalkLastIssueText()?.contains("phase: connect") == true)
|
|
#expect(manager._test_gatewayTalkCurrentFallbackIssue() == issue)
|
|
}
|
|
|
|
@Test func `gateway talk issue details drive realtime failure display`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
let error = GatewayResponseError(
|
|
method: "talk.session.create",
|
|
code: "UNAVAILABLE",
|
|
message: "Error: OpenAI API key rejected with 401",
|
|
details: [
|
|
"talkIssue": AnyCodable([
|
|
"code": "realtime_unavailable",
|
|
"message": "OpenAI API key rejected with 401",
|
|
"provider": "openai",
|
|
"model": "gpt-realtime-2",
|
|
"transport": "gateway-relay",
|
|
"phase": "request",
|
|
]),
|
|
])
|
|
|
|
let issue = manager._test_realtimeIssue(from: error, phase: "start")
|
|
|
|
#expect(issue.code == .realtimeUnavailable)
|
|
#expect(issue.displayMessage == "OpenAI API key rejected with 401")
|
|
#expect(issue.provider == "openai")
|
|
#expect(issue.model == "gpt-realtime-2")
|
|
#expect(issue.transport == "gateway-relay")
|
|
#expect(issue.phase == "request")
|
|
}
|
|
|
|
@Test func `relay startup issue survives until ready status`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
let issue = TalkRuntimeIssue(
|
|
code: .realtimeUnavailable,
|
|
message: "OpenAI API key rejected with 401",
|
|
provider: "openai",
|
|
model: "gpt-realtime-2",
|
|
transport: "gateway-relay",
|
|
phase: "connect")
|
|
|
|
manager._test_recordRealtimeIssue(issue)
|
|
manager._test_handleRealtimeRelayStatus("Connecting realtime…")
|
|
|
|
#expect(manager._test_gatewayTalkActiveModeTitle() == "Realtime unavailable")
|
|
#expect(manager._test_gatewayTalkLastIssueText()?.contains("OpenAI API key rejected") == true)
|
|
|
|
manager._test_handleRealtimeRelayStatus("Listening (Realtime)")
|
|
|
|
#expect(manager.statusText == "Listening (Realtime)")
|
|
#expect(manager._test_gatewayTalkLastIssueText() == nil)
|
|
#expect(manager._test_gatewayTalkCurrentFallbackIssue() == nil)
|
|
}
|
|
|
|
@Test func `relay close clears active realtime mode`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
|
|
manager._test_handleRealtimeRelayStatus("Listening (Realtime)")
|
|
#expect(manager.statusText == "Listening (Realtime)")
|
|
#expect(manager._test_gatewayTalkActiveModeTitle() != "Not active")
|
|
|
|
manager._test_handleRealtimeRelayStatus("Ready")
|
|
|
|
#expect(manager.statusText == "Ready")
|
|
#expect(manager._test_gatewayTalkActiveModeTitle() == "Not active")
|
|
#expect(manager._test_gatewayTalkActiveModeSubtitle() == nil)
|
|
}
|
|
|
|
@Test func `relay close restarts enabled continuous realtime`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
manager._test_prepareEnabledRealtimeSessionForClose()
|
|
|
|
manager._test_handleRealtimeRelayStatus("Listening (Realtime)")
|
|
manager._test_handleRealtimeRelayStatus("Ready")
|
|
|
|
#expect(manager.statusText == "Reconnecting")
|
|
#expect(manager._test_rapidRealtimeRestartCount() == 1)
|
|
manager.isEnabled = false
|
|
}
|
|
|
|
@Test func `recurring realtime ready status preserves push to talk capture`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
|
|
#expect(manager._test_realtimeStatusPreservesPushToTalkCapture())
|
|
}
|
|
|
|
@Test func `relay retry clears stale fallback trigger but keeps last issue visible`() {
|
|
let manager = TalkModeManager(allowSimulatorCapture: true)
|
|
let issue = TalkRuntimeIssue(
|
|
code: .realtimeUnavailable,
|
|
message: "Realtime closed before it became ready.",
|
|
provider: "openai",
|
|
model: "gpt-realtime-2",
|
|
transport: "gateway-relay",
|
|
phase: "connect")
|
|
|
|
manager._test_recordRealtimeIssue(issue)
|
|
manager._test_markNativeFallbackActive(after: issue)
|
|
#expect(manager._test_hasPendingRealtimeIssue())
|
|
#expect(manager._test_gatewayTalkCurrentFallbackIssue() == issue)
|
|
|
|
manager._test_prepareRealtimeRelayStart()
|
|
|
|
#expect(!manager._test_hasPendingRealtimeIssue())
|
|
#expect(manager._test_gatewayTalkCurrentFallbackIssue() == nil)
|
|
#expect(manager._test_gatewayTalkLastIssueText()?.contains("Realtime closed before") == true)
|
|
}
|
|
|
|
@Test func `maps web RTC realtime transport to native web RTC on IOS`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "openai",
|
|
"mode": "realtime",
|
|
"transport": "webrtc",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .realtimeWebRTC)
|
|
}
|
|
|
|
@Test func `keeps Azure open AI realtime on gateway relay`() {
|
|
for providerConfig in [
|
|
["azureEndpoint": "https://example.openai.azure.com"],
|
|
["azureDeployment": "realtime-prod"],
|
|
] {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "openai",
|
|
"providers": ["openai": providerConfig],
|
|
"mode": "realtime",
|
|
"transport": "webrtc",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
let routing = TalkModeRoutingResolver.resolve(
|
|
parsed: parsed,
|
|
providerSelection: .openAIRealtime,
|
|
defaultProvider: "elevenlabs",
|
|
defaultRealtimeModelId: "gpt-realtime-2")
|
|
|
|
#expect(parsed.executionMode == .realtimeRelay)
|
|
#expect(routing.route == .realtimeRelay)
|
|
}
|
|
}
|
|
|
|
@Test func `open AI selection keeps its Azure config on gateway relay`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"providers": [
|
|
"google": ["model": "gemini-live"],
|
|
"OpenAI": ["azureDeployment": "realtime-prod"],
|
|
],
|
|
"mode": "realtime",
|
|
"transport": "webrtc",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
let routing = TalkModeRoutingResolver.resolve(
|
|
parsed: parsed,
|
|
providerSelection: .openAIRealtime,
|
|
defaultProvider: "elevenlabs",
|
|
defaultRealtimeModelId: "gpt-realtime-2")
|
|
|
|
#expect(parsed.realtimeProvider == "google")
|
|
#expect(routing.route == .realtimeRelay)
|
|
}
|
|
|
|
@Test func `restarts an enabled continuous realtime session after provider close`() {
|
|
#expect(TalkModeManager._test_shouldRestartRealtimeSession(
|
|
isEnabled: true,
|
|
gatewayConnected: true,
|
|
captureIsContinuous: true))
|
|
#expect(!TalkModeManager._test_shouldRestartRealtimeSession(
|
|
isEnabled: false,
|
|
gatewayConnected: true,
|
|
captureIsContinuous: true))
|
|
#expect(!TalkModeManager._test_shouldRestartRealtimeSession(
|
|
isEnabled: true,
|
|
gatewayConnected: false,
|
|
captureIsContinuous: true))
|
|
#expect(!TalkModeManager._test_shouldRestartRealtimeSession(
|
|
isEnabled: true,
|
|
gatewayConnected: true,
|
|
captureIsContinuous: false))
|
|
|
|
#expect(TalkModeManager._test_realtimeRestartAttempt(
|
|
previousRapidRestarts: 1,
|
|
activeDuration: 5) == 2)
|
|
#expect(TalkModeManager._test_realtimeRestartAttempt(
|
|
previousRapidRestarts: 2,
|
|
activeDuration: 31) == 1)
|
|
#expect(TalkModeManager._test_realtimeRestartDelayNanoseconds(attempt: 1) == 500_000_000)
|
|
#expect(TalkModeManager._test_realtimeRestartDelayNanoseconds(attempt: 2) == 2_000_000_000)
|
|
#expect(TalkModeManager._test_realtimeRestartDelayNanoseconds(attempt: 3) == nil)
|
|
}
|
|
|
|
@Test func `keeps provider web socket realtime transport on gateway relay`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"mode": "realtime",
|
|
"transport": "provider-websocket",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .realtimeRelay)
|
|
}
|
|
|
|
@Test func `leaves native mode for unsupported realtime brain`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"mode": "realtime",
|
|
"transport": "gateway-relay",
|
|
"brain": "direct-tools",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .native)
|
|
}
|
|
|
|
@Test func `keeps non open AI realtime default transport on gateway relay`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"mode": "realtime",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .realtimeRelay)
|
|
}
|
|
|
|
@Test func `keeps non open AI web RTC transport on gateway relay`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"model": "gemini-live-2.5-flash-preview",
|
|
"mode": "realtime",
|
|
"transport": "webrtc",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .realtimeRelay)
|
|
}
|
|
|
|
@Test func `open AI selection overrides non open AI web RTC provider`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"mode": "realtime",
|
|
"transport": "webrtc",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
let routing = TalkModeRoutingResolver.resolve(
|
|
parsed: parsed,
|
|
providerSelection: .openAIRealtime,
|
|
defaultProvider: "elevenlabs",
|
|
defaultRealtimeModelId: "gpt-realtime-2")
|
|
|
|
#expect(routing.activeProvider == "openai")
|
|
#expect(routing.realtimeProvider == "openai")
|
|
#expect(routing.realtimeModelId == "gpt-realtime-2")
|
|
#expect(routing.executionMode == .realtimeWebRTC)
|
|
#expect(routing.route == .realtimeWebRTC)
|
|
}
|
|
|
|
@Test func `open AI selection preserves explicit gateway owned transport`() {
|
|
for transport in ["gateway-relay", "provider-websocket"] {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "google",
|
|
"mode": "realtime",
|
|
"transport": transport,
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
let routing = TalkModeRoutingResolver.resolve(
|
|
parsed: parsed,
|
|
providerSelection: .openAIRealtime,
|
|
defaultProvider: "elevenlabs",
|
|
defaultRealtimeModelId: "gpt-realtime-2")
|
|
|
|
#expect(routing.realtimeProvider == "openai")
|
|
#expect(routing.executionMode == .realtimeRelay)
|
|
#expect(routing.route == .realtimeRelay)
|
|
}
|
|
}
|
|
|
|
@Test func `speaker preference preserves external audio routes`() {
|
|
let externalRouteOptions = TalkAudioRoute.categoryOptions(speakerphoneEnabled: false)
|
|
#expect(externalRouteOptions.contains(.allowBluetoothHFP))
|
|
#expect(externalRouteOptions.contains(.allowBluetoothA2DP))
|
|
#expect(externalRouteOptions.contains(.allowAirPlay))
|
|
#expect(!externalRouteOptions.contains(.defaultToSpeaker))
|
|
#expect(TalkAudioRoute.categoryOptions(speakerphoneEnabled: true).contains(.defaultToSpeaker))
|
|
|
|
#expect(TalkAudioRoute.shouldForceSpeaker(
|
|
preferenceEnabled: true,
|
|
outputPortTypes: [.builtInReceiver]))
|
|
#expect(TalkAudioRoute.shouldForceSpeaker(
|
|
preferenceEnabled: true,
|
|
outputPortTypes: [.builtInSpeaker]))
|
|
#expect(!TalkAudioRoute.shouldForceSpeaker(
|
|
preferenceEnabled: false,
|
|
outputPortTypes: [.builtInReceiver]))
|
|
#expect(!TalkAudioRoute.shouldForceSpeaker(
|
|
preferenceEnabled: true,
|
|
outputPortTypes: []))
|
|
|
|
let externalOutputs: [AVAudioSession.Port] = [
|
|
.airPlay,
|
|
.bluetoothA2DP,
|
|
.bluetoothHFP,
|
|
.bluetoothLE,
|
|
.carAudio,
|
|
.headphones,
|
|
.HDMI,
|
|
.lineOut,
|
|
.usbAudio,
|
|
]
|
|
for output in externalOutputs {
|
|
#expect(!TalkAudioRoute.shouldForceSpeaker(
|
|
preferenceEnabled: true,
|
|
outputPortTypes: [output]))
|
|
}
|
|
}
|
|
|
|
@Test func `maps open AI realtime default transport to native web RTC`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "openai",
|
|
"mode": "realtime",
|
|
"brain": "agent-consult",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .realtimeWebRTC)
|
|
}
|
|
|
|
@Test func `parses redacted gateway realtime config`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"providers": [
|
|
"elevenlabs": [
|
|
"apiKey": "__OPENCLAW_REDACTED__",
|
|
"voiceId": "bIHbv24MWmeRgasZH58o",
|
|
],
|
|
],
|
|
"realtime": [
|
|
"provider": "openai",
|
|
"providers": [
|
|
"openai": [
|
|
"model": "gpt-realtime-2",
|
|
"voice": "cedar",
|
|
],
|
|
],
|
|
"model": "gpt-realtime-2",
|
|
"mode": "realtime",
|
|
"transport": "webrtc",
|
|
"brain": "agent-consult",
|
|
],
|
|
"provider": "elevenlabs",
|
|
"resolved": [
|
|
"provider": "elevenlabs",
|
|
"config": [
|
|
"apiKey": "__OPENCLAW_REDACTED__",
|
|
"voiceId": "bIHbv24MWmeRgasZH58o",
|
|
],
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.activeProvider == "elevenlabs")
|
|
#expect(parsed.executionMode == .realtimeWebRTC)
|
|
#expect(parsed.realtimeProvider == "openai")
|
|
#expect(parsed.realtimeModelId == "gpt-realtime-2")
|
|
#expect(parsed.realtimeVoiceId == "cedar")
|
|
#expect(parsed.rawConfigApiKey == "__OPENCLAW_REDACTED__")
|
|
}
|
|
|
|
@Test func `leaves native mode for managed room realtime transport`() {
|
|
let config: [String: Any] = [
|
|
"talk": [
|
|
"realtime": [
|
|
"provider": "openai",
|
|
"mode": "realtime",
|
|
"transport": "managed-room",
|
|
],
|
|
],
|
|
]
|
|
|
|
let parsed = TalkModeGatewayConfigParser.parse(
|
|
config: config,
|
|
defaultProvider: "elevenlabs",
|
|
defaultModelIdFallback: "eleven_v3",
|
|
defaultRealtimeModelIdFallback: "gpt-realtime-2",
|
|
defaultSilenceTimeoutMs: 900)
|
|
|
|
#expect(parsed.executionMode == .native)
|
|
}
|
|
|
|
@Test func `detects PCM format rejection from eleven labs error`() {
|
|
let error = NSError(
|
|
domain: "ElevenLabsTTS",
|
|
code: 403,
|
|
userInfo: [
|
|
NSLocalizedDescriptionKey: "ElevenLabs failed: 403 subscription_required output_format=pcm_44100",
|
|
])
|
|
#expect(TalkModeManager._test_isPCMFormatRejectedByAPI(error))
|
|
}
|
|
|
|
@Test func `ignores generic playback failures for PCM format rejection`() {
|
|
let error = NSError(
|
|
domain: "StreamingAudio",
|
|
code: -1,
|
|
userInfo: [NSLocalizedDescriptionKey: "queue enqueue failed"])
|
|
#expect(TalkModeManager._test_isPCMFormatRejectedByAPI(error) == false)
|
|
}
|
|
}
|