From 26bfa88bb2f0a65b187ce20cbcfcf7c9a7bfeeff Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Jul 2026 01:55:47 -0400 Subject: [PATCH] fix: restore native Talk transcripts, fallback, and microphone access (#115577) * fix(ios): decode frameless Talk events * fix(android): keep browser-only Talk models native * fix(macos): gate dashboard media capture * docs(talk): clarify native platform fallbacks * chore(i18n): refresh native source inventory * fix(android): gate relay entry on provider-level browser-only models too --- apps/.i18n/native-source.json | 104 +++++++-------- .../openclaw/app/GatewayTalkSetupReadiness.kt | 7 + .../app/voice/TalkModeGatewayConfig.kt | 20 +++ .../ai/openclaw/app/voice/TalkModeManager.kt | 38 +++++- .../app/GatewayTalkSetupReadinessTest.kt | 10 ++ .../app/voice/TalkModeConfigParsingTest.kt | 36 ++++++ .../openclaw/app/voice/TalkModeManagerTest.kt | 31 +++++ .../Sources/Voice/TalkModeGatewayConfig.swift | 4 +- .../Voice/TalkRealtimeClientSession.swift | 10 ++ .../Voice/TalkRealtimeWebRTCSession.swift | 120 ++++++++++++------ ...TalkRealtimeServerEventDecodingTests.swift | 27 ++++ .../OpenClaw/DashboardWindowController.swift | 62 +++++++++ .../DashboardGatewaysTests.swift | 19 +++ docs/nodes/talk.md | 4 +- 14 files changed, 399 insertions(+), 93 deletions(-) create mode 100644 apps/ios/Tests/TalkRealtimeServerEventDecodingTests.swift diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index 42a140c257c1..555f35fc4bda 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -451,7 +451,7 @@ }, { "kind": "ui-call", - "line": 106, + "line": 113, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Ready", "surface": "android", @@ -459,7 +459,7 @@ }, { "kind": "ui-call", - "line": 107, + "line": 114, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Needs setup", "surface": "android", @@ -467,7 +467,7 @@ }, { "kind": "ui-call", - "line": 108, + "line": 115, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Unverified", "surface": "android", @@ -475,7 +475,7 @@ }, { "kind": "ui-call", - "line": 116, + "line": 123, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "${state.provider.label} via Gateway relay", "surface": "android", @@ -483,7 +483,7 @@ }, { "kind": "ui-call", - "line": 123, + "line": 130, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Gateway talk catalog not loaded", "surface": "android", @@ -491,7 +491,7 @@ }, { "kind": "ui-call", - "line": 124, + "line": 131, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Could not load Gateway talk catalog", "surface": "android", @@ -499,7 +499,7 @@ }, { "kind": "ui-call", - "line": 126, + "line": 133, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Gateway did not return ${issue.target.title} setup", "surface": "android", @@ -507,7 +507,7 @@ }, { "kind": "ui-call", - "line": 128, + "line": 135, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "No ${issue.target.title} provider is configured on the Gateway", "surface": "android", @@ -515,7 +515,7 @@ }, { "kind": "ui-call", - "line": 130, + "line": 137, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Gateway selected unknown provider ${issue.providerId}", "surface": "android", @@ -523,7 +523,7 @@ }, { "kind": "ui-call", - "line": 132, + "line": 139, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Gateway did not return ${issue.target.title} readiness", "surface": "android", @@ -531,7 +531,7 @@ }, { "kind": "ui-call", - "line": 134, + "line": 141, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Configure a ${issue.target.title} provider on the Gateway", "surface": "android", @@ -539,7 +539,7 @@ }, { "kind": "ui-call", - "line": 136, + "line": 143, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Gateway did not identify the active ${issue.target.title} provider", "surface": "android", @@ -547,7 +547,7 @@ }, { "kind": "ui-call", - "line": 138, + "line": 145, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Choose a supported ${issue.target.title} provider on the Gateway", "surface": "android", @@ -555,7 +555,7 @@ }, { "kind": "ui-call", - "line": 140, + "line": 147, "path": "apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt", "source": "Configure ${issue.providerLabel} on the Gateway", "surface": "android", @@ -14011,15 +14011,7 @@ }, { "kind": "ui-call", - "line": 579, - "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", - "source": "Speech recognizer unavailable", - "surface": "android", - "id": "native.android.1f198e7aceb8411f" - }, - { - "kind": "ui-call", - "line": 1013, + "line": 1019, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Start failed: $message", "surface": "android", @@ -14027,7 +14019,7 @@ }, { "kind": "ui-call", - "line": 1109, + "line": 1115, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Connecting…", "surface": "android", @@ -14035,7 +14027,15 @@ }, { "kind": "ui-call", - "line": 1184, + "line": 1186, + "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", + "source": "Speech recognizer unavailable", + "surface": "android", + "id": "native.android.1f198e7aceb8411f" + }, + { + "kind": "ui-call", + "line": 1218, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Off", "surface": "android", @@ -14043,7 +14043,7 @@ }, { "kind": "ui-call", - "line": 1187, + "line": 1221, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Talk failed: Realtime provider closed unexpectedly.", "surface": "android", @@ -14051,7 +14051,7 @@ }, { "kind": "ui-call", - "line": 1192, + "line": 1226, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Talk failed: Realtime provider closed: $reason", "surface": "android", @@ -14059,7 +14059,7 @@ }, { "kind": "ui-call", - "line": 2317, + "line": 2351, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Thinking…", "surface": "android", @@ -14067,7 +14067,7 @@ }, { "kind": "ui-call", - "line": 2334, + "line": 2368, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Gateway not connected", "surface": "android", @@ -14075,7 +14075,7 @@ }, { "kind": "ui-call", - "line": 2347, + "line": 2381, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Aborted", "surface": "android", @@ -14083,7 +14083,7 @@ }, { "kind": "ui-call", - "line": 2347, + "line": 2381, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Chat error", "surface": "android", @@ -14091,7 +14091,7 @@ }, { "kind": "ui-call", - "line": 2364, + "line": 2398, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "No reply", "surface": "android", @@ -14099,7 +14099,7 @@ }, { "kind": "ui-call", - "line": 2380, + "line": 2414, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Talk failed: $message", "surface": "android", @@ -14107,7 +14107,7 @@ }, { "kind": "ui-call", - "line": 2490, + "line": 2524, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Ready", "surface": "android", @@ -14115,7 +14115,7 @@ }, { "kind": "ui-call", - "line": 2689, + "line": 2723, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Generating voice…", "surface": "android", @@ -14123,7 +14123,7 @@ }, { "kind": "ui-call", - "line": 2717, + "line": 2751, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Speak failed: $message", "surface": "android", @@ -14131,7 +14131,7 @@ }, { "kind": "ui-call", - "line": 2866, + "line": 2900, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Speaking…", "surface": "android", @@ -14139,7 +14139,7 @@ }, { "kind": "ui-call", - "line": 3202, + "line": 3238, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Microphone permission required", "surface": "android", @@ -14147,7 +14147,7 @@ }, { "kind": "ui-call", - "line": 3208, + "line": 3244, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Audio error", "surface": "android", @@ -14155,7 +14155,7 @@ }, { "kind": "ui-call", - "line": 3209, + "line": 3245, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Client error", "surface": "android", @@ -14163,7 +14163,7 @@ }, { "kind": "ui-call", - "line": 3210, + "line": 3246, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Network error", "surface": "android", @@ -14171,7 +14171,7 @@ }, { "kind": "ui-call", - "line": 3211, + "line": 3247, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Network timeout", "surface": "android", @@ -14179,7 +14179,7 @@ }, { "kind": "ui-call", - "line": 3214, + "line": 3250, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Listening", "surface": "android", @@ -14187,7 +14187,7 @@ }, { "kind": "ui-call", - "line": 3214, + "line": 3250, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Listening (PTT)", "surface": "android", @@ -14195,7 +14195,7 @@ }, { "kind": "ui-call", - "line": 3215, + "line": 3251, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Recognizer busy", "surface": "android", @@ -14203,7 +14203,7 @@ }, { "kind": "ui-call", - "line": 3216, + "line": 3252, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Server error", "surface": "android", @@ -14211,7 +14211,7 @@ }, { "kind": "ui-call", - "line": 3217, + "line": 3253, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Speech error ($error)", "surface": "android", @@ -28227,7 +28227,7 @@ }, { "kind": "conditional-branch", - "line": 634, + "line": 605, "path": "apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift", "source": "Asking OpenClaw", "surface": "apple", @@ -28235,7 +28235,7 @@ }, { "kind": "conditional-branch", - "line": 634, + "line": 605, "path": "apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift", "source": "Updating OpenClaw", "surface": "apple", @@ -28243,7 +28243,7 @@ }, { "kind": "conditional-branch", - "line": 723, + "line": 694, "path": "apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift", "source": "Confirmation needed", "surface": "apple", @@ -28251,7 +28251,7 @@ }, { "kind": "conditional-branch", - "line": 723, + "line": 694, "path": "apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift", "source": "OpenClaw unavailable", "surface": "apple", @@ -31819,7 +31819,7 @@ }, { "kind": "conditional-branch", - "line": 905, + "line": 906, "path": "apps/macos/Sources/OpenClaw/DashboardWindowController.swift", "source": "[\\(host)]", "surface": "apple", diff --git a/apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt b/apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt index eac417d7f142..9c06155c4ed7 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt @@ -101,6 +101,13 @@ val GatewayTalkSetupState.isReady: Boolean val GatewayTalkSetupState.requiresSetup: Boolean get() = this is GatewayTalkSetupState.NeedsSetup +internal fun isAndroidRealtimeRelayModelSupported(model: String?): Boolean { + val normalized = model?.trim()?.lowercase() ?: return true + // extensions/openai/realtime-quicksilver.ts makes gpt-live WebRTC-only; + // the Gateway relay rejects it instead of providing a usable Android session. + return normalized != "gpt-live" && !normalized.startsWith("gpt-live-") +} + fun gatewayTalkSetupStatusText(state: GatewayTalkSetupState): String = when (state) { is GatewayTalkSetupState.Ready -> nativeString("Ready") diff --git a/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeGatewayConfig.kt b/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeGatewayConfig.kt index 59a8a7013401..78ad8bfd449e 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeGatewayConfig.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeGatewayConfig.kt @@ -1,5 +1,6 @@ package ai.openclaw.app.voice +import ai.openclaw.app.isAndroidRealtimeRelayModelSupported import ai.openclaw.app.normalizeMainKey import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject @@ -13,18 +14,37 @@ internal data class TalkModeGatewayConfigState( val speechLocale: String?, val interruptOnSpeech: Boolean?, val silenceTimeoutMs: Long, + val realtimeRelayModelSupported: Boolean, ) internal object TalkModeGatewayConfigParser { /** Reads gateway talk/session config into the runtime state TalkMode needs. */ fun parse(config: JsonObject?): TalkModeGatewayConfigState { val talk = config?.get("talk").asObjectOrNull() + // talk.config carries the top-level model (plus voice-model default) in + // realtime.model, but a provider-level providers..model is NOT promoted + // into it — fall back to the selected provider's entry so a gpt-live model + // configured only at provider level still routes Android to native Talk. + val realtime = talk?.get("realtime").asObjectOrNull() + val realtimeProvider = realtime?.get("provider").asStringOrNull() + val realtimeModel = + realtime?.get("model").asStringOrNull() + ?: realtimeProvider?.let { provider -> + realtime + ?.get("providers") + .asObjectOrNull() + ?.get(provider) + .asObjectOrNull() + ?.get("model") + .asStringOrNull() + } val sessionCfg = config?.get("session").asObjectOrNull() return TalkModeGatewayConfigState( mainSessionKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull()), speechLocale = normalizeSpeechLocaleTag(talk?.get("speechLocale").asStringOrNull()), interruptOnSpeech = talk?.get("interruptOnSpeech").asBooleanOrNull(), silenceTimeoutMs = resolvedSilenceTimeoutMs(talk), + realtimeRelayModelSupported = isAndroidRealtimeRelayModelSupported(realtimeModel), ) } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt b/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt index b598fb634547..cdc9de6649b8 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt @@ -330,6 +330,7 @@ class TalkModeManager internal constructor( private var interruptOnSpeech: Boolean = false private var mainSessionKey: String = "main" private var speechLocale: String? = null + private var realtimeRelayModelSupported = true @Volatile private var pendingRunId: String? = null private var pendingFinal: CompletableDeferred? = null @@ -463,6 +464,7 @@ class TalkModeManager internal constructor( silenceWindowMs = TalkDefaults.defaultSilenceTimeoutMs interruptOnSpeech = false speechLocale = null + realtimeRelayModelSupported = true } private suspend fun requestGateway( @@ -1007,7 +1009,11 @@ class TalkModeManager internal constructor( try { ensureConfigLoaded() if (generation != startGeneration.get() || !_isEnabled.value || stopRequested) return@launch - startRealtimeRelay(generation) + if (realtimeRelayModelSupported) { + startRealtimeRelay(generation) + } else { + startNativeTalk(generation) + } } catch (err: Throwable) { if (err is CancellationException) return@launch setStatus(nativeText("Start failed: \$message", err.message ?: err::class.simpleName.orEmpty())) @@ -1162,6 +1168,34 @@ class TalkModeManager internal constructor( Log.d(tag, "realtime session started relaySessionId=$sessionId") } + private suspend fun startNativeTalk(generation: Long) { + if (!isConnected()) { + setStatus(nativeText("Gateway not connected")) + disableRealtimeModeAndNotifyOwner() + return + } + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) { + setStatus(nativeText("Microphone permission required")) + disableRealtimeModeAndNotifyOwner() + return + } + if (!SpeechRecognizer.isRecognitionAvailable(context)) { + setStatus(nativeText("Speech recognizer unavailable")) + disableRealtimeModeAndNotifyOwner() + return + } + withContext(Dispatchers.Main) { + if (generation != startGeneration.get() || !_isEnabled.value || stopRequested) return@withContext + recognizer?.cancel() + recognizer?.destroy() + recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } + startListeningInternal(markListening = true) + } + } + private fun disableRealtimeModeAndNotifyOwner() { if (!_isEnabled.value) return _isEnabled.value = false @@ -3076,12 +3110,14 @@ class TalkModeManager internal constructor( if (generation != gatewayGeneration.get()) return silenceWindowMs = parsed.silenceTimeoutMs speechLocale = parsed.speechLocale + realtimeRelayModelSupported = parsed.realtimeRelayModelSupported parsed.interruptOnSpeech?.let { interruptOnSpeech = it } configLoaded = true } catch (_: Throwable) { if (generation != gatewayGeneration.get()) return silenceWindowMs = TalkDefaults.defaultSilenceTimeoutMs speechLocale = null + realtimeRelayModelSupported = true configLoaded = false } } diff --git a/apps/android/app/src/test/java/ai/openclaw/app/GatewayTalkSetupReadinessTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/GatewayTalkSetupReadinessTest.kt index da06f3c29c94..282ea18c6f2b 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/GatewayTalkSetupReadinessTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/GatewayTalkSetupReadinessTest.kt @@ -4,6 +4,7 @@ import ai.openclaw.app.i18n.NativeText import kotlinx.serialization.json.Json import kotlinx.serialization.json.jsonObject import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test @@ -92,6 +93,15 @@ class GatewayTalkSetupReadinessTest { assertEquals("Deepgram", dictation.provider.label) } + @Test + fun browserOnlyModelsSkipAndroidRealtimeRelay() { + assertFalse(isAndroidRealtimeRelayModelSupported("gpt-live")) + assertFalse(isAndroidRealtimeRelayModelSupported(" GPT-LIVE-future ")) + assertTrue(isAndroidRealtimeRelayModelSupported("gpt-realtime-2.1")) + assertTrue(isAndroidRealtimeRelayModelSupported("gpt-liveness")) + assertTrue(isAndroidRealtimeRelayModelSupported(null)) + } + @Test fun activeProviderAliasSelectsCanonicalCatalogEntry() { val readiness = diff --git a/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeConfigParsingTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeConfigParsingTest.kt index 12dadf4e20b2..7659eabef211 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeConfigParsingTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeConfigParsingTest.kt @@ -5,6 +5,8 @@ import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.put import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue import org.junit.Test class TalkModeConfigParsingTest { @@ -43,6 +45,40 @@ class TalkModeConfigParsingTest { assertEquals(null, realtimeTranscriptionLanguage("fil-PH")) } + @Test + fun gatesAndroidRealtimeRelayFromEffectiveModel() { + val browserOnly = + json + .parseToJsonElement( + """{"talk":{"realtime":{"model":"gpt-live-future"}}}""", + ).jsonObject + val relayCapable = + json + .parseToJsonElement( + """{"talk":{"realtime":{"model":"gpt-realtime-2.1"}}}""", + ).jsonObject + + assertFalse(TalkModeGatewayConfigParser.parse(browserOnly).realtimeRelayModelSupported) + assertTrue(TalkModeGatewayConfigParser.parse(relayCapable).realtimeRelayModelSupported) + } + + @Test + fun gatesAndroidRealtimeRelayFromProviderLevelModel() { + val providerLevelBrowserOnly = + json + .parseToJsonElement( + """{"talk":{"realtime":{"provider":"openai","providers":{"openai":{"model":"gpt-live-1-codex"}}}}}""", + ).jsonObject + val topLevelWins = + json + .parseToJsonElement( + """{"talk":{"realtime":{"provider":"openai","model":"gpt-realtime-2.1","providers":{"openai":{"model":"gpt-live-1-codex"}}}}}""", + ).jsonObject + + assertFalse(TalkModeGatewayConfigParser.parse(providerLevelBrowserOnly).realtimeRelayModelSupported) + assertTrue(TalkModeGatewayConfigParser.parse(topLevelWins).realtimeRelayModelSupported) + } + @Test fun resolvesRealtimeLanguageFromConfigThenWatchThenPhone() { assertEquals( diff --git a/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt index b58b16f0c96c..ecd2a1c667f2 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt @@ -837,6 +837,37 @@ class TalkModeManagerTest { assertTrue(stoppedByRelay.get()) } + @Test + @OptIn(ExperimentalCoroutinesApi::class) + fun browserOnlyRealtimeConfigStartsNativeTalkInsteadOfRelay() = + runTest { + val app = RuntimeEnvironment.getApplication() + shadowOf(app).grantPermissions(Manifest.permission.RECORD_AUDIO) + val packageManager = shadowOf(app.packageManager) + val speechService = ComponentName(app, "TestSpeechRecognitionService") + packageManager.addServiceIfNotPresent(speechService) + packageManager.addIntentFilterForService(speechService, IntentFilter(RecognitionService.SERVICE_INTERFACE)) + Dispatchers.setMain(StandardTestDispatcher(testScheduler)) + val manager = + createManager( + scope = this, + ) + try { + setPrivateField(manager, "configLoaded", true) + setPrivateField(manager, "realtimeRelayModelSupported", false) + manager.setEnabled(true) + advanceUntilIdle() + + assertTrue(manager.isEnabled.value) + assertTrue(manager.isListening.value) + assertNull(readPrivateField(manager, "realtimeSessionId")) + assertEquals("Listening", manager.statusText.value) + } finally { + manager.setEnabled(false) + Dispatchers.resetMain() + } + } + @Test fun textReadyDoesNotEnterSpeakingUntilAudioPlaybackStarts() = runTest { diff --git a/apps/ios/Sources/Voice/TalkModeGatewayConfig.swift b/apps/ios/Sources/Voice/TalkModeGatewayConfig.swift index 7cf33f826176..03d0e918c114 100644 --- a/apps/ios/Sources/Voice/TalkModeGatewayConfig.swift +++ b/apps/ios/Sources/Voice/TalkModeGatewayConfig.swift @@ -314,13 +314,13 @@ enum TalkModeRealtimeVoiceSelection { "alloy", "ash", "ballad", + "cedar", "coral", "echo", + "marin", "sage", "shimmer", "verse", - "marin", - "cedar", ] static func resolvedOverride(_ raw: String?) -> String? { diff --git a/apps/ios/Sources/Voice/TalkRealtimeClientSession.swift b/apps/ios/Sources/Voice/TalkRealtimeClientSession.swift index 26abe556a2fb..9a9f2c16904c 100644 --- a/apps/ios/Sources/Voice/TalkRealtimeClientSession.swift +++ b/apps/ios/Sources/Voice/TalkRealtimeClientSession.swift @@ -57,6 +57,7 @@ struct TalkRealtimeServerEvent: Decodable { let error: TalkRealtimeServerError? let itemId: String? let item: TalkRealtimeServerItem? + let turn: TalkRealtimeServerTurn? let callId: String? let name: String? let delta: String? @@ -69,6 +70,7 @@ struct TalkRealtimeServerEvent: Decodable { case error case itemId = "item_id" case item + case turn case callId = "call_id" case name case delta @@ -103,9 +105,16 @@ struct TalkRealtimeServerError: Decodable { let message: String? } +struct TalkRealtimeServerTurn: Decodable { + let id: String? + let role: String? + let transcript: String? +} + struct TalkRealtimeServerItem: Decodable { let id: String? let type: String? + let text: String? let callId: String? let name: String? let arguments: String? @@ -113,6 +122,7 @@ struct TalkRealtimeServerItem: Decodable { enum CodingKeys: String, CodingKey { case id case type + case text case callId = "call_id" case name case arguments diff --git a/apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift b/apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift index 2b9eb5f65e46..b90f95ac3271 100644 --- a/apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift +++ b/apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift @@ -478,41 +478,10 @@ final class TalkRealtimeWebRTCSession: NSObject { if self.handleRealtimeAudioStateEvent(event) { return } + if self.handleRealtimeTranscriptEvent(event) { + return + } switch event.type { - case "conversation.input_transcript.delta", - "conversation.item.input_audio_transcription.delta": - if !self.loggedFirstServerSpeech { - self.loggedFirstServerSpeech = true - self.trace("server speech/transcript first delta") - } - if let text = event.delta ?? event.transcript { - self.delegate?.realtimeSession(self, didReceiveUserTranscript: text) - } - case "conversation.input_transcript.done", - "conversation.item.input_audio_transcription.completed": - if let text = event.transcript ?? event.text { - self.delegate?.realtimeSession(self, didReceiveUserTranscript: text) - self.recordFinalTranscript(role: .user, text: text) - } - case "conversation.output_transcript.delta", - "response.output_text.delta", - "response.audio_transcript.delta", - "response.output_audio_transcript.delta": - if !self.loggedFirstAssistantSignal { - self.loggedFirstAssistantSignal = true - self.trace("assistant first output signal type=\(event.type)") - } - if let text = event.delta ?? event.transcript ?? event.text { - self.delegate?.realtimeSession(self, didReceiveAssistantTranscript: text) - } - case "conversation.output_transcript.done", - "response.output_text.done", - "response.audio_transcript.done", - "response.output_audio_transcript.done": - if let text = event.transcript ?? event.text { - self.delegate?.realtimeSession(self, didReceiveAssistantTranscript: text) - self.recordFinalTranscript(role: .assistant, text: text) - } case "response.function_call_arguments.delta": self.bufferToolDelta(event) case "response.output_item.added": @@ -535,14 +504,16 @@ final class TalkRealtimeWebRTCSession: NSObject { private func handleRealtimeAudioStateEvent(_ event: TalkRealtimeServerEvent) -> Bool { switch event.type { - case "response.audio.delta", "response.output_audio.delta", "conversation.output_audio.delta": + case "output_audio.delta", "response.audio.delta", "response.output_audio.delta", + "conversation.output_audio.delta": self.markAssistantAudioActive() return true case "response.created": self.trace("response created") self.markAssistantAudioActive() return true - case "response.audio.done", "response.output_audio.done", "conversation.output_audio.done", "response.done": + case "output_audio.done", "response.audio.done", "response.output_audio.done", + "conversation.output_audio.done", "response.done": self.scheduleAssistantAudioFinished() return true case "input_audio_buffer.speech_started": @@ -1210,6 +1181,83 @@ extension TalkRealtimeWebRTCSession { } } +extension TalkRealtimeWebRTCSession { + private func handleRealtimeTranscriptEvent(_ event: TalkRealtimeServerEvent) -> Bool { + switch event.type { + case "input_transcript.added": + if let text = event.item?.text, !text.isEmpty { + self.delegate?.realtimeSession(self, didReceiveUserTranscript: text) + } + case "output_transcript.added": + self.markFirstAssistantSignal(event) + if let text = event.item?.text, !text.isEmpty { + self.delegate?.realtimeSession(self, didReceiveAssistantTranscript: text) + } + case "turn.done": + self.handleFramelessTurnDone(event.turn) + case "conversation.input_transcript.delta", + "conversation.item.input_audio_transcription.delta": + if !self.loggedFirstServerSpeech { + self.loggedFirstServerSpeech = true + self.trace("server speech/transcript first delta") + } + if let text = event.delta ?? event.transcript { + self.delegate?.realtimeSession(self, didReceiveUserTranscript: text) + } + case "conversation.input_transcript.done", + "conversation.item.input_audio_transcription.completed": + if let text = event.transcript ?? event.text { + self.delegate?.realtimeSession(self, didReceiveUserTranscript: text) + self.recordFinalTranscript(role: .user, text: text) + } + case "conversation.output_transcript.delta", + "response.output_text.delta", + "response.audio_transcript.delta", + "response.output_audio_transcript.delta": + self.markFirstAssistantSignal(event) + if let text = event.delta ?? event.transcript ?? event.text { + self.delegate?.realtimeSession(self, didReceiveAssistantTranscript: text) + } + case "conversation.output_transcript.done", + "response.output_text.done", + "response.audio_transcript.done", + "response.output_audio_transcript.done": + if let text = event.transcript ?? event.text { + self.delegate?.realtimeSession(self, didReceiveAssistantTranscript: text) + self.recordFinalTranscript(role: .assistant, text: text) + } + default: + return false + } + return true + } + + private func handleFramelessTurnDone(_ turn: TalkRealtimeServerTurn?) { + guard let turn else { return } + if let text = turn.transcript, !text.isEmpty { + switch turn.role { + case "user": + self.delegate?.realtimeSession(self, didReceiveUserTranscript: text) + self.recordFinalTranscript(role: .user, text: text) + case "assistant": + self.delegate?.realtimeSession(self, didReceiveAssistantTranscript: text) + self.recordFinalTranscript(role: .assistant, text: text) + default: + break + } + } + if turn.role == "assistant" { + self.scheduleAssistantAudioFinished() + } + } + + private func markFirstAssistantSignal(_ event: TalkRealtimeServerEvent) { + guard !self.loggedFirstAssistantSignal else { return } + self.loggedFirstAssistantSignal = true + self.trace("assistant first output signal type=\(event.type)") + } +} + extension TalkRealtimeWebRTCSession: RTCPeerConnectionDelegate { nonisolated func peerConnection(_: RTCPeerConnection, didChange _: RTCSignalingState) {} nonisolated func peerConnection(_: RTCPeerConnection, didAdd stream: RTCMediaStream) { diff --git a/apps/ios/Tests/TalkRealtimeServerEventDecodingTests.swift b/apps/ios/Tests/TalkRealtimeServerEventDecodingTests.swift new file mode 100644 index 000000000000..2f164ccace67 --- /dev/null +++ b/apps/ios/Tests/TalkRealtimeServerEventDecodingTests.swift @@ -0,0 +1,27 @@ +import Foundation +import Testing +@testable import OpenClaw + +struct TalkRealtimeServerEventDecodingTests { + @Test func `decodes frameless realtime transcript and turn events`() throws { + let userDelta = try JSONDecoder().decode( + TalkRealtimeServerEvent.self, + from: Data(#"{"type":"input_transcript.added","item":{"id":"user-1","text":"Hello"}}"#.utf8)) + #expect(userDelta.item?.id == "user-1") + #expect(userDelta.item?.text == "Hello") + + let assistantDelta = try JSONDecoder().decode( + TalkRealtimeServerEvent.self, + from: Data(#"{"type":"output_transcript.added","item":{"id":"assistant-1","text":"Hi"}}"#.utf8)) + #expect(assistantDelta.item?.id == "assistant-1") + #expect(assistantDelta.item?.text == "Hi") + + let turnDone = try JSONDecoder().decode( + TalkRealtimeServerEvent.self, + from: Data( + #"{"type":"turn.done","turn":{"id":"turn-1","role":"assistant","transcript":"Hi there"}}"#.utf8)) + #expect(turnDone.turn?.id == "turn-1") + #expect(turnDone.turn?.role == "assistant") + #expect(turnDone.turn?.transcript == "Hi there") + } +} diff --git a/apps/macos/Sources/OpenClaw/DashboardWindowController.swift b/apps/macos/Sources/OpenClaw/DashboardWindowController.swift index c65cbee1a95c..651e346f5289 100644 --- a/apps/macos/Sources/OpenClaw/DashboardWindowController.swift +++ b/apps/macos/Sources/OpenClaw/DashboardWindowController.swift @@ -1,4 +1,5 @@ import AppKit +import AVFoundation import Foundation import OpenClawKit import WebKit @@ -1039,6 +1040,67 @@ final class DashboardWindowController: NSWindowController, WKNavigationDelegate, } extension DashboardWindowController { + func webView( + _ webView: WKWebView, + requestMediaCapturePermissionFor origin: WKSecurityOrigin, + initiatedByFrame frame: WKFrameInfo, + type: WKMediaCaptureType, + decisionHandler: @escaping @MainActor @Sendable (WKPermissionDecision) -> Void) + { + guard webView === self.webView else { + decisionHandler(.deny) + return + } + let mediaTypes: [AVMediaType] + switch type { + case .camera: + mediaTypes = [.video] + case .microphone: + mediaTypes = [.audio] + case .cameraAndMicrophone: + mediaTypes = [.video, .audio] + @unknown default: + decisionHandler(.deny) + return + } + guard frame.isMainFrame, + Self.isTrustedMediaCaptureOrigin( + protocol: origin.protocol, + host: origin.host, + port: origin.port, + dashboardURL: self.currentURL) + else { + decisionHandler(.prompt) + return + } + let authorized = mediaTypes.allSatisfy { AVCaptureDevice.authorizationStatus(for: $0) == .authorized } + decisionHandler(authorized ? .grant : .prompt) + } + + static func isTrustedMediaCaptureOrigin( + protocol scheme: String, + host: String, + port: Int, + dashboardURL: URL) -> Bool + { + guard scheme.caseInsensitiveCompare(dashboardURL.scheme ?? "") == .orderedSame, + host.caseInsensitiveCompare(dashboardURL.host ?? "") == .orderedSame + else { + return false + } + let requestedPort = port == 0 ? Self.defaultPort(for: scheme) : port + let dashboardPort = dashboardURL.port ?? Self.defaultPort(for: dashboardURL.scheme) + return requestedPort == dashboardPort + } + + private static func defaultPort(for scheme: String?) -> Int? { + switch scheme?.lowercased() { + case "http": 80 + case "https": 443 + default: nil + } + } + static func shouldReloadDashboard( currentURL: URL, newURL: URL, diff --git a/apps/macos/Tests/OpenClawIPCTests/DashboardGatewaysTests.swift b/apps/macos/Tests/OpenClawIPCTests/DashboardGatewaysTests.swift index 6fffb6a9d225..b9f7b2c30230 100644 --- a/apps/macos/Tests/OpenClawIPCTests/DashboardGatewaysTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/DashboardGatewaysTests.swift @@ -111,6 +111,25 @@ struct DashboardGatewaysBridgeTests { port: 443, dashboardURL: url)) } + + @Test func `media capture trust requires the dashboard origin`() throws { + let url = try #require(URL(string: "https://gateway.example/control/")) + #expect(DashboardWindowController.isTrustedMediaCaptureOrigin( + protocol: "https", + host: "gateway.example", + port: 443, + dashboardURL: url)) + #expect(!DashboardWindowController.isTrustedMediaCaptureOrigin( + protocol: "https", + host: "other.example", + port: 443, + dashboardURL: url)) + #expect(!DashboardWindowController.isTrustedMediaCaptureOrigin( + protocol: "http", + host: "gateway.example", + port: 80, + dashboardURL: url)) + } } @Suite(.serialized) diff --git a/docs/nodes/talk.md b/docs/nodes/talk.md index 24fe3440a3d0..15fac8c3f11a 100644 --- a/docs/nodes/talk.md +++ b/docs/nodes/talk.md @@ -9,9 +9,9 @@ title: "Talk mode" Talk mode covers five runtime shapes: - **Native macOS/iOS/Android Talk**: native speech recognition, Gateway chat, and `talk.speak` TTS. Apple Speech recognition on macOS/iOS may use network services; Android behavior depends on the installed speech service. Nodes advertise the `talk` capability and declare which `talk.*` commands they support. -- **iOS Talk (realtime)**: client-owned WebRTC for OpenAI realtime configs that select `webrtc` transport or omit transport. Explicit `gateway-relay`, `provider-websocket`, and non-OpenAI realtime configs stay on the Gateway-owned relay; non-realtime configs use the native speech loop. +- **iOS Talk (realtime)**: client-owned WebRTC for OpenAI realtime configs that select `webrtc` transport or omit transport, including framed and frameless transcript/audio events. Explicit `gateway-relay`, `provider-websocket`, and non-OpenAI realtime configs stay on the Gateway-owned relay; non-realtime configs use the native speech loop. - **Browser Talk**: `talk.client.create` for client-owned `webrtc`/`provider-websocket` sessions, or `talk.session.create` for Gateway-owned `gateway-relay` sessions. `managed-room` is reserved for Gateway handoff and walkie-talkie rooms. -- **Android Talk (realtime)**: Android always uses Gateway-owned relay realtime when `talk.catalog` reports the realtime group ready; it never opens a client-owned WebRTC session. When realtime is not ready, Android stays on native speech recognition, Gateway chat, and `talk.speak`. Caveat: readiness is computed for the configured transport's surface, so a browser-only model such as `gpt-live-*` can read ready while the relay session Android opens is rejected with an explicit error; an Android-side gate is a tracked follow-up. +- **Android Talk (realtime)**: Android uses Gateway-owned relay realtime when `talk.catalog` reports the realtime group ready and the configured model supports relay; it never opens a client-owned WebRTC session. Browser-only `gpt-live-*` models skip relay, so Android stays on native speech recognition, Gateway chat, and `talk.speak` just as it does when realtime is not ready. - **Transcription-only clients**: `talk.session.create({ mode: "transcription", transport: "gateway-relay", brain: "none" })`, then `talk.session.appendAudio`, `talk.session.cancelTurn`, and `talk.session.close` for captions/dictation without an assistant voice response. One-shot uploaded voice notes still use the [media understanding](/nodes/media-understanding) audio path. Native Talk is a continuous loop: listen for speech, send the transcript to the model through the active session, wait for the response, then speak it via the configured Talk provider (`talk.speak`).