diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index e4ed0cc1f2b4..39eb1514a48e 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -139,7 +139,7 @@ }, { "kind": "ui-state-text", - "line": 121, + "line": 125, "path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt", "source": "Searching…", "surface": "android", @@ -147,7 +147,7 @@ }, { "kind": "ui-state-text", - "line": 212, + "line": 216, "path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt", "source": "Mic off", "surface": "android", @@ -155,7 +155,7 @@ }, { "kind": "ui-state-text", - "line": 222, + "line": 226, "path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt", "source": "Off", "surface": "android", @@ -203,7 +203,7 @@ }, { "kind": "conditional-branch", - "line": 2683, + "line": 2701, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Failed: this host requires wss:// or Tailscale Serve. No TLS endpoint detected.", "surface": "android", @@ -211,7 +211,7 @@ }, { "kind": "conditional-branch", - "line": 2685, + "line": 2703, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Failed: secure endpoint reached, but TLS fingerprint verification timed out. Check Tailscale Serve or gateway TLS and retry.", "surface": "android", @@ -219,7 +219,7 @@ }, { "kind": "conditional-branch", - "line": 2687, + "line": 2705, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Failed: couldn't reach the secure gateway endpoint for this host.", "surface": "android", @@ -5763,7 +5763,7 @@ }, { "kind": "ui-call", - "line": 199, + "line": 218, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt", "source": "Type a message…", "surface": "android", @@ -5771,7 +5771,7 @@ }, { "kind": "ui-named-argument", - "line": 209, + "line": 231, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt", "source": "Gateway is offline. Text messages are queued and sent after reconnecting.", "surface": "android", @@ -5779,7 +5779,7 @@ }, { "kind": "ui-named-argument", - "line": 238, + "line": 260, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt", "source": "Select thinking level", "surface": "android", @@ -5787,7 +5787,7 @@ }, { "kind": "ui-named-argument", - "line": 262, + "line": 284, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt", "source": "Attach", "surface": "android", @@ -5795,7 +5795,7 @@ }, { "kind": "ui-named-argument", - "line": 270, + "line": 299, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt", "source": "Refresh", "surface": "android", @@ -5803,7 +5803,7 @@ }, { "kind": "ui-named-argument", - "line": 278, + "line": 307, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt", "source": "Abort", "surface": "android", @@ -5811,7 +5811,7 @@ }, { "kind": "ui-named-argument", - "line": 323, + "line": 356, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt", "source": "Send", "surface": "android", @@ -5819,7 +5819,7 @@ }, { "kind": "ui-named-argument", - "line": 349, + "line": 382, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt", "source": "No commands found", "surface": "android", @@ -5979,7 +5979,7 @@ }, { "kind": "ui-named-argument", - "line": 260, + "line": 261, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Preview · $domain", "surface": "android", @@ -5987,7 +5987,7 @@ }, { "kind": "ui-named-argument", - "line": 269, + "line": 270, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Expand link preview", "surface": "android", @@ -5995,7 +5995,7 @@ }, { "kind": "ui-call", - "line": 293, + "line": 294, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Loading preview…", "surface": "android", @@ -6003,7 +6003,7 @@ }, { "kind": "ui-call", - "line": 294, + "line": 295, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "No preview available", "surface": "android", @@ -6011,7 +6011,7 @@ }, { "kind": "ui-call", - "line": 339, + "line": 340, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Thinking...", "surface": "android", @@ -6019,7 +6019,7 @@ }, { "kind": "ui-named-argument", - "line": 355, + "line": 356, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Tools", "surface": "android", @@ -6027,7 +6027,7 @@ }, { "kind": "ui-call", - "line": 358, + "line": 359, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Running tools...", "surface": "android", @@ -6035,7 +6035,7 @@ }, { "kind": "ui-call", - "line": 361, + "line": 362, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "${display.emoji} ${display.label}", "surface": "android", @@ -6043,7 +6043,7 @@ }, { "kind": "ui-named-argument", - "line": 379, + "line": 380, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "... +${toolCalls.size - 6} more", "surface": "android", @@ -6051,7 +6051,7 @@ }, { "kind": "ui-named-argument", - "line": 410, + "line": 411, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "You", "surface": "android", @@ -6059,7 +6059,7 @@ }, { "kind": "ui-named-argument", - "line": 424, + "line": 425, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Retry", "surface": "android", @@ -6067,7 +6067,7 @@ }, { "kind": "ui-named-argument", - "line": 427, + "line": 428, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Delete", "surface": "android", @@ -6075,7 +6075,7 @@ }, { "kind": "ui-named-argument", - "line": 459, + "line": 460, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "OpenClaw · Live", "surface": "android", @@ -6083,7 +6083,7 @@ }, { "kind": "conditional-branch", - "line": 496, + "line": 497, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "System", "surface": "android", @@ -6091,7 +6091,7 @@ }, { "kind": "conditional-branch", - "line": 497, + "line": 498, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "OpenClaw", "surface": "android", @@ -6099,7 +6099,7 @@ }, { "kind": "ui-call", - "line": 523, + "line": 524, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Unsupported attachment", "surface": "android", @@ -6107,7 +6107,7 @@ }, { "kind": "ui-named-argument", - "line": 276, + "line": 290, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Chat needs attention", "surface": "android", @@ -6115,7 +6115,7 @@ }, { "kind": "ui-named-argument", - "line": 436, + "line": 448, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "All", "surface": "android", @@ -6123,7 +6123,7 @@ }, { "kind": "ui-named-argument", - "line": 494, + "line": 506, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "OpenClaw", "surface": "android", @@ -6131,7 +6131,7 @@ }, { "kind": "ui-named-argument", - "line": 515, + "line": 527, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "New chat", "surface": "android", @@ -6139,7 +6139,7 @@ }, { "kind": "ui-named-argument", - "line": 520, + "line": 532, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "More new chat options", "surface": "android", @@ -6147,7 +6147,7 @@ }, { "kind": "ui-named-argument", - "line": 535, + "line": 547, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Refresh chat", "surface": "android", @@ -6155,7 +6155,7 @@ }, { "kind": "ui-named-argument", - "line": 538, + "line": 550, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Chat", "surface": "android", @@ -6163,7 +6163,7 @@ }, { "kind": "ui-named-argument", - "line": 686, + "line": 698, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Loading session", "surface": "android", @@ -6171,7 +6171,7 @@ }, { "kind": "ui-named-argument", - "line": 713, + "line": 725, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Jump to latest", "surface": "android", @@ -6179,7 +6179,7 @@ }, { "kind": "conditional-branch", - "line": 739, + "line": 751, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Ready when you are", "surface": "android", @@ -6187,7 +6187,7 @@ }, { "kind": "ui-named-argument", - "line": 770, + "line": 782, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Fix connection", "surface": "android", @@ -6195,7 +6195,7 @@ }, { "kind": "ui-named-argument", - "line": 771, + "line": 783, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Copy diagnostics", "surface": "android", @@ -6203,7 +6203,7 @@ }, { "kind": "conditional-branch", - "line": 955, + "line": 967, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Preparing audio…", "surface": "android", @@ -6211,7 +6211,7 @@ }, { "kind": "conditional-branch", - "line": 955, + "line": 967, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Speaking…", "surface": "android", @@ -6219,7 +6219,7 @@ }, { "kind": "ui-named-argument", - "line": 976, + "line": 988, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Tools running", "surface": "android", @@ -6227,7 +6227,7 @@ }, { "kind": "ui-named-argument", - "line": 978, + "line": 990, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "OpenClaw is working", "surface": "android", @@ -6235,7 +6235,7 @@ }, { "kind": "ui-named-argument", - "line": 981, + "line": 993, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "+${toolCalls.size - 4} more", "surface": "android", @@ -6243,7 +6243,7 @@ }, { "kind": "ui-named-argument", - "line": 991, + "line": 1003, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Thinking", "surface": "android", @@ -6251,7 +6251,7 @@ }, { "kind": "ui-named-argument", - "line": 992, + "line": 1004, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "OpenClaw is preparing a response.", "surface": "android", @@ -6259,7 +6259,7 @@ }, { "kind": "ui-named-argument", - "line": 1126, + "line": 1161, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Stop", "surface": "android", @@ -6267,7 +6267,7 @@ }, { "kind": "ui-named-argument", - "line": 1192, + "line": 1227, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Default", "surface": "android", @@ -6275,7 +6275,7 @@ }, { "kind": "conditional-branch", - "line": 1258, + "line": 1293, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Pin model", "surface": "android", @@ -6283,7 +6283,7 @@ }, { "kind": "conditional-branch", - "line": 1258, + "line": 1293, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Unpin model", "surface": "android", @@ -6291,7 +6291,7 @@ }, { "kind": "ui-named-argument", - "line": 1275, + "line": 1310, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "No commands found", "surface": "android", @@ -6299,7 +6299,7 @@ }, { "kind": "ui-named-argument", - "line": 1337, + "line": 1372, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Gateway offline", "surface": "android", @@ -6307,7 +6307,7 @@ }, { "kind": "ui-named-argument", - "line": 1427, + "line": 1464, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Attach image", "surface": "android", @@ -6315,7 +6315,7 @@ }, { "kind": "ui-named-argument", - "line": 1442, + "line": 1483, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Message OpenClaw", "surface": "android", @@ -6323,15 +6323,15 @@ }, { "kind": "ui-named-argument", - "line": 1457, + "line": 1498, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", - "source": "Voice", + "source": "Open voice", "surface": "android", - "id": "native.android.19396391033f4fa8" + "id": "native.android.97602a28629a331f" }, { "kind": "ui-named-argument", - "line": 1495, + "line": 1547, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Remove attachment", "surface": "android", @@ -6339,7 +6339,7 @@ }, { "kind": "conditional-branch", - "line": 1516, + "line": 1568, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Main", "surface": "android", @@ -6347,7 +6347,7 @@ }, { "kind": "ui-named-argument", - "line": 1572, + "line": 1624, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Send", "surface": "android", @@ -6355,7 +6355,7 @@ }, { "kind": "conditional-branch", - "line": 1611, + "line": 1663, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "$contextLabel · ${contextMeterThinkingLabel(thinkingLevel)}", "surface": "android", @@ -6363,12 +6363,52 @@ }, { "kind": "ui-named-argument", - "line": 281, + "line": 291, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatSheetContent.kt", "source": "CHAT ERROR", "surface": "android", "id": "native.android.826804c0189e03d0" }, + { + "kind": "ui-named-argument", + "line": 109, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt", + "source": "Preparing voice note…", + "surface": "android", + "id": "native.android.3df4e24a0fc8810b" + }, + { + "kind": "ui-named-argument", + "line": 129, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt", + "source": "Record voice note", + "surface": "android", + "id": "native.android.06ab95879f652d3e" + }, + { + "kind": "ui-named-argument", + "line": 167, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt", + "source": "Cancel voice note", + "surface": "android", + "id": "native.android.8a7653317387033f" + }, + { + "kind": "ui-named-argument", + "line": 178, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt", + "source": "Finish voice note", + "surface": "android", + "id": "native.android.fb327cd3030aae44" + }, + { + "kind": "ui-named-argument", + "line": 205, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt", + "source": "Voice note", + "surface": "android", + "id": "native.android.e9d238668cb46e71" + }, { "kind": "ui-named-argument", "line": 537, diff --git a/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt b/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt index 944720d2cb07..7affe780ec5e 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt @@ -17,10 +17,12 @@ import ai.openclaw.app.node.SmsManager import ai.openclaw.app.ui.GatewayConnectPlan import ai.openclaw.app.ui.GatewaySavedAuthAction import ai.openclaw.app.voice.VoiceConversationEntry +import android.Manifest import android.app.Application import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.viewModelScope +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow @@ -57,6 +59,8 @@ class MainViewModel( private val gatewayConfigOperationSeq = AtomicLong() private val gatewayConfigOperationMutex = Mutex() + @Volatile private var permissionRequester: PermissionRequester? = null + @Volatile private var foreground = false @Volatile private var runtimeStartupQueued = false @@ -264,6 +268,7 @@ class MainViewModel( runtime.camera.attachLifecycleOwner(owner) runtime.camera.attachPermissionRequester(permissionRequester) runtime.sms.attachPermissionRequester(permissionRequester) + this.permissionRequester = permissionRequester } /** @@ -536,6 +541,23 @@ class MainViewModel( ensureRuntime().setTalkModeEnabled(enabled) } + suspend fun requestVoiceNotePermission(): Boolean { + val requester = permissionRequester ?: return false + return try { + requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO))[Manifest.permission.RECORD_AUDIO] == true + } catch (error: CancellationException) { + throw error + } catch (_: Throwable) { + false + } + } + + internal fun tryAcquireVoiceNoteMic(): Boolean = runtimeRef.value?.tryAcquireVoiceNoteMic() == true + + internal fun releaseVoiceNoteMic() { + runtimeRef.value?.releaseVoiceNoteMic() + } + fun setSpeakerEnabled(enabled: Boolean) { ensureRuntime().setSpeakerEnabled(enabled) } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt b/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt index 5c4b35b1cbf6..72f4cbc1d665 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt @@ -742,6 +742,7 @@ class NodeRuntime private constructor( // Keep ownership epochs and their service/capture state transitions atomic. // Otherwise stale PTT cleanup can pass its epoch check before a UI mode change. private val voiceCaptureOwnershipLock = Any() + private var voiceNoteOwnsMic = false private val voiceCapturePreparationMutex = Mutex() private var didAutoRequestCanvasRehydrate = false @@ -1865,6 +1866,19 @@ class NodeRuntime private constructor( setVoiceCaptureMode(if (value) VoiceCaptureMode.ManualMic else VoiceCaptureMode.Off) } + internal fun tryAcquireVoiceNoteMic(): Boolean = + synchronized(voiceCaptureOwnershipLock) { + if (voiceNoteOwnsMic || !isVoiceCaptureModeActive(VoiceCaptureMode.Off)) return@synchronized false + voiceNoteOwnsMic = true + true + } + + internal fun releaseVoiceNoteMic() { + synchronized(voiceCaptureOwnershipLock) { + voiceNoteOwnsMic = false + } + } + fun cancelMicCapture() { micCapture.cancelMicCapture() setVoiceCaptureMode(VoiceCaptureMode.Off, persistManualMic = false) @@ -2024,6 +2038,9 @@ class NodeRuntime private constructor( ) { throw IllegalStateException("NODE_BACKGROUND_UNAVAILABLE: command requires foreground") } + if (voiceNoteOwnsMic) { + throw IllegalStateException("MIC_BUSY: voice note recording is active") + } if (!hasRecordAudioPermission()) { throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") } @@ -2251,6 +2268,7 @@ class NodeRuntime private constructor( persistManualMic: Boolean = true, ) { synchronized(voiceCaptureOwnershipLock) { + if (mode != VoiceCaptureMode.Off && voiceNoteOwnsMic) return talkPttCommandEpoch.incrementAndGet() voiceCaptureOwnershipEpoch.incrementAndGet() val permissionDenied = mode.requiresMicrophonePermission && !hasRecordAudioPermission() diff --git a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt index fc7b0e13616b..022bf129541b 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt @@ -682,6 +682,7 @@ class ChatController internal constructor( mimeType = att.mimeType, fileName = att.fileName, base64 = att.base64, + durationMs = att.durationMs, ), ) } @@ -2221,29 +2222,60 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? { text = obj["text"].asStringOrNull() ?: obj["content"].asStringOrNull(), ) - "image" -> + "image", "audio" -> ChatMessageContent( - type = "image", + type = obj["type"].asStringOrNull() ?: "image", mimeType = obj["mimeType"].asStringOrNull(), fileName = obj["fileName"].asStringOrNull(), base64 = obj["content"].asStringOrNull()?.takeIf { it.isNotBlank() }, ) + "attachment" -> { + val attachment = obj["attachment"].asObjectOrNull() ?: return null + val mimeType = attachment["mimeType"].asStringOrNull() + if (attachment["kind"].asStringOrNull() != "audio" && mimeType?.startsWith("audio/") != true) return null + ChatMessageContent( + type = "audio", + mimeType = mimeType, + fileName = attachment["label"].asStringOrNull(), + ) + } + else -> null } } internal fun parseChatMessageContents(obj: JsonObject): List { - obj["content"].asArrayOrNull()?.let { content -> - return content.mapNotNull(::parseChatMessageContent) + val content = + obj["content"].asArrayOrNull()?.mapNotNull(::parseChatMessageContent) + ?: obj["content"].asStringOrNull()?.let { listOf(ChatMessageContent(type = "text", text = it)) } + ?: obj["text"].asStringOrNull()?.let { listOf(ChatMessageContent(type = "text", text = it)) } + ?: emptyList() + val transcriptAudio = parseTranscriptAudioContents(obj) + if (transcriptAudio.isEmpty()) return content + return content + + transcriptAudio.filterNot { audio -> + content.any { it.mimeType == audio.mimeType && it.fileName == audio.fileName } + } +} + +private fun parseTranscriptAudioContents(obj: JsonObject): List { + val paths = + obj["MediaPaths"].asArrayOrNull()?.mapNotNull { it.asStringOrNull() } + ?: obj["MediaPath"].asStringOrNull()?.let { listOf(it) } + ?: return emptyList() + val types = + obj["MediaTypes"].asArrayOrNull()?.map { it.asStringOrNull().orEmpty() } + ?: obj["MediaType"].asStringOrNull()?.let { listOf(it) } + ?: emptyList() + return paths.mapIndexedNotNull { index, path -> + val mimeType = types.getOrNull(index)?.takeIf { it.startsWith("audio/") } ?: return@mapIndexedNotNull null + ChatMessageContent( + type = "audio", + mimeType = mimeType, + fileName = path.substringAfterLast('/').takeIf(String::isNotBlank), + ) } - obj["content"].asStringOrNull()?.let { text -> - return listOf(ChatMessageContent(type = "text", text = text)) - } - obj["text"].asStringOrNull()?.let { text -> - return listOf(ChatMessageContent(type = "text", text = text)) - } - return emptyList() } private fun parseCreatedSessionKey( @@ -2330,21 +2362,48 @@ internal fun reconcileMessageIds( ): List { if (previous.isEmpty() || incoming.isEmpty()) return incoming - val idsByKey = LinkedHashMap>() + val messagesByKey = LinkedHashMap>() for (message in previous) { val key = messageIdentityKey(message) ?: continue - idsByKey.getOrPut(key) { ArrayDeque() }.addLast(message.id) + messagesByKey.getOrPut(key) { ArrayDeque() }.addLast(message) } return incoming.map { message -> val key = messageIdentityKey(message) ?: return@map message - val ids = idsByKey[key] ?: return@map message - val reusedId = ids.removeFirstOrNull() ?: return@map message - if (ids.isEmpty()) { - idsByKey.remove(key) + val matches = messagesByKey[key] ?: return@map message + val previousMessage = matches.removeFirstOrNull() ?: return@map message + if (matches.isEmpty()) { + messagesByKey.remove(key) } - if (reusedId == message.id) return@map message - message.copy(id = reusedId) + message.copy( + id = previousMessage.id, + content = preserveOptimisticAudioDuration(previous = previousMessage, incoming = message), + ) + } +} + +private fun preserveOptimisticAudioDuration( + previous: ChatMessage, + incoming: ChatMessage, +): List { + val idempotencyKey = incoming.idempotencyKey?.trim().orEmpty() + if (idempotencyKey.isEmpty() || idempotencyKey != previous.idempotencyKey?.trim()) return incoming.content + + val remainingAudio = + previous.content + .filter { it.mimeType?.startsWith("audio/") == true && it.durationMs != null } + .toMutableList() + if (remainingAudio.isEmpty()) return incoming.content + + return incoming.content.map { part -> + if (part.durationMs != null || part.mimeType?.startsWith("audio/") != true) return@map part + if (remainingAudio.isEmpty()) return@map part + val exactIndex = + remainingAudio.indexOfFirst { + it.mimeType == part.mimeType && it.fileName == part.fileName + } + val match = remainingAudio.removeAt(if (exactIndex >= 0) exactIndex else 0) + part.copy(durationMs = match.durationMs) } } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatModels.kt b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatModels.kt index d5d03cd3a1ac..4ae0e6a355cb 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatModels.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatModels.kt @@ -31,6 +31,7 @@ data class ChatMessageContent( val mimeType: String? = null, val fileName: String? = null, val base64: String? = null, + val durationMs: Long? = null, ) /** @@ -106,4 +107,5 @@ data class OutgoingAttachment( val mimeType: String, val fileName: String, val base64: String, + val durationMs: Long? = null, ) diff --git a/apps/android/app/src/main/java/ai/openclaw/app/chat/VoiceNoteRecorderController.kt b/apps/android/app/src/main/java/ai/openclaw/app/chat/VoiceNoteRecorderController.kt new file mode 100644 index 000000000000..1d553e3e334d --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/app/chat/VoiceNoteRecorderController.kt @@ -0,0 +1,285 @@ +package ai.openclaw.app.chat + +import android.content.Context +import android.media.MediaRecorder +import android.os.SystemClock +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import java.io.File +import java.io.RandomAccessFile +import java.util.UUID + +internal const val VOICE_NOTE_MAX_DURATION_MS = 180_000L +internal const val VOICE_NOTE_MAX_BYTES = 3_500_000L +internal const val VOICE_NOTE_MIME_TYPE = "audio/mp4" + +internal sealed interface VoiceNoteRecorderState { + data object Idle : VoiceNoteRecorderState + + data class Recording( + val startedAtMillis: Long, + ) : VoiceNoteRecorderState + + data object Preparing : VoiceNoteRecorderState + + data class Failure( + val message: String, + ) : VoiceNoteRecorderState +} + +internal data class VoiceNoteRecording( + val file: File, + val durationMs: Long, +) + +internal interface VoiceNoteRecordingEngine { + fun start(outputFile: File) + + fun stop(): Long + + fun cancel() +} + +/** Owns voice-note recording state and temporary-file cleanup. */ +internal class VoiceNoteRecorderController( + private val scope: CoroutineScope, + private val outputDirectory: File, + private val engine: VoiceNoteRecordingEngine, + private val requestPermission: suspend () -> Boolean, + private val acquireMic: () -> Boolean, + private val releaseMic: () -> Unit, + private val onFinished: (VoiceNoteRecording) -> Unit, + private val elapsedRealtimeMillis: () -> Long = SystemClock::elapsedRealtime, +) { + private val lock = Any() + private val _state = MutableStateFlow(VoiceNoteRecorderState.Idle) + val state: StateFlow = _state.asStateFlow() + + private val _elapsedMs = MutableStateFlow(0L) + val elapsedMs: StateFlow = _elapsedMs.asStateFlow() + + private var outputFile: File? = null + private var elapsedJob: Job? = null + private var ownsMic = false + + suspend fun start(): Boolean { + synchronized(lock) { + if (_state.value !is VoiceNoteRecorderState.Idle && _state.value !is VoiceNoteRecorderState.Failure) return false + } + if (!requestPermission()) { + fail("Microphone permission is required to record a voice note.") + return false + } + + return synchronized(lock) { + if (_state.value !is VoiceNoteRecorderState.Idle && _state.value !is VoiceNoteRecorderState.Failure) { + return@synchronized false + } + if (!acquireMic()) { + failLocked("Voice capture is already using the microphone.") + return@synchronized false + } + ownsMic = true + + val startedAt = elapsedRealtimeMillis() + val file = File(outputDirectory, "voice-note-${UUID.randomUUID()}.m4a") + try { + engine.start(file) + } catch (_: Throwable) { + engine.cancel() + releaseMicLocked() + file.delete() + failLocked("Could not start voice-note recording.") + return@synchronized false + } + + outputFile = file + _elapsedMs.value = 0L + _state.value = VoiceNoteRecorderState.Recording(startedAtMillis = startedAt) + startElapsedUpdates(startedAt) + true + } + } + + fun finish(): Boolean { + val recording = + synchronized(lock) { + if (_state.value !is VoiceNoteRecorderState.Recording) return false + val file = outputFile ?: return false + elapsedJob?.cancel() + elapsedJob = null + + val durationMs = + try { + engine.stop().coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS) + } catch (_: Throwable) { + engine.cancel() + file.delete() + finishFailureLocked("Could not finish voice-note recording.") + return false + } + + try { + normalizeM4aContainerBrand(file) + } catch (_: Throwable) { + file.delete() + finishFailureLocked("Could not finish voice-note recording.") + return false + } + + if (file.length() > VOICE_NOTE_MAX_BYTES) { + file.delete() + finishFailureLocked("Voice note is too large. Record a shorter message.") + return false + } + + // The file stays owned by the controller through Preparing: the staging + // coroutine is composition-scoped and may be cancelled before it runs, + // so cancel() must still be able to delete the handed-off recording. + _elapsedMs.value = 0L + _state.value = VoiceNoteRecorderState.Preparing + releaseMicLocked() + VoiceNoteRecording(file = file, durationMs = durationMs) + } + onFinished(recording) + return true + } + + fun completePreparation() { + synchronized(lock) { + if (_state.value is VoiceNoteRecorderState.Preparing) { + outputFile = null + _state.value = VoiceNoteRecorderState.Idle + } + } + } + + fun cancel() { + synchronized(lock) { + elapsedJob?.cancel() + elapsedJob = null + if (_state.value is VoiceNoteRecorderState.Recording) { + engine.cancel() + } + releaseMicLocked() + outputFile?.delete() + outputFile = null + _elapsedMs.value = 0L + _state.value = VoiceNoteRecorderState.Idle + } + } + + fun reportFailure(message: String) { + synchronized(lock) { + outputFile?.delete() + outputFile = null + failLocked(message) + } + } + + private fun startElapsedUpdates(startedAt: Long) { + elapsedJob?.cancel() + elapsedJob = + scope.launch { + while (isActive && state.value is VoiceNoteRecorderState.Recording) { + val elapsed = (elapsedRealtimeMillis() - startedAt).coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS) + _elapsedMs.value = elapsed + // MediaRecorder's duration callback races its asynchronous auto-stop. + // Own the cap here so every successful finish calls stop() exactly once. + if (elapsed >= VOICE_NOTE_MAX_DURATION_MS) { + finish() + return@launch + } + delay(250L) + } + } + } + + private fun fail(message: String) { + synchronized(lock) { failLocked(message) } + } + + private fun failLocked(message: String) { + _state.value = VoiceNoteRecorderState.Failure(message) + } + + private fun finishFailureLocked(message: String) { + releaseMicLocked() + outputFile = null + _elapsedMs.value = 0L + _state.value = VoiceNoteRecorderState.Failure(message) + } + + private fun releaseMicLocked() { + if (!ownsMic) return + ownsMic = false + releaseMic() + } +} + +/** Marks AAC-only MPEG-4 output as audio so gateway byte sniffing cannot classify it as video. */ +internal fun normalizeM4aContainerBrand(file: File) { + RandomAccessFile(file, "rw").use { output -> + if (output.length() < 12L) return + output.seek(4L) + val boxType = ByteArray(4) + output.readFully(boxType) + if (!boxType.contentEquals("ftyp".toByteArray(Charsets.US_ASCII))) return + output.seek(8L) + output.write("M4A ".toByteArray(Charsets.US_ASCII)) + } +} + +/** Android AAC/m4a engine kept behind [VoiceNoteRecordingEngine] for JVM tests. */ +internal class AndroidVoiceNoteRecordingEngine( + private val context: Context, + private val elapsedRealtime: () -> Long = SystemClock::elapsedRealtime, +) : VoiceNoteRecordingEngine { + private var recorder: MediaRecorder? = null + private var startedAtElapsedMs = 0L + + override fun start(outputFile: File) { + check(recorder == null) + val next = MediaRecorder(context) + try { + next.setAudioSource(MediaRecorder.AudioSource.MIC) + next.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4) + next.setAudioEncoder(MediaRecorder.AudioEncoder.AAC) + next.setAudioChannels(1) + next.setAudioEncodingBitRate(32_000) + next.setAudioSamplingRate(44_100) + next.setOutputFile(outputFile.absolutePath) + next.prepare() + next.start() + startedAtElapsedMs = elapsedRealtime() + recorder = next + } catch (error: Throwable) { + next.release() + throw error + } + } + + override fun stop(): Long { + val active = checkNotNull(recorder) + recorder = null + return try { + active.stop() + (elapsedRealtime() - startedAtElapsedMs).coerceAtLeast(0L) + } finally { + active.release() + } + } + + override fun cancel() { + val active = recorder ?: return + recorder = null + runCatching { active.stop() } + active.release() + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt index da15060c7a24..09b933afe43a 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt @@ -3,6 +3,7 @@ package ai.openclaw.app.ui.chat import ai.openclaw.app.ChatDraft import ai.openclaw.app.ChatDraftPlacement import ai.openclaw.app.chat.ChatCommandEntry +import ai.openclaw.app.chat.VoiceNoteRecorderState import ai.openclaw.app.ui.mobileAccent import ai.openclaw.app.ui.mobileAccentBorderStrong import ai.openclaw.app.ui.mobileAccentSoft @@ -35,6 +36,7 @@ import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.filled.Send import androidx.compose.material.icons.filled.ArrowDropDown import androidx.compose.material.icons.filled.AttachFile +import androidx.compose.material.icons.filled.Mic import androidx.compose.material.icons.filled.Refresh import androidx.compose.material.icons.filled.Stop import androidx.compose.material3.Button @@ -130,16 +132,22 @@ internal fun applyDraftText( /** Chat input surface for text, image attachments, thinking level, and run controls. */ @Composable -fun ChatComposer( +internal fun ChatComposer( draftText: ChatDraft?, healthOk: Boolean, thinkingLevel: String, pendingRunCount: Int, commands: List, - attachments: List, + attachments: List, onDraftApplied: () -> Unit, onPickImages: () -> Unit, onRemoveAttachment: (id: String) -> Unit, + voiceNoteState: VoiceNoteRecorderState, + voiceNoteElapsedMs: Long, + recordVoiceNoteEnabled: Boolean, + onStartVoiceNote: () -> Unit, + onCancelVoiceNote: () -> Unit, + onFinishVoiceNote: () -> Unit, onSetThinkingLevel: (level: String) -> Unit, onRefresh: () -> Unit, onAbort: () -> Unit, @@ -176,6 +184,8 @@ fun ChatComposer( input.trim().isNotEmpty() && attachments.isEmpty() } val sendBusy = pendingRunCount > 0 + val recordingVoiceNote = voiceNoteState is VoiceNoteRecorderState.Recording + val preparingVoiceNote = voiceNoteState is VoiceNoteRecorderState.Preparing Column(modifier = Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(8.dp)) { if (attachments.isNotEmpty()) { @@ -192,17 +202,29 @@ fun ChatComposer( ) } - OutlinedTextField( - value = input, - onValueChange = { input = it }, - modifier = Modifier.fillMaxWidth(), - placeholder = { Text("Type a message…", style = mobileBodyStyle(), color = mobileTextTertiary) }, - minLines = 2, - maxLines = 5, - textStyle = mobileBodyStyle().copy(color = mobileText), - shape = RoundedCornerShape(14.dp), - colors = chatTextFieldColors(), - ) + if (recordingVoiceNote) { + VoiceNoteRecordingControls( + elapsedMs = voiceNoteElapsedMs, + onCancel = onCancelVoiceNote, + onDone = onFinishVoiceNote, + ) + } else if (preparingVoiceNote) { + VoiceNotePreparing() + } else { + OutlinedTextField( + value = input, + onValueChange = { input = it }, + modifier = Modifier.fillMaxWidth(), + placeholder = { Text("Type a message…", style = mobileBodyStyle(), color = mobileTextTertiary) }, + minLines = 2, + maxLines = 5, + textStyle = mobileBodyStyle().copy(color = mobileText), + shape = RoundedCornerShape(14.dp), + colors = chatTextFieldColors(), + ) + } + + VoiceNoteRecorderError(voiceNoteState) if (!healthOk) { Text( @@ -266,6 +288,13 @@ fun ChatComposer( onClick = onPickImages, ) + if (!recordingVoiceNote) { + VoiceNoteRecordButton( + enabled = recordVoiceNoteEnabled && !preparingVoiceNote, + onClick = onStartVoiceNote, + ) + } + SecondaryActionButton( label = "Refresh", icon = Icons.Default.Refresh, @@ -300,7 +329,7 @@ fun ChatComposer( } } }, - enabled = canSend, + enabled = canSend && !recordingVoiceNote && !preparingVoiceNote, modifier = Modifier.height(44.dp), shape = RoundedCornerShape(14.dp), contentPadding = PaddingValues(horizontal = 20.dp), @@ -311,7 +340,11 @@ fun ChatComposer( disabledContainerColor = mobileBorderStrong, disabledContentColor = mobileTextTertiary, ), - border = BorderStroke(1.dp, if (canSend) mobileAccentBorderStrong else mobileBorderStrong), + border = + BorderStroke( + 1.dp, + if (canSend && !recordingVoiceNote && !preparingVoiceNote) mobileAccentBorderStrong else mobileBorderStrong, + ), ) { if (sendBusy) { CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp, color = Color.White) @@ -456,7 +489,7 @@ private fun thinkingLabel(raw: String): String = @Composable private fun AttachmentsStrip( - attachments: List, + attachments: List, onRemoveAttachment: (id: String) -> Unit, ) { Row( @@ -465,7 +498,7 @@ private fun AttachmentsStrip( ) { for (att in attachments) { AttachmentChip( - fileName = att.fileName, + attachment = att, onRemove = { onRemoveAttachment(att.id) }, ) } @@ -474,7 +507,7 @@ private fun AttachmentsStrip( @Composable private fun AttachmentChip( - fileName: String, + attachment: PendingAttachment, onRemove: () -> Unit, ) { Surface( @@ -487,8 +520,13 @@ private fun AttachmentChip( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp), ) { + if (attachment.mimeType.startsWith("audio/")) { + Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(14.dp), tint = mobileTextSecondary) + } Text( - text = fileName, + text = + attachment.durationMs?.let { duration -> "Voice note · ${formatVoiceNoteDuration(duration)}" } + ?: attachment.fileName, style = mobileCaption1, color = mobileText, maxLines = 1, diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatImageCodec.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatImageCodec.kt index 4adf80cc06d8..666f849a7d10 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatImageCodec.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatImageCodec.kt @@ -30,7 +30,7 @@ private val decodedBitmapCache = internal fun loadSizedImageAttachment( resolver: ContentResolver, uri: Uri, -): PendingImageAttachment { +): PendingAttachment { val fileName = normalizeAttachmentFileName((uri.lastPathSegment ?: "image").substringAfterLast('/')) val bitmap = decodeScaledBitmap(resolver, uri, maxDimension = CHAT_ATTACHMENT_MAX_WIDTH) if (bitmap == null) { @@ -67,7 +67,7 @@ internal fun loadSizedImageAttachment( }, ) val base64 = Base64.encodeToString(encoded.bytes, Base64.NO_WRAP) - return PendingImageAttachment( + return PendingAttachment( id = uri.toString() + "#" + System.currentTimeMillis().toString(), fileName = fileName, mimeType = "image/jpeg", diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt index f27c5f4f9391..53535933c40e 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt @@ -89,7 +89,7 @@ internal fun ChatMessageBubble( when (part.type) { "text" -> !part.text.isNullOrBlank() "image" -> !part.base64.isNullOrBlank() - else -> false + else -> part.isAudioAttachment() } } @@ -200,11 +200,12 @@ private fun ChatMessageBody( ) { Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { for (part in content) { - when (part.type) { - "text" -> { + when { + part.type == "text" -> { val text = part.text ?: continue ChatMarkdown(text = text, textColor = textColor) } + part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs) else -> { val b64 = part.base64 ?: continue ChatBase64Image(base64 = b64, mimeType = part.mimeType) diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt index 9fdea4668051..f0a536c5c3b6 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt @@ -11,7 +11,7 @@ import ai.openclaw.app.chat.ChatPendingToolCall import ai.openclaw.app.chat.ChatSessionEntry import ai.openclaw.app.chat.MessageSpeechPhase import ai.openclaw.app.chat.MessageSpeechState -import ai.openclaw.app.chat.OutgoingAttachment +import ai.openclaw.app.chat.VoiceNoteRecorderState import ai.openclaw.app.resolveAgentIdFromMainSessionKey import ai.openclaw.app.ui.copyGatewayDiagnosticsReport import ai.openclaw.app.ui.design.ClawListItem @@ -58,6 +58,7 @@ import androidx.compose.material.icons.filled.AttachFile import androidx.compose.material.icons.filled.Close import androidx.compose.material.icons.filled.Cloud import androidx.compose.material.icons.filled.ContentCopy +import androidx.compose.material.icons.filled.GraphicEq import androidx.compose.material.icons.filled.HourglassEmpty import androidx.compose.material.icons.filled.Mic import androidx.compose.material.icons.filled.MoreHoriz @@ -139,6 +140,11 @@ fun ChatScreen( val modelFavorites by viewModel.modelFavorites.collectAsState() val modelRecents by viewModel.modelRecents.collectAsState() val selectedModelRef by viewModel.chatSelectedModelRef.collectAsState() + val micEnabled by viewModel.micEnabled.collectAsState() + val micIsListening by viewModel.micIsListening.collectAsState() + val micCooldown by viewModel.micCooldown.collectAsState() + val talkModeEnabled by viewModel.talkModeEnabled.collectAsState() + val talkModeListening by viewModel.talkModeListening.collectAsState() val contextUsage = resolveChatContextUsage(sessionKey = sessionKey, mainSessionKey = mainSessionKey, sessions = sessions) val gatewayAddress = gatewayDiagnosticsEndpoint(remoteAddress = remoteAddress, manualHost = manualHost, manualPort = manualPort, manualTls = manualTls) val gatewayProblemMessage = gatewayConnectionDisplay.problem?.message?.takeIf { it.isNotBlank() } @@ -149,7 +155,7 @@ fun ChatScreen( val context = LocalContext.current val resolver = context.contentResolver val scope = rememberCoroutineScope() - val attachments = remember { mutableStateListOf() } + val attachments = remember { mutableStateListOf() } var showModelPicker by rememberSaveable { mutableStateOf(false) } DisposableEffect(viewModel) { @@ -168,6 +174,14 @@ fun ChatScreen( modelCatalog.firstOrNull { it.providerQualifiedRef() == selected }?.name?.takeIf { it.isNotBlank() } ?: selected.substringAfterLast('/') } ?: "Model" + val micCaptureActive = micEnabled || micIsListening || micCooldown || talkModeEnabled || talkModeListening + val voiceNoteRecorder = + rememberVoiceNoteRecorderController( + viewModel = viewModel, + onFinished = attachments::add, + ) + val voiceNoteState by voiceNoteRecorder.state.collectAsState() + val voiceNoteElapsedMs by voiceNoteRecorder.elapsedMs.collectAsState() val pickImages = rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris -> if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult @@ -319,6 +333,12 @@ fun ChatScreen( onOpenModelPicker = { showModelPicker = true }, onPickImages = { pickImages.launch("image/*") }, onRemoveAttachment = { id -> attachments.removeAll { it.id == id } }, + voiceNoteState = voiceNoteState, + voiceNoteElapsedMs = voiceNoteElapsedMs, + recordVoiceNoteEnabled = pendingRunCount == 0 && !micCaptureActive, + onStartVoiceNote = { scope.launch { voiceNoteRecorder.start() } }, + onCancelVoiceNote = voiceNoteRecorder::cancel, + onFinishVoiceNote = voiceNoteRecorder::finish, onVoice = onVoice, onFixConnection = onOpenGatewaySettings, onCopyDiagnostics = { @@ -333,15 +353,7 @@ fun ChatScreen( onSend = { val message = input.trim() if (message.isEmpty() && attachments.isEmpty()) return@ChatComposer - val outgoing = - attachments.map { attachment -> - OutgoingAttachment( - type = "image", - mimeType = attachment.mimeType, - fileName = attachment.fileName, - base64 = attachment.base64, - ) - } + val outgoing = attachments.map(PendingAttachment::toOutgoingAttachment) val pendingAttachments = attachments.toList() input = "" attachments.clear() @@ -847,7 +859,7 @@ private fun ChatBubble( when (part.type) { "text" -> !part.text.isNullOrBlank() "image" -> !part.base64.isNullOrBlank() - else -> false + else -> part.isAudioAttachment() } } if (displayableContent.isEmpty()) return @@ -896,10 +908,10 @@ private fun ChatBubble( color = ClawTheme.colors.text, ) displayableContent.forEach { part -> - if (part.type == "text") { - ChatText(text = part.text.orEmpty(), textColor = ClawTheme.colors.text, isStreaming = live) - } else { - Text(text = part.fileName ?: "Attachment", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted) + when { + part.type == "text" -> ChatText(text = part.text.orEmpty(), textColor = ClawTheme.colors.text, isStreaming = live) + part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs) + else -> Text(text = part.fileName ?: "Attachment", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted) } } if (messageId != null) { @@ -1024,7 +1036,7 @@ private fun ChatNotice( private fun ChatComposer( value: String, onValueChange: (String) -> Unit, - attachments: List, + attachments: List, thinkingLevel: String, contextUsage: ChatContextUsage, selectedModelLabel: String, @@ -1038,6 +1050,12 @@ private fun ChatComposer( onOpenModelPicker: () -> Unit, onPickImages: () -> Unit, onRemoveAttachment: (String) -> Unit, + voiceNoteState: VoiceNoteRecorderState, + voiceNoteElapsedMs: Long, + recordVoiceNoteEnabled: Boolean, + onStartVoiceNote: () -> Unit, + onCancelVoiceNote: () -> Unit, + onFinishVoiceNote: () -> Unit, onVoice: () -> Unit, onFixConnection: () -> Unit, onCopyDiagnostics: () -> Unit, @@ -1080,17 +1098,32 @@ private fun ChatComposer( } Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(6.dp)) { - ChatInputPill( - value = value, - onValueChange = onValueChange, - onPickImages = onPickImages, - onVoice = onVoice, - modifier = Modifier.weight(1f), - ) + if (voiceNoteState is VoiceNoteRecorderState.Recording) { + VoiceNoteRecordingControls( + elapsedMs = voiceNoteElapsedMs, + onCancel = onCancelVoiceNote, + onDone = onFinishVoiceNote, + modifier = Modifier.weight(1f), + ) + } else if (voiceNoteState is VoiceNoteRecorderState.Preparing) { + VoiceNotePreparing(modifier = Modifier.weight(1f)) + } else { + ChatInputPill( + value = value, + onValueChange = onValueChange, + onPickImages = onPickImages, + onStartVoiceNote = onStartVoiceNote, + recordVoiceNoteEnabled = recordVoiceNoteEnabled, + onVoice = onVoice, + modifier = Modifier.weight(1f), + ) + } SendButton( // Offline, only text sends are enabled: they queue durably (text-only v1). enabled = - pendingRunCount == 0 && + voiceNoteState !is VoiceNoteRecorderState.Recording && + voiceNoteState !is VoiceNoteRecorderState.Preparing && + pendingRunCount == 0 && if (healthOk) { value.trim().isNotEmpty() || attachments.isNotEmpty() } else { @@ -1100,6 +1133,8 @@ private fun ChatComposer( ) } + VoiceNoteRecorderError(voiceNoteState) + if (!healthOk && gatewayOffline) { ChatOfflineNotice( status = offlineStatus, @@ -1407,6 +1442,8 @@ private fun ChatInputPill( value: String, onValueChange: (String) -> Unit, onPickImages: () -> Unit, + onStartVoiceNote: () -> Unit, + recordVoiceNoteEnabled: Boolean, onVoice: () -> Unit, modifier: Modifier = Modifier, ) { @@ -1427,6 +1464,10 @@ private fun ChatInputPill( Icon(imageVector = Icons.Default.AttachFile, contentDescription = "Attach image", modifier = Modifier.size(16.dp)) } } + VoiceNoteRecordButton( + enabled = recordVoiceNoteEnabled, + onClick = onStartVoiceNote, + ) Box(modifier = Modifier.weight(1f)) { BasicTextField( value = value, @@ -1454,7 +1495,7 @@ private fun ChatInputPill( contentColor = ClawTheme.colors.text, ) { Box(contentAlignment = Alignment.Center) { - Icon(imageVector = Icons.Default.Mic, contentDescription = "Voice", modifier = Modifier.size(18.dp)) + Icon(imageVector = Icons.Default.GraphicEq, contentDescription = "Open voice", modifier = Modifier.size(18.dp)) } } } @@ -1463,19 +1504,19 @@ private fun ChatInputPill( @Composable private fun AttachmentStrip( - attachments: List, + attachments: List, onRemoveAttachment: (String) -> Unit, ) { Row(modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), horizontalArrangement = Arrangement.spacedBy(6.dp)) { attachments.forEach { attachment -> - AttachmentChip(fileName = attachment.fileName, onRemove = { onRemoveAttachment(attachment.id) }) + AttachmentChip(attachment = attachment, onRemove = { onRemoveAttachment(attachment.id) }) } } } @Composable private fun AttachmentChip( - fileName: String, + attachment: PendingAttachment, onRemove: () -> Unit, ) { Surface( @@ -1489,7 +1530,18 @@ private fun AttachmentChip( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(6.dp), ) { - Text(text = fileName, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted, maxLines = 1, overflow = TextOverflow.Ellipsis) + if (attachment.mimeType.startsWith("audio/")) { + Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(14.dp), tint = ClawTheme.colors.textMuted) + } + Text( + text = + attachment.durationMs?.let { duration -> "Voice note · ${formatVoiceNoteDuration(duration)}" } + ?: attachment.fileName, + style = ClawTheme.type.caption, + color = ClawTheme.colors.textMuted, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + ) Surface(onClick = onRemove, modifier = Modifier.size(22.dp), shape = CircleShape, color = ClawTheme.colors.canvas, contentColor = ClawTheme.colors.text) { Box(contentAlignment = Alignment.Center) { Icon(imageVector = Icons.Default.Close, contentDescription = "Remove attachment", modifier = Modifier.size(13.dp)) diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatSheetContent.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatSheetContent.kt index 16c0f0c855c8..96f7ffa761d7 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatSheetContent.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatSheetContent.kt @@ -2,7 +2,6 @@ package ai.openclaw.app.ui.chat import ai.openclaw.app.MainViewModel import ai.openclaw.app.chat.ChatSessionEntry -import ai.openclaw.app.chat.OutgoingAttachment import ai.openclaw.app.ui.mobileAccent import ai.openclaw.app.ui.mobileAccentBorderStrong import ai.openclaw.app.ui.mobileBorderStrong @@ -91,6 +90,11 @@ fun ChatSheetContent(viewModel: MainViewModel) { val messageSpeechState by viewModel.chatMessageSpeech.collectAsState() val gatewayConnectionDisplay by viewModel.gatewayConnectionDisplay.collectAsState() val gatewayOffline = !gatewayConnectionDisplay.isConnected + val micEnabled by viewModel.micEnabled.collectAsState() + val micIsListening by viewModel.micIsListening.collectAsState() + val micCooldown by viewModel.micCooldown.collectAsState() + val talkModeEnabled by viewModel.talkModeEnabled.collectAsState() + val talkModeListening by viewModel.talkModeListening.collectAsState() DisposableEffect(viewModel) { onDispose(viewModel::stopChatMessageSpeech) @@ -124,7 +128,15 @@ fun ChatSheetContent(viewModel: MainViewModel) { val resolver = context.contentResolver val scope = rememberCoroutineScope() - val attachments = remember { mutableStateListOf() } + val attachments = remember { mutableStateListOf() } + val micCaptureActive = micEnabled || micIsListening || micCooldown || talkModeEnabled || talkModeListening + val voiceNoteRecorder = + rememberVoiceNoteRecorderController( + viewModel = viewModel, + onFinished = attachments::add, + ) + val voiceNoteState by voiceNoteRecorder.state.collectAsState() + val voiceNoteElapsedMs by voiceNoteRecorder.elapsedMs.collectAsState() val pickImages = rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris -> @@ -198,6 +210,12 @@ fun ChatSheetContent(viewModel: MainViewModel) { onDraftApplied = viewModel::clearChatDraft, onPickImages = { pickImages.launch("image/*") }, onRemoveAttachment = { id -> attachments.removeAll { it.id == id } }, + voiceNoteState = voiceNoteState, + voiceNoteElapsedMs = voiceNoteElapsedMs, + recordVoiceNoteEnabled = pendingRunCount == 0 && !micCaptureActive, + onStartVoiceNote = { scope.launch { voiceNoteRecorder.start() } }, + onCancelVoiceNote = voiceNoteRecorder::cancel, + onFinishVoiceNote = voiceNoteRecorder::finish, onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) }, onRefresh = { viewModel.refreshChat() @@ -206,15 +224,7 @@ fun ChatSheetContent(viewModel: MainViewModel) { }, onAbort = { viewModel.abortChat() }, onSend = { text -> - val outgoing = - attachments.map { att -> - OutgoingAttachment( - type = "image", - mimeType = att.mimeType, - fileName = att.fileName, - base64 = att.base64, - ) - } + val outgoing = attachments.map(PendingAttachment::toOutgoingAttachment) val pendingAttachments = attachments.toList() attachments.clear() val accepted = viewModel.sendChatAwaitAcceptance(message = text, thinking = thinkingLevel, attachments = outgoing) @@ -286,13 +296,3 @@ private fun ChatErrorRail(errorText: String) { } } } - -/** - * Image selected in the composer and held in memory until the next chat.send call. - */ -data class PendingImageAttachment( - val id: String, - val fileName: String, - val mimeType: String, - val base64: String, -) diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatTimeline.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatTimeline.kt index 514846380aaf..2836ef7d4551 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatTimeline.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatTimeline.kt @@ -124,6 +124,8 @@ private fun stableMessageVersion(message: ChatMessage): String { append(content.fileName.orEmpty()) append(',') append(content.base64?.length ?: 0) + append(',') + append(content.durationMs ?: "") } } } @@ -163,6 +165,8 @@ private fun latestContentVersion( append(content.fileName.orEmpty()) append(',') append(content.base64?.length ?: 0) + append(',') + append(content.durationMs ?: "") } append(":runs=") append(pendingRunCount) diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/PendingAttachment.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/PendingAttachment.kt new file mode 100644 index 000000000000..c96cef6246f1 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/PendingAttachment.kt @@ -0,0 +1,51 @@ +package ai.openclaw.app.ui.chat + +import ai.openclaw.app.chat.OutgoingAttachment +import ai.openclaw.app.chat.VOICE_NOTE_MIME_TYPE +import ai.openclaw.app.chat.VoiceNoteRecording +import java.util.Base64 + +/** Attachment staged in a composer until the next chat.send call. */ +data class PendingAttachment( + val id: String, + val fileName: String, + val mimeType: String, + val base64: String, + val durationMs: Long? = null, +) + +internal fun PendingAttachment.toOutgoingAttachment(): OutgoingAttachment = + OutgoingAttachment( + type = attachmentTypeForMimeType(mimeType), + mimeType = mimeType, + fileName = fileName, + base64 = base64, + durationMs = durationMs, + ) + +internal fun attachmentTypeForMimeType(mimeType: String): String = + when { + mimeType.startsWith("audio/") -> "audio" + mimeType.startsWith("image/") -> "image" + else -> "file" + } + +internal fun stageVoiceNoteAttachment(recording: VoiceNoteRecording): PendingAttachment = + try { + PendingAttachment( + id = recording.file.absolutePath + "#" + recording.durationMs, + fileName = recording.file.name, + mimeType = VOICE_NOTE_MIME_TYPE, + base64 = Base64.getEncoder().encodeToString(recording.file.readBytes()), + durationMs = recording.durationMs, + ) + } finally { + recording.file.delete() + } + +internal fun formatVoiceNoteDuration(durationMs: Long): String { + val totalSeconds = durationMs.coerceAtLeast(0L) / 1_000L + val minutes = totalSeconds / 60L + val seconds = totalSeconds % 60L + return "$minutes:${seconds.toString().padStart(2, '0')}" +} diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt new file mode 100644 index 000000000000..4036876ea2e1 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt @@ -0,0 +1,214 @@ +package ai.openclaw.app.ui.chat + +import ai.openclaw.app.MainViewModel +import ai.openclaw.app.chat.AndroidVoiceNoteRecordingEngine +import ai.openclaw.app.chat.ChatMessageContent +import ai.openclaw.app.chat.VoiceNoteRecorderController +import ai.openclaw.app.chat.VoiceNoteRecorderState +import ai.openclaw.app.ui.design.ClawTheme +import androidx.compose.foundation.BorderStroke +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Check +import androidx.compose.material.icons.filled.Close +import androidx.compose.material.icons.filled.Mic +import androidx.compose.material3.Icon +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.rememberUpdatedState +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleEventObserver +import androidx.lifecycle.compose.LocalLifecycleOwner +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +@Composable +internal fun rememberVoiceNoteRecorderController( + viewModel: MainViewModel, + onFinished: (PendingAttachment) -> Unit, +): VoiceNoteRecorderController { + val context = LocalContext.current.applicationContext + val lifecycleOwner = LocalLifecycleOwner.current + val scope = rememberCoroutineScope() + val currentOnFinished by rememberUpdatedState(onFinished) + lateinit var controller: VoiceNoteRecorderController + controller = + remember(context, viewModel, scope) { + VoiceNoteRecorderController( + scope = scope, + outputDirectory = context.cacheDir, + engine = AndroidVoiceNoteRecordingEngine(context), + requestPermission = viewModel::requestVoiceNotePermission, + acquireMic = viewModel::tryAcquireVoiceNoteMic, + releaseMic = viewModel::releaseVoiceNoteMic, + onFinished = { recording -> + scope.launch(Dispatchers.IO) { + val attachment = runCatching { stageVoiceNoteAttachment(recording) } + withContext(Dispatchers.Main) { + attachment.fold( + onSuccess = { + currentOnFinished(it) + controller.completePreparation() + }, + onFailure = { controller.reportFailure("Could not prepare voice note.") }, + ) + } + } + }, + ) + } + DisposableEffect(controller, lifecycleOwner) { + val observer = + LifecycleEventObserver { _, event -> + if (event == Lifecycle.Event.ON_STOP) controller.cancel() + } + lifecycleOwner.lifecycle.addObserver(observer) + onDispose { + lifecycleOwner.lifecycle.removeObserver(observer) + controller.cancel() + } + } + return controller +} + +@Composable +internal fun VoiceNotePreparing(modifier: Modifier = Modifier) { + Surface( + modifier = modifier.fillMaxWidth().heightIn(min = ClawTheme.spacing.touchTarget), + shape = RoundedCornerShape(ClawTheme.radii.control), + color = ClawTheme.colors.surfaceRaised, + contentColor = ClawTheme.colors.textSubtle, + border = BorderStroke(1.dp, ClawTheme.colors.border), + ) { + Row( + modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(10.dp), + ) { + Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(18.dp)) + Text(text = "Preparing voice note…", style = ClawTheme.type.label) + } + } +} + +@Composable +internal fun VoiceNoteRecordButton( + enabled: Boolean, + onClick: () -> Unit, + modifier: Modifier = Modifier, +) { + Surface( + onClick = onClick, + enabled = enabled, + modifier = modifier.size(ClawTheme.spacing.touchTarget), + shape = CircleShape, + color = ClawTheme.colors.surfaceRaised, + contentColor = if (enabled) ClawTheme.colors.text else ClawTheme.colors.textSubtle, + ) { + Box(contentAlignment = Alignment.Center) { + Icon(imageVector = Icons.Default.Mic, contentDescription = "Record voice note", modifier = Modifier.size(18.dp)) + } + } +} + +@Composable +internal fun VoiceNoteRecordingControls( + elapsedMs: Long, + onCancel: () -> Unit, + onDone: () -> Unit, + modifier: Modifier = Modifier, +) { + Surface( + modifier = modifier.fillMaxWidth().heightIn(min = ClawTheme.spacing.touchTarget), + shape = RoundedCornerShape(ClawTheme.radii.control), + color = ClawTheme.colors.surfaceRaised, + contentColor = ClawTheme.colors.text, + border = BorderStroke(1.dp, ClawTheme.colors.border), + ) { + Row( + modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(10.dp), + ) { + Box(modifier = Modifier.size(8.dp).background(ClawTheme.colors.danger, CircleShape)) + Text( + text = formatVoiceNoteDuration(elapsedMs), + style = ClawTheme.type.label.copy(fontWeight = FontWeight.SemiBold), + modifier = Modifier.weight(1f), + ) + Surface( + onClick = onCancel, + modifier = Modifier.size(36.dp), + shape = CircleShape, + color = ClawTheme.colors.canvas, + contentColor = ClawTheme.colors.text, + ) { + Box(contentAlignment = Alignment.Center) { + Icon(imageVector = Icons.Default.Close, contentDescription = "Cancel voice note", modifier = Modifier.size(17.dp)) + } + } + Surface( + onClick = onDone, + modifier = Modifier.size(36.dp), + shape = CircleShape, + color = ClawTheme.colors.primary, + contentColor = ClawTheme.colors.primaryText, + ) { + Box(contentAlignment = Alignment.Center) { + Icon(imageVector = Icons.Default.Check, contentDescription = "Finish voice note", modifier = Modifier.size(17.dp)) + } + } + } + } +} + +@Composable +internal fun VoiceNoteRecorderError(state: VoiceNoteRecorderState) { + val message = (state as? VoiceNoteRecorderState.Failure)?.message ?: return + Text(text = message, style = ClawTheme.type.caption, color = ClawTheme.colors.danger) +} + +internal fun ChatMessageContent.isAudioAttachment(): Boolean = type == "audio" || mimeType?.startsWith("audio/") == true + +@Composable +internal fun VoiceNoteMessageRow(durationMs: Long?) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Icon( + imageVector = Icons.Default.Mic, + contentDescription = null, + modifier = Modifier.size(16.dp), + tint = ClawTheme.colors.textMuted, + ) + Text(text = "Voice note", style = ClawTheme.type.body, color = ClawTheme.colors.text) + durationMs?.let { duration -> + Text( + text = formatVoiceNoteDuration(duration), + style = ClawTheme.type.caption, + color = ClawTheme.colors.textMuted, + ) + } + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/app/GatewayBootstrapAuthTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/GatewayBootstrapAuthTest.kt index 1fd69e89ae2c..c7e1d386024c 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/GatewayBootstrapAuthTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/GatewayBootstrapAuthTest.kt @@ -847,6 +847,32 @@ class GatewayBootstrapAuthTest { assertFalse(runtime.prefs.voiceMicEnabled.value) } + @Test + @OptIn(ExperimentalCoroutinesApi::class) + fun voiceNoteMicOwnershipBlocksLocalVoiceAndGatewayPtt() = + runBlocking { + val app = RuntimeEnvironment.getApplication() + shadowOf(app).grantPermissions(Manifest.permission.RECORD_AUDIO) + val runtime = createTestRuntime(app) + val dispatcher = readField(runtime, "invokeDispatcher") + Dispatchers.setMain(Dispatchers.Unconfined) + try { + assertTrue(runtime.tryAcquireVoiceNoteMic()) + + runtime.setMicEnabled(true) + runtime.setTalkModeEnabled(true) + val ptt = dispatcher.handleInvoke(OpenClawTalkCommand.PttStart.rawValue, null) + + assertEquals(VoiceCaptureMode.Off, runtime.voiceCaptureMode.value) + assertEquals("MIC_BUSY", ptt.error?.code) + assertEquals("MIC_BUSY: voice note recording is active", ptt.error?.message) + assertFalse(readField>(runtime, "externalAudioCaptureActive").value) + } finally { + runtime.releaseVoiceNoteMic() + Dispatchers.resetMain() + } + } + @Test @OptIn(ExperimentalCoroutinesApi::class) fun talkPttStart_cleansPreparedCaptureWhenBeginFails() = diff --git a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatControllerMessageIdentityTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatControllerMessageIdentityTest.kt index edb84cbc5429..d78bbeec3f26 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatControllerMessageIdentityTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatControllerMessageIdentityTest.kt @@ -153,6 +153,63 @@ class ChatControllerMessageIdentityTest { assertNotEquals(reconciled[0].id, reconciled[1].id) } + @Test + fun reconcileMessageIdsPreservesOptimisticVoiceNoteDuration() { + val previous = + ChatMessage( + id = "local-user", + role = "user", + content = + listOf( + ChatMessageContent(type = "text", text = "See attached."), + ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "voice-note.m4a", durationMs = 4_321L), + ), + timestampMs = 1_000L, + idempotencyKey = "run:user", + ) + val incoming = + previous.copy( + id = "gateway-user", + content = previous.content.map { it.copy(durationMs = null) }, + ) + + val reconciled = reconcileMessageIds(previous = listOf(previous), incoming = listOf(incoming)).single() + + assertEquals("local-user", reconciled.id) + assertEquals(4_321L, reconciled.content[1].durationMs) + } + + @Test + fun reconcileMessageIdsPreservesMultipleVoiceNoteDurationsInOrder() { + val previous = + ChatMessage( + id = "local-user", + role = "user", + content = + listOf( + ChatMessageContent(type = "text", text = "See attached."), + ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "first.m4a", durationMs = 1_000L), + ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "second.m4a", durationMs = 2_000L), + ), + timestampMs = 1_000L, + idempotencyKey = "run:user", + ) + val incoming = + previous.copy( + id = "gateway-user", + content = + listOf( + ChatMessageContent(type = "text", text = "See attached."), + ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "stored-a.m4a"), + ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "stored-b.m4a"), + ), + ) + + val reconciled = reconcileMessageIds(previous = listOf(previous), incoming = listOf(incoming)).single() + + assertEquals(listOf(1_000L, 2_000L), reconciled.content.drop(1).map { it.durationMs }) + } + @Test fun mergeOptimisticMessagesKeepsOutgoingUserTurnWhenHistoryOmitsIt() { val optimistic = diff --git a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatMessageContentParsingTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatMessageContentParsingTest.kt index 3946cdbbf135..2061a8fca3c2 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatMessageContentParsingTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatMessageContentParsingTest.kt @@ -1,6 +1,7 @@ package ai.openclaw.app.chat import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject import org.junit.Assert.assertEquals import org.junit.Assert.assertNull import org.junit.Test @@ -46,4 +47,42 @@ class ChatMessageContentParsingTest { parseChatMessageContent(managedImage), ) } + + @Test + fun parsesDirectAndAttachmentAudioBlocks() { + val direct = + Json.parseToJsonElement( + """{"type":"audio","mimeType":"audio/mp4","fileName":"voice.m4a"}""", + ) + val attachment = + Json.parseToJsonElement( + """{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mpeg","label":"reply.mp3"}}""", + ) + + assertEquals( + ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "voice.m4a"), + parseChatMessageContent(direct), + ) + assertEquals( + ChatMessageContent(type = "audio", mimeType = "audio/mpeg", fileName = "reply.mp3"), + parseChatMessageContent(attachment), + ) + } + + @Test + fun parsesTranscriptAudioMediaFieldsAlongsideCaption() { + val message = + Json + .parseToJsonElement( + """{"content":[{"type":"text","text":"See attached."}],"MediaPaths":["media/inbound/voice.m4a"],"MediaTypes":["audio/x-m4a"]}""", + ).jsonObject + + assertEquals( + listOf( + ChatMessageContent(type = "text", text = "See attached."), + ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "voice.m4a"), + ), + parseChatMessageContents(message), + ) + } } diff --git a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatVoiceNoteAttachmentTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatVoiceNoteAttachmentTest.kt new file mode 100644 index 000000000000..9561e5b02fb0 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatVoiceNoteAttachmentTest.kt @@ -0,0 +1,74 @@ +package ai.openclaw.app.chat + +import ai.openclaw.app.ui.chat.stageVoiceNoteAttachment +import ai.openclaw.app.ui.chat.toOutgoingAttachment +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.nio.file.Files + +@OptIn(ExperimentalCoroutinesApi::class) +class ChatVoiceNoteAttachmentTest { + @Test + fun stagedVoiceNoteUsesAudioPayloadAndDurationInLocalEcho() = + runTest { + val file = Files.createTempFile("voice-note-", ".m4a").toFile() + file.writeBytes("voice-bytes".encodeToByteArray()) + val pending = stageVoiceNoteAttachment(VoiceNoteRecording(file = file, durationMs = 12_345L)) + val outgoing = pending.toOutgoingAttachment() + var sentParams: JsonObject? = null + val json = Json { ignoreUnknownKeys = true } + val chat = + ChatController( + scope = backgroundScope, + json = json, + requestGateway = { method, paramsJson -> + when (method) { + "chat.send" -> { + sentParams = json.parseToJsonElement(paramsJson.orEmpty()) as JsonObject + """{"runId":"voice-run","status":"started"}""" + } + else -> "{}" + } + }, + ) + chat.handleGatewayEvent("health", null) + runCurrent() + + val accepted = + chat.sendMessageAwaitAcceptance( + message = "listen", + thinkingLevel = "off", + attachments = listOf(outgoing), + ) + + assertTrue(accepted) + assertFalse(file.exists()) + assertEquals("audio", outgoing.type) + assertEquals(VOICE_NOTE_MIME_TYPE, outgoing.mimeType) + assertTrue(outgoing.fileName.endsWith(".m4a")) + assertEquals("dm9pY2UtYnl0ZXM=", outgoing.base64) + + val sentAttachment = ((sentParams?.get("attachments") as JsonArray).single() as JsonObject) + assertEquals("audio", (sentAttachment["type"] as JsonPrimitive).content) + assertEquals(VOICE_NOTE_MIME_TYPE, (sentAttachment["mimeType"] as JsonPrimitive).content) + assertTrue((sentAttachment["fileName"] as JsonPrimitive).content.endsWith(".m4a")) + assertEquals("dm9pY2UtYnl0ZXM=", (sentAttachment["content"] as JsonPrimitive).content) + + val echoedAudio = + chat.messages.value + .single() + .content + .single { it.type == "audio" } + assertEquals(VOICE_NOTE_MIME_TYPE, echoedAudio.mimeType) + assertEquals(12_345L, echoedAudio.durationMs) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/app/chat/VoiceNoteRecorderControllerTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/chat/VoiceNoteRecorderControllerTest.kt new file mode 100644 index 000000000000..16c3273502e2 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/app/chat/VoiceNoteRecorderControllerTest.kt @@ -0,0 +1,294 @@ +package ai.openclaw.app.chat + +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.File +import java.nio.file.Files + +@OptIn(ExperimentalCoroutinesApi::class) +class VoiceNoteRecorderControllerTest { + private class FakeEngine( + var durationMs: Long = 1_200L, + var outputBytes: ByteArray = byteArrayOf(1, 2, 3), + ) : VoiceNoteRecordingEngine { + var startCount = 0 + var stopCount = 0 + var cancelCount = 0 + var outputFile: File? = null + + override fun start(outputFile: File) { + startCount += 1 + this.outputFile = outputFile + outputFile.writeBytes(outputBytes) + } + + override fun stop(): Long { + stopCount += 1 + return durationMs + } + + override fun cancel() { + cancelCount += 1 + } + } + + @Test + fun startTransitionsToRecordingAndPublishesElapsedTime() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + var now = 1_000L + val engine = FakeEngine() + val controller = controller(directory, engine, elapsedRealtimeMillis = { now }) + + assertTrue(controller.start()) + assertEquals(VoiceNoteRecorderState.Recording(startedAtMillis = 1_000L), controller.state.value) + + now = 3_500L + advanceTimeBy(250L) + runCurrent() + assertEquals(2_500L, controller.elapsedMs.value) + + controller.cancel() + directory.deleteRecursively() + } + + @Test + fun stopReturnsRetainedFileAndDuration() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine(durationMs = 4_321L) + val finished = mutableListOf() + val controller = controller(directory, engine, onFinished = finished::add) + + controller.start() + assertTrue(controller.finish()) + + val recording = finished.single() + assertEquals(4_321L, recording.durationMs) + assertTrue(recording.file.exists()) + assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value) + controller.completePreparation() + assertEquals(VoiceNoteRecorderState.Idle, controller.state.value) + recording.file.delete() + directory.deleteRecursively() + } + + @Test + fun stopMarksMpeg4ContainerAsM4aForGatewaySniffing() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val mpeg4 = ByteArray(24) + "ftypmp42".toByteArray(Charsets.US_ASCII).copyInto(mpeg4, destinationOffset = 4) + val engine = FakeEngine(outputBytes = mpeg4) + val finished = mutableListOf() + val controller = controller(directory, engine, onFinished = finished::add) + + controller.start() + controller.finish() + + val recording = finished.single() + val majorBrand = + recording.file + .readBytes() + .copyOfRange(8, 12) + .toString(Charsets.US_ASCII) + assertEquals("M4A ", majorBrand) + recording.file.delete() + directory.deleteRecursively() + } + + @Test + fun cancelDeletesTemporaryFileAndReturnsIdle() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine() + val controller = controller(directory, engine) + + controller.start() + val file = requireNotNull(engine.outputFile) + controller.cancel() + + assertFalse(file.exists()) + assertEquals(1, engine.cancelCount) + assertEquals(VoiceNoteRecorderState.Idle, controller.state.value) + directory.deleteRecursively() + } + + @Test + fun durationCapUsesNormalFinishPath() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + var now = 1_000L + val engine = FakeEngine(durationMs = VOICE_NOTE_MAX_DURATION_MS) + val finished = mutableListOf() + val controller = controller(directory, engine, onFinished = finished::add, elapsedRealtimeMillis = { now }) + + controller.start() + now += VOICE_NOTE_MAX_DURATION_MS + advanceTimeBy(250L) + runCurrent() + + assertEquals(1, engine.stopCount) + assertEquals(VOICE_NOTE_MAX_DURATION_MS, finished.single().durationMs) + finished.single().file.delete() + directory.deleteRecursively() + } + + @Test + fun oversizeRecordingFailsAndDeletesFile() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine(outputBytes = ByteArray(VOICE_NOTE_MAX_BYTES.toInt() + 1)) + val finished = mutableListOf() + val controller = controller(directory, engine, onFinished = finished::add) + + controller.start() + val file = requireNotNull(engine.outputFile) + assertFalse(controller.finish()) + + assertFalse(file.exists()) + assertTrue(finished.isEmpty()) + assertEquals( + VoiceNoteRecorderState.Failure("Voice note is too large. Record a shorter message."), + controller.state.value, + ) + directory.deleteRecursively() + } + + @Test + fun startIsRefusedWhileAlreadyRecording() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine() + val controller = controller(directory, engine) + + assertTrue(controller.start()) + assertFalse(controller.start()) + assertEquals(1, engine.startCount) + + controller.cancel() + directory.deleteRecursively() + } + + @Test + fun startIsRefusedWhilePreparingAttachment() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine() + val finished = mutableListOf() + val controller = controller(directory, engine, onFinished = finished::add) + + controller.start() + controller.finish() + + assertFalse(controller.start()) + assertEquals(1, engine.startCount) + assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value) + + finished.single().file.delete() + directory.deleteRecursively() + } + + @Test + fun cancelDuringPreparingDeletesHandedOffFile() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine() + val finished = mutableListOf() + val controller = controller(directory, engine, onFinished = finished::add) + + controller.start() + controller.finish() + assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value) + assertTrue(finished.single().file.exists()) + + // Composition-scoped staging may be cancelled before it runs; cancel() + // must still delete the handed-off recording. + controller.cancel() + + assertEquals(VoiceNoteRecorderState.Idle, controller.state.value) + assertFalse(finished.single().file.exists()) + directory.deleteRecursively() + } + + @Test + fun reportFailureDuringPreparingDeletesHandedOffFile() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine() + val finished = mutableListOf() + val controller = controller(directory, engine, onFinished = finished::add) + + controller.start() + controller.finish() + + controller.reportFailure("Could not prepare voice note.") + + assertEquals( + VoiceNoteRecorderState.Failure("Could not prepare voice note."), + controller.state.value, + ) + assertFalse(finished.single().file.exists()) + directory.deleteRecursively() + } + + @Test + fun startIsRefusedWhileVoiceCaptureOwnsMic() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine() + val controller = controller(directory, engine, acquireMic = { false }) + + assertFalse(controller.start()) + + assertEquals(0, engine.startCount) + assertEquals( + VoiceNoteRecorderState.Failure("Voice capture is already using the microphone."), + controller.state.value, + ) + directory.deleteRecursively() + } + + @Test + fun permissionDeniedIsUserVisibleAndDoesNotStartEngine() = + runTest { + val directory = Files.createTempDirectory("voice-note-test").toFile() + val engine = FakeEngine() + val controller = controller(directory, engine, requestPermission = { false }) + + assertFalse(controller.start()) + + assertEquals(0, engine.startCount) + assertEquals( + VoiceNoteRecorderState.Failure("Microphone permission is required to record a voice note."), + controller.state.value, + ) + directory.deleteRecursively() + } + + private fun kotlinx.coroutines.test.TestScope.controller( + directory: File, + engine: FakeEngine, + requestPermission: suspend () -> Boolean = { true }, + acquireMic: () -> Boolean = { true }, + releaseMic: () -> Unit = {}, + onFinished: (VoiceNoteRecording) -> Unit = {}, + elapsedRealtimeMillis: () -> Long = { 1_000L }, + ): VoiceNoteRecorderController = + VoiceNoteRecorderController( + scope = this, + outputDirectory = directory, + engine = engine, + requestPermission = requestPermission, + acquireMic = acquireMic, + releaseMic = releaseMic, + onFinished = onFinished, + elapsedRealtimeMillis = elapsedRealtimeMillis, + ) +}