feat(android): record and send voice notes from the chat composer (#101193)

* feat(android): record and send voice notes from the chat composer

Adds tap-to-record voice notes to both Android chat composers: MediaRecorder
AAC/m4a mono capture with a 3-minute cap, lazy RECORD_AUDIO permission,
cancel/finish controls, audio attachment chips with duration, and voice-note
transcript rows. Recording is mutually exclusive with MicCaptureManager voice
capture (no mic arbiter exists), and the MPEG-4 container brand is normalized
to M4A so gateway byte sniffing classifies the upload as audio, not video.
Attachments stay online-only per the outbox contract.

Related: #100709

* fix(android): coordinate voice note microphone ownership

* chore(android): refresh native localization inventory
This commit is contained in:
Peter Steinberger
2026-07-06 23:54:44 +01:00
committed by GitHub
parent 66c764b269
commit ef1767baca
19 changed files with 1435 additions and 159 deletions

View File

@@ -139,7 +139,7 @@
},
{
"kind": "ui-state-text",
"line": 121,
"line": 125,
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
"source": "Searching…",
"surface": "android",
@@ -147,7 +147,7 @@
},
{
"kind": "ui-state-text",
"line": 212,
"line": 216,
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
"source": "Mic off",
"surface": "android",
@@ -155,7 +155,7 @@
},
{
"kind": "ui-state-text",
"line": 222,
"line": 226,
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
"source": "Off",
"surface": "android",
@@ -203,7 +203,7 @@
},
{
"kind": "conditional-branch",
"line": 2683,
"line": 2701,
"path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt",
"source": "Failed: this host requires wss:// or Tailscale Serve. No TLS endpoint detected.",
"surface": "android",
@@ -211,7 +211,7 @@
},
{
"kind": "conditional-branch",
"line": 2685,
"line": 2703,
"path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt",
"source": "Failed: secure endpoint reached, but TLS fingerprint verification timed out. Check Tailscale Serve or gateway TLS and retry.",
"surface": "android",
@@ -219,7 +219,7 @@
},
{
"kind": "conditional-branch",
"line": 2687,
"line": 2705,
"path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt",
"source": "Failed: couldn't reach the secure gateway endpoint for this host.",
"surface": "android",
@@ -5763,7 +5763,7 @@
},
{
"kind": "ui-call",
"line": 199,
"line": 218,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
"source": "Type a message…",
"surface": "android",
@@ -5771,7 +5771,7 @@
},
{
"kind": "ui-named-argument",
"line": 209,
"line": 231,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
"source": "Gateway is offline. Text messages are queued and sent after reconnecting.",
"surface": "android",
@@ -5779,7 +5779,7 @@
},
{
"kind": "ui-named-argument",
"line": 238,
"line": 260,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
"source": "Select thinking level",
"surface": "android",
@@ -5787,7 +5787,7 @@
},
{
"kind": "ui-named-argument",
"line": 262,
"line": 284,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
"source": "Attach",
"surface": "android",
@@ -5795,7 +5795,7 @@
},
{
"kind": "ui-named-argument",
"line": 270,
"line": 299,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
"source": "Refresh",
"surface": "android",
@@ -5803,7 +5803,7 @@
},
{
"kind": "ui-named-argument",
"line": 278,
"line": 307,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
"source": "Abort",
"surface": "android",
@@ -5811,7 +5811,7 @@
},
{
"kind": "ui-named-argument",
"line": 323,
"line": 356,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
"source": "Send",
"surface": "android",
@@ -5819,7 +5819,7 @@
},
{
"kind": "ui-named-argument",
"line": 349,
"line": 382,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
"source": "No commands found",
"surface": "android",
@@ -5979,7 +5979,7 @@
},
{
"kind": "ui-named-argument",
"line": 260,
"line": 261,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Preview · $domain",
"surface": "android",
@@ -5987,7 +5987,7 @@
},
{
"kind": "ui-named-argument",
"line": 269,
"line": 270,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Expand link preview",
"surface": "android",
@@ -5995,7 +5995,7 @@
},
{
"kind": "ui-call",
"line": 293,
"line": 294,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Loading preview…",
"surface": "android",
@@ -6003,7 +6003,7 @@
},
{
"kind": "ui-call",
"line": 294,
"line": 295,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "No preview available",
"surface": "android",
@@ -6011,7 +6011,7 @@
},
{
"kind": "ui-call",
"line": 339,
"line": 340,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Thinking...",
"surface": "android",
@@ -6019,7 +6019,7 @@
},
{
"kind": "ui-named-argument",
"line": 355,
"line": 356,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Tools",
"surface": "android",
@@ -6027,7 +6027,7 @@
},
{
"kind": "ui-call",
"line": 358,
"line": 359,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Running tools...",
"surface": "android",
@@ -6035,7 +6035,7 @@
},
{
"kind": "ui-call",
"line": 361,
"line": 362,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "${display.emoji} ${display.label}",
"surface": "android",
@@ -6043,7 +6043,7 @@
},
{
"kind": "ui-named-argument",
"line": 379,
"line": 380,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "... +${toolCalls.size - 6} more",
"surface": "android",
@@ -6051,7 +6051,7 @@
},
{
"kind": "ui-named-argument",
"line": 410,
"line": 411,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "You",
"surface": "android",
@@ -6059,7 +6059,7 @@
},
{
"kind": "ui-named-argument",
"line": 424,
"line": 425,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Retry",
"surface": "android",
@@ -6067,7 +6067,7 @@
},
{
"kind": "ui-named-argument",
"line": 427,
"line": 428,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Delete",
"surface": "android",
@@ -6075,7 +6075,7 @@
},
{
"kind": "ui-named-argument",
"line": 459,
"line": 460,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "OpenClaw · Live",
"surface": "android",
@@ -6083,7 +6083,7 @@
},
{
"kind": "conditional-branch",
"line": 496,
"line": 497,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "System",
"surface": "android",
@@ -6091,7 +6091,7 @@
},
{
"kind": "conditional-branch",
"line": 497,
"line": 498,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "OpenClaw",
"surface": "android",
@@ -6099,7 +6099,7 @@
},
{
"kind": "ui-call",
"line": 523,
"line": 524,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
"source": "Unsupported attachment",
"surface": "android",
@@ -6107,7 +6107,7 @@
},
{
"kind": "ui-named-argument",
"line": 276,
"line": 290,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Chat needs attention",
"surface": "android",
@@ -6115,7 +6115,7 @@
},
{
"kind": "ui-named-argument",
"line": 436,
"line": 448,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "All",
"surface": "android",
@@ -6123,7 +6123,7 @@
},
{
"kind": "ui-named-argument",
"line": 494,
"line": 506,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "OpenClaw",
"surface": "android",
@@ -6131,7 +6131,7 @@
},
{
"kind": "ui-named-argument",
"line": 515,
"line": 527,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "New chat",
"surface": "android",
@@ -6139,7 +6139,7 @@
},
{
"kind": "ui-named-argument",
"line": 520,
"line": 532,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "More new chat options",
"surface": "android",
@@ -6147,7 +6147,7 @@
},
{
"kind": "ui-named-argument",
"line": 535,
"line": 547,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Refresh chat",
"surface": "android",
@@ -6155,7 +6155,7 @@
},
{
"kind": "ui-named-argument",
"line": 538,
"line": 550,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Chat",
"surface": "android",
@@ -6163,7 +6163,7 @@
},
{
"kind": "ui-named-argument",
"line": 686,
"line": 698,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Loading session",
"surface": "android",
@@ -6171,7 +6171,7 @@
},
{
"kind": "ui-named-argument",
"line": 713,
"line": 725,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Jump to latest",
"surface": "android",
@@ -6179,7 +6179,7 @@
},
{
"kind": "conditional-branch",
"line": 739,
"line": 751,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Ready when you are",
"surface": "android",
@@ -6187,7 +6187,7 @@
},
{
"kind": "ui-named-argument",
"line": 770,
"line": 782,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Fix connection",
"surface": "android",
@@ -6195,7 +6195,7 @@
},
{
"kind": "ui-named-argument",
"line": 771,
"line": 783,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Copy diagnostics",
"surface": "android",
@@ -6203,7 +6203,7 @@
},
{
"kind": "conditional-branch",
"line": 955,
"line": 967,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Preparing audio…",
"surface": "android",
@@ -6211,7 +6211,7 @@
},
{
"kind": "conditional-branch",
"line": 955,
"line": 967,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Speaking…",
"surface": "android",
@@ -6219,7 +6219,7 @@
},
{
"kind": "ui-named-argument",
"line": 976,
"line": 988,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Tools running",
"surface": "android",
@@ -6227,7 +6227,7 @@
},
{
"kind": "ui-named-argument",
"line": 978,
"line": 990,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "OpenClaw is working",
"surface": "android",
@@ -6235,7 +6235,7 @@
},
{
"kind": "ui-named-argument",
"line": 981,
"line": 993,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "+${toolCalls.size - 4} more",
"surface": "android",
@@ -6243,7 +6243,7 @@
},
{
"kind": "ui-named-argument",
"line": 991,
"line": 1003,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Thinking",
"surface": "android",
@@ -6251,7 +6251,7 @@
},
{
"kind": "ui-named-argument",
"line": 992,
"line": 1004,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "OpenClaw is preparing a response.",
"surface": "android",
@@ -6259,7 +6259,7 @@
},
{
"kind": "ui-named-argument",
"line": 1126,
"line": 1161,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Stop",
"surface": "android",
@@ -6267,7 +6267,7 @@
},
{
"kind": "ui-named-argument",
"line": 1192,
"line": 1227,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Default",
"surface": "android",
@@ -6275,7 +6275,7 @@
},
{
"kind": "conditional-branch",
"line": 1258,
"line": 1293,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Pin model",
"surface": "android",
@@ -6283,7 +6283,7 @@
},
{
"kind": "conditional-branch",
"line": 1258,
"line": 1293,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Unpin model",
"surface": "android",
@@ -6291,7 +6291,7 @@
},
{
"kind": "ui-named-argument",
"line": 1275,
"line": 1310,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "No commands found",
"surface": "android",
@@ -6299,7 +6299,7 @@
},
{
"kind": "ui-named-argument",
"line": 1337,
"line": 1372,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Gateway offline",
"surface": "android",
@@ -6307,7 +6307,7 @@
},
{
"kind": "ui-named-argument",
"line": 1427,
"line": 1464,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Attach image",
"surface": "android",
@@ -6315,7 +6315,7 @@
},
{
"kind": "ui-named-argument",
"line": 1442,
"line": 1483,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Message OpenClaw",
"surface": "android",
@@ -6323,15 +6323,15 @@
},
{
"kind": "ui-named-argument",
"line": 1457,
"line": 1498,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Voice",
"source": "Open voice",
"surface": "android",
"id": "native.android.19396391033f4fa8"
"id": "native.android.97602a28629a331f"
},
{
"kind": "ui-named-argument",
"line": 1495,
"line": 1547,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Remove attachment",
"surface": "android",
@@ -6339,7 +6339,7 @@
},
{
"kind": "conditional-branch",
"line": 1516,
"line": 1568,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Main",
"surface": "android",
@@ -6347,7 +6347,7 @@
},
{
"kind": "ui-named-argument",
"line": 1572,
"line": 1624,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "Send",
"surface": "android",
@@ -6355,7 +6355,7 @@
},
{
"kind": "conditional-branch",
"line": 1611,
"line": 1663,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
"source": "$contextLabel · ${contextMeterThinkingLabel(thinkingLevel)}",
"surface": "android",
@@ -6363,12 +6363,52 @@
},
{
"kind": "ui-named-argument",
"line": 281,
"line": 291,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatSheetContent.kt",
"source": "CHAT ERROR",
"surface": "android",
"id": "native.android.826804c0189e03d0"
},
{
"kind": "ui-named-argument",
"line": 109,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
"source": "Preparing voice note…",
"surface": "android",
"id": "native.android.3df4e24a0fc8810b"
},
{
"kind": "ui-named-argument",
"line": 129,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
"source": "Record voice note",
"surface": "android",
"id": "native.android.06ab95879f652d3e"
},
{
"kind": "ui-named-argument",
"line": 167,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
"source": "Cancel voice note",
"surface": "android",
"id": "native.android.8a7653317387033f"
},
{
"kind": "ui-named-argument",
"line": 178,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
"source": "Finish voice note",
"surface": "android",
"id": "native.android.fb327cd3030aae44"
},
{
"kind": "ui-named-argument",
"line": 205,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
"source": "Voice note",
"surface": "android",
"id": "native.android.e9d238668cb46e71"
},
{
"kind": "ui-named-argument",
"line": 537,

View File

@@ -17,10 +17,12 @@ import ai.openclaw.app.node.SmsManager
import ai.openclaw.app.ui.GatewayConnectPlan
import ai.openclaw.app.ui.GatewaySavedAuthAction
import ai.openclaw.app.voice.VoiceConversationEntry
import android.Manifest
import android.app.Application
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.flow.MutableStateFlow
@@ -57,6 +59,8 @@ class MainViewModel(
private val gatewayConfigOperationSeq = AtomicLong()
private val gatewayConfigOperationMutex = Mutex()
@Volatile private var permissionRequester: PermissionRequester? = null
@Volatile private var foreground = false
@Volatile private var runtimeStartupQueued = false
@@ -264,6 +268,7 @@ class MainViewModel(
runtime.camera.attachLifecycleOwner(owner)
runtime.camera.attachPermissionRequester(permissionRequester)
runtime.sms.attachPermissionRequester(permissionRequester)
this.permissionRequester = permissionRequester
}
/**
@@ -536,6 +541,23 @@ class MainViewModel(
ensureRuntime().setTalkModeEnabled(enabled)
}
suspend fun requestVoiceNotePermission(): Boolean {
val requester = permissionRequester ?: return false
return try {
requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO))[Manifest.permission.RECORD_AUDIO] == true
} catch (error: CancellationException) {
throw error
} catch (_: Throwable) {
false
}
}
internal fun tryAcquireVoiceNoteMic(): Boolean = runtimeRef.value?.tryAcquireVoiceNoteMic() == true
internal fun releaseVoiceNoteMic() {
runtimeRef.value?.releaseVoiceNoteMic()
}
fun setSpeakerEnabled(enabled: Boolean) {
ensureRuntime().setSpeakerEnabled(enabled)
}

View File

@@ -742,6 +742,7 @@ class NodeRuntime private constructor(
// Keep ownership epochs and their service/capture state transitions atomic.
// Otherwise stale PTT cleanup can pass its epoch check before a UI mode change.
private val voiceCaptureOwnershipLock = Any()
private var voiceNoteOwnsMic = false
private val voiceCapturePreparationMutex = Mutex()
private var didAutoRequestCanvasRehydrate = false
@@ -1865,6 +1866,19 @@ class NodeRuntime private constructor(
setVoiceCaptureMode(if (value) VoiceCaptureMode.ManualMic else VoiceCaptureMode.Off)
}
internal fun tryAcquireVoiceNoteMic(): Boolean =
synchronized(voiceCaptureOwnershipLock) {
if (voiceNoteOwnsMic || !isVoiceCaptureModeActive(VoiceCaptureMode.Off)) return@synchronized false
voiceNoteOwnsMic = true
true
}
internal fun releaseVoiceNoteMic() {
synchronized(voiceCaptureOwnershipLock) {
voiceNoteOwnsMic = false
}
}
fun cancelMicCapture() {
micCapture.cancelMicCapture()
setVoiceCaptureMode(VoiceCaptureMode.Off, persistManualMic = false)
@@ -2024,6 +2038,9 @@ class NodeRuntime private constructor(
) {
throw IllegalStateException("NODE_BACKGROUND_UNAVAILABLE: command requires foreground")
}
if (voiceNoteOwnsMic) {
throw IllegalStateException("MIC_BUSY: voice note recording is active")
}
if (!hasRecordAudioPermission()) {
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
@@ -2251,6 +2268,7 @@ class NodeRuntime private constructor(
persistManualMic: Boolean = true,
) {
synchronized(voiceCaptureOwnershipLock) {
if (mode != VoiceCaptureMode.Off && voiceNoteOwnsMic) return
talkPttCommandEpoch.incrementAndGet()
voiceCaptureOwnershipEpoch.incrementAndGet()
val permissionDenied = mode.requiresMicrophonePermission && !hasRecordAudioPermission()

View File

@@ -682,6 +682,7 @@ class ChatController internal constructor(
mimeType = att.mimeType,
fileName = att.fileName,
base64 = att.base64,
durationMs = att.durationMs,
),
)
}
@@ -2221,29 +2222,60 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
text = obj["text"].asStringOrNull() ?: obj["content"].asStringOrNull(),
)
"image" ->
"image", "audio" ->
ChatMessageContent(
type = "image",
type = obj["type"].asStringOrNull() ?: "image",
mimeType = obj["mimeType"].asStringOrNull(),
fileName = obj["fileName"].asStringOrNull(),
base64 = obj["content"].asStringOrNull()?.takeIf { it.isNotBlank() },
)
"attachment" -> {
val attachment = obj["attachment"].asObjectOrNull() ?: return null
val mimeType = attachment["mimeType"].asStringOrNull()
if (attachment["kind"].asStringOrNull() != "audio" && mimeType?.startsWith("audio/") != true) return null
ChatMessageContent(
type = "audio",
mimeType = mimeType,
fileName = attachment["label"].asStringOrNull(),
)
}
else -> null
}
}
internal fun parseChatMessageContents(obj: JsonObject): List<ChatMessageContent> {
obj["content"].asArrayOrNull()?.let { content ->
return content.mapNotNull(::parseChatMessageContent)
val content =
obj["content"].asArrayOrNull()?.mapNotNull(::parseChatMessageContent)
?: obj["content"].asStringOrNull()?.let { listOf(ChatMessageContent(type = "text", text = it)) }
?: obj["text"].asStringOrNull()?.let { listOf(ChatMessageContent(type = "text", text = it)) }
?: emptyList()
val transcriptAudio = parseTranscriptAudioContents(obj)
if (transcriptAudio.isEmpty()) return content
return content +
transcriptAudio.filterNot { audio ->
content.any { it.mimeType == audio.mimeType && it.fileName == audio.fileName }
}
}
private fun parseTranscriptAudioContents(obj: JsonObject): List<ChatMessageContent> {
val paths =
obj["MediaPaths"].asArrayOrNull()?.mapNotNull { it.asStringOrNull() }
?: obj["MediaPath"].asStringOrNull()?.let { listOf(it) }
?: return emptyList()
val types =
obj["MediaTypes"].asArrayOrNull()?.map { it.asStringOrNull().orEmpty() }
?: obj["MediaType"].asStringOrNull()?.let { listOf(it) }
?: emptyList()
return paths.mapIndexedNotNull { index, path ->
val mimeType = types.getOrNull(index)?.takeIf { it.startsWith("audio/") } ?: return@mapIndexedNotNull null
ChatMessageContent(
type = "audio",
mimeType = mimeType,
fileName = path.substringAfterLast('/').takeIf(String::isNotBlank),
)
}
obj["content"].asStringOrNull()?.let { text ->
return listOf(ChatMessageContent(type = "text", text = text))
}
obj["text"].asStringOrNull()?.let { text ->
return listOf(ChatMessageContent(type = "text", text = text))
}
return emptyList()
}
private fun parseCreatedSessionKey(
@@ -2330,21 +2362,48 @@ internal fun reconcileMessageIds(
): List<ChatMessage> {
if (previous.isEmpty() || incoming.isEmpty()) return incoming
val idsByKey = LinkedHashMap<String, ArrayDeque<String>>()
val messagesByKey = LinkedHashMap<String, ArrayDeque<ChatMessage>>()
for (message in previous) {
val key = messageIdentityKey(message) ?: continue
idsByKey.getOrPut(key) { ArrayDeque() }.addLast(message.id)
messagesByKey.getOrPut(key) { ArrayDeque() }.addLast(message)
}
return incoming.map { message ->
val key = messageIdentityKey(message) ?: return@map message
val ids = idsByKey[key] ?: return@map message
val reusedId = ids.removeFirstOrNull() ?: return@map message
if (ids.isEmpty()) {
idsByKey.remove(key)
val matches = messagesByKey[key] ?: return@map message
val previousMessage = matches.removeFirstOrNull() ?: return@map message
if (matches.isEmpty()) {
messagesByKey.remove(key)
}
if (reusedId == message.id) return@map message
message.copy(id = reusedId)
message.copy(
id = previousMessage.id,
content = preserveOptimisticAudioDuration(previous = previousMessage, incoming = message),
)
}
}
private fun preserveOptimisticAudioDuration(
previous: ChatMessage,
incoming: ChatMessage,
): List<ChatMessageContent> {
val idempotencyKey = incoming.idempotencyKey?.trim().orEmpty()
if (idempotencyKey.isEmpty() || idempotencyKey != previous.idempotencyKey?.trim()) return incoming.content
val remainingAudio =
previous.content
.filter { it.mimeType?.startsWith("audio/") == true && it.durationMs != null }
.toMutableList()
if (remainingAudio.isEmpty()) return incoming.content
return incoming.content.map { part ->
if (part.durationMs != null || part.mimeType?.startsWith("audio/") != true) return@map part
if (remainingAudio.isEmpty()) return@map part
val exactIndex =
remainingAudio.indexOfFirst {
it.mimeType == part.mimeType && it.fileName == part.fileName
}
val match = remainingAudio.removeAt(if (exactIndex >= 0) exactIndex else 0)
part.copy(durationMs = match.durationMs)
}
}

View File

@@ -31,6 +31,7 @@ data class ChatMessageContent(
val mimeType: String? = null,
val fileName: String? = null,
val base64: String? = null,
val durationMs: Long? = null,
)
/**
@@ -106,4 +107,5 @@ data class OutgoingAttachment(
val mimeType: String,
val fileName: String,
val base64: String,
val durationMs: Long? = null,
)

View File

@@ -0,0 +1,285 @@
package ai.openclaw.app.chat
import android.content.Context
import android.media.MediaRecorder
import android.os.SystemClock
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import java.io.File
import java.io.RandomAccessFile
import java.util.UUID
internal const val VOICE_NOTE_MAX_DURATION_MS = 180_000L
internal const val VOICE_NOTE_MAX_BYTES = 3_500_000L
internal const val VOICE_NOTE_MIME_TYPE = "audio/mp4"
internal sealed interface VoiceNoteRecorderState {
data object Idle : VoiceNoteRecorderState
data class Recording(
val startedAtMillis: Long,
) : VoiceNoteRecorderState
data object Preparing : VoiceNoteRecorderState
data class Failure(
val message: String,
) : VoiceNoteRecorderState
}
internal data class VoiceNoteRecording(
val file: File,
val durationMs: Long,
)
internal interface VoiceNoteRecordingEngine {
fun start(outputFile: File)
fun stop(): Long
fun cancel()
}
/** Owns voice-note recording state and temporary-file cleanup. */
internal class VoiceNoteRecorderController(
private val scope: CoroutineScope,
private val outputDirectory: File,
private val engine: VoiceNoteRecordingEngine,
private val requestPermission: suspend () -> Boolean,
private val acquireMic: () -> Boolean,
private val releaseMic: () -> Unit,
private val onFinished: (VoiceNoteRecording) -> Unit,
private val elapsedRealtimeMillis: () -> Long = SystemClock::elapsedRealtime,
) {
private val lock = Any()
private val _state = MutableStateFlow<VoiceNoteRecorderState>(VoiceNoteRecorderState.Idle)
val state: StateFlow<VoiceNoteRecorderState> = _state.asStateFlow()
private val _elapsedMs = MutableStateFlow(0L)
val elapsedMs: StateFlow<Long> = _elapsedMs.asStateFlow()
private var outputFile: File? = null
private var elapsedJob: Job? = null
private var ownsMic = false
suspend fun start(): Boolean {
synchronized(lock) {
if (_state.value !is VoiceNoteRecorderState.Idle && _state.value !is VoiceNoteRecorderState.Failure) return false
}
if (!requestPermission()) {
fail("Microphone permission is required to record a voice note.")
return false
}
return synchronized(lock) {
if (_state.value !is VoiceNoteRecorderState.Idle && _state.value !is VoiceNoteRecorderState.Failure) {
return@synchronized false
}
if (!acquireMic()) {
failLocked("Voice capture is already using the microphone.")
return@synchronized false
}
ownsMic = true
val startedAt = elapsedRealtimeMillis()
val file = File(outputDirectory, "voice-note-${UUID.randomUUID()}.m4a")
try {
engine.start(file)
} catch (_: Throwable) {
engine.cancel()
releaseMicLocked()
file.delete()
failLocked("Could not start voice-note recording.")
return@synchronized false
}
outputFile = file
_elapsedMs.value = 0L
_state.value = VoiceNoteRecorderState.Recording(startedAtMillis = startedAt)
startElapsedUpdates(startedAt)
true
}
}
fun finish(): Boolean {
val recording =
synchronized(lock) {
if (_state.value !is VoiceNoteRecorderState.Recording) return false
val file = outputFile ?: return false
elapsedJob?.cancel()
elapsedJob = null
val durationMs =
try {
engine.stop().coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS)
} catch (_: Throwable) {
engine.cancel()
file.delete()
finishFailureLocked("Could not finish voice-note recording.")
return false
}
try {
normalizeM4aContainerBrand(file)
} catch (_: Throwable) {
file.delete()
finishFailureLocked("Could not finish voice-note recording.")
return false
}
if (file.length() > VOICE_NOTE_MAX_BYTES) {
file.delete()
finishFailureLocked("Voice note is too large. Record a shorter message.")
return false
}
// The file stays owned by the controller through Preparing: the staging
// coroutine is composition-scoped and may be cancelled before it runs,
// so cancel() must still be able to delete the handed-off recording.
_elapsedMs.value = 0L
_state.value = VoiceNoteRecorderState.Preparing
releaseMicLocked()
VoiceNoteRecording(file = file, durationMs = durationMs)
}
onFinished(recording)
return true
}
fun completePreparation() {
synchronized(lock) {
if (_state.value is VoiceNoteRecorderState.Preparing) {
outputFile = null
_state.value = VoiceNoteRecorderState.Idle
}
}
}
fun cancel() {
synchronized(lock) {
elapsedJob?.cancel()
elapsedJob = null
if (_state.value is VoiceNoteRecorderState.Recording) {
engine.cancel()
}
releaseMicLocked()
outputFile?.delete()
outputFile = null
_elapsedMs.value = 0L
_state.value = VoiceNoteRecorderState.Idle
}
}
fun reportFailure(message: String) {
synchronized(lock) {
outputFile?.delete()
outputFile = null
failLocked(message)
}
}
private fun startElapsedUpdates(startedAt: Long) {
elapsedJob?.cancel()
elapsedJob =
scope.launch {
while (isActive && state.value is VoiceNoteRecorderState.Recording) {
val elapsed = (elapsedRealtimeMillis() - startedAt).coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS)
_elapsedMs.value = elapsed
// MediaRecorder's duration callback races its asynchronous auto-stop.
// Own the cap here so every successful finish calls stop() exactly once.
if (elapsed >= VOICE_NOTE_MAX_DURATION_MS) {
finish()
return@launch
}
delay(250L)
}
}
}
private fun fail(message: String) {
synchronized(lock) { failLocked(message) }
}
private fun failLocked(message: String) {
_state.value = VoiceNoteRecorderState.Failure(message)
}
private fun finishFailureLocked(message: String) {
releaseMicLocked()
outputFile = null
_elapsedMs.value = 0L
_state.value = VoiceNoteRecorderState.Failure(message)
}
private fun releaseMicLocked() {
if (!ownsMic) return
ownsMic = false
releaseMic()
}
}
/** Marks AAC-only MPEG-4 output as audio so gateway byte sniffing cannot classify it as video. */
internal fun normalizeM4aContainerBrand(file: File) {
RandomAccessFile(file, "rw").use { output ->
if (output.length() < 12L) return
output.seek(4L)
val boxType = ByteArray(4)
output.readFully(boxType)
if (!boxType.contentEquals("ftyp".toByteArray(Charsets.US_ASCII))) return
output.seek(8L)
output.write("M4A ".toByteArray(Charsets.US_ASCII))
}
}
/** Android AAC/m4a engine kept behind [VoiceNoteRecordingEngine] for JVM tests. */
internal class AndroidVoiceNoteRecordingEngine(
private val context: Context,
private val elapsedRealtime: () -> Long = SystemClock::elapsedRealtime,
) : VoiceNoteRecordingEngine {
private var recorder: MediaRecorder? = null
private var startedAtElapsedMs = 0L
override fun start(outputFile: File) {
check(recorder == null)
val next = MediaRecorder(context)
try {
next.setAudioSource(MediaRecorder.AudioSource.MIC)
next.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
next.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
next.setAudioChannels(1)
next.setAudioEncodingBitRate(32_000)
next.setAudioSamplingRate(44_100)
next.setOutputFile(outputFile.absolutePath)
next.prepare()
next.start()
startedAtElapsedMs = elapsedRealtime()
recorder = next
} catch (error: Throwable) {
next.release()
throw error
}
}
override fun stop(): Long {
val active = checkNotNull(recorder)
recorder = null
return try {
active.stop()
(elapsedRealtime() - startedAtElapsedMs).coerceAtLeast(0L)
} finally {
active.release()
}
}
override fun cancel() {
val active = recorder ?: return
recorder = null
runCatching { active.stop() }
active.release()
}
}

View File

@@ -3,6 +3,7 @@ package ai.openclaw.app.ui.chat
import ai.openclaw.app.ChatDraft
import ai.openclaw.app.ChatDraftPlacement
import ai.openclaw.app.chat.ChatCommandEntry
import ai.openclaw.app.chat.VoiceNoteRecorderState
import ai.openclaw.app.ui.mobileAccent
import ai.openclaw.app.ui.mobileAccentBorderStrong
import ai.openclaw.app.ui.mobileAccentSoft
@@ -35,6 +36,7 @@ import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.automirrored.filled.Send
import androidx.compose.material.icons.filled.ArrowDropDown
import androidx.compose.material.icons.filled.AttachFile
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Stop
import androidx.compose.material3.Button
@@ -130,16 +132,22 @@ internal fun applyDraftText(
/** Chat input surface for text, image attachments, thinking level, and run controls. */
@Composable
fun ChatComposer(
internal fun ChatComposer(
draftText: ChatDraft?,
healthOk: Boolean,
thinkingLevel: String,
pendingRunCount: Int,
commands: List<ChatCommandEntry>,
attachments: List<PendingImageAttachment>,
attachments: List<PendingAttachment>,
onDraftApplied: () -> Unit,
onPickImages: () -> Unit,
onRemoveAttachment: (id: String) -> Unit,
voiceNoteState: VoiceNoteRecorderState,
voiceNoteElapsedMs: Long,
recordVoiceNoteEnabled: Boolean,
onStartVoiceNote: () -> Unit,
onCancelVoiceNote: () -> Unit,
onFinishVoiceNote: () -> Unit,
onSetThinkingLevel: (level: String) -> Unit,
onRefresh: () -> Unit,
onAbort: () -> Unit,
@@ -176,6 +184,8 @@ fun ChatComposer(
input.trim().isNotEmpty() && attachments.isEmpty()
}
val sendBusy = pendingRunCount > 0
val recordingVoiceNote = voiceNoteState is VoiceNoteRecorderState.Recording
val preparingVoiceNote = voiceNoteState is VoiceNoteRecorderState.Preparing
Column(modifier = Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(8.dp)) {
if (attachments.isNotEmpty()) {
@@ -192,17 +202,29 @@ fun ChatComposer(
)
}
OutlinedTextField(
value = input,
onValueChange = { input = it },
modifier = Modifier.fillMaxWidth(),
placeholder = { Text("Type a message…", style = mobileBodyStyle(), color = mobileTextTertiary) },
minLines = 2,
maxLines = 5,
textStyle = mobileBodyStyle().copy(color = mobileText),
shape = RoundedCornerShape(14.dp),
colors = chatTextFieldColors(),
)
if (recordingVoiceNote) {
VoiceNoteRecordingControls(
elapsedMs = voiceNoteElapsedMs,
onCancel = onCancelVoiceNote,
onDone = onFinishVoiceNote,
)
} else if (preparingVoiceNote) {
VoiceNotePreparing()
} else {
OutlinedTextField(
value = input,
onValueChange = { input = it },
modifier = Modifier.fillMaxWidth(),
placeholder = { Text("Type a message…", style = mobileBodyStyle(), color = mobileTextTertiary) },
minLines = 2,
maxLines = 5,
textStyle = mobileBodyStyle().copy(color = mobileText),
shape = RoundedCornerShape(14.dp),
colors = chatTextFieldColors(),
)
}
VoiceNoteRecorderError(voiceNoteState)
if (!healthOk) {
Text(
@@ -266,6 +288,13 @@ fun ChatComposer(
onClick = onPickImages,
)
if (!recordingVoiceNote) {
VoiceNoteRecordButton(
enabled = recordVoiceNoteEnabled && !preparingVoiceNote,
onClick = onStartVoiceNote,
)
}
SecondaryActionButton(
label = "Refresh",
icon = Icons.Default.Refresh,
@@ -300,7 +329,7 @@ fun ChatComposer(
}
}
},
enabled = canSend,
enabled = canSend && !recordingVoiceNote && !preparingVoiceNote,
modifier = Modifier.height(44.dp),
shape = RoundedCornerShape(14.dp),
contentPadding = PaddingValues(horizontal = 20.dp),
@@ -311,7 +340,11 @@ fun ChatComposer(
disabledContainerColor = mobileBorderStrong,
disabledContentColor = mobileTextTertiary,
),
border = BorderStroke(1.dp, if (canSend) mobileAccentBorderStrong else mobileBorderStrong),
border =
BorderStroke(
1.dp,
if (canSend && !recordingVoiceNote && !preparingVoiceNote) mobileAccentBorderStrong else mobileBorderStrong,
),
) {
if (sendBusy) {
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp, color = Color.White)
@@ -456,7 +489,7 @@ private fun thinkingLabel(raw: String): String =
@Composable
private fun AttachmentsStrip(
attachments: List<PendingImageAttachment>,
attachments: List<PendingAttachment>,
onRemoveAttachment: (id: String) -> Unit,
) {
Row(
@@ -465,7 +498,7 @@ private fun AttachmentsStrip(
) {
for (att in attachments) {
AttachmentChip(
fileName = att.fileName,
attachment = att,
onRemove = { onRemoveAttachment(att.id) },
)
}
@@ -474,7 +507,7 @@ private fun AttachmentsStrip(
@Composable
private fun AttachmentChip(
fileName: String,
attachment: PendingAttachment,
onRemove: () -> Unit,
) {
Surface(
@@ -487,8 +520,13 @@ private fun AttachmentChip(
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
if (attachment.mimeType.startsWith("audio/")) {
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(14.dp), tint = mobileTextSecondary)
}
Text(
text = fileName,
text =
attachment.durationMs?.let { duration -> "Voice note · ${formatVoiceNoteDuration(duration)}" }
?: attachment.fileName,
style = mobileCaption1,
color = mobileText,
maxLines = 1,

View File

@@ -30,7 +30,7 @@ private val decodedBitmapCache =
internal fun loadSizedImageAttachment(
resolver: ContentResolver,
uri: Uri,
): PendingImageAttachment {
): PendingAttachment {
val fileName = normalizeAttachmentFileName((uri.lastPathSegment ?: "image").substringAfterLast('/'))
val bitmap = decodeScaledBitmap(resolver, uri, maxDimension = CHAT_ATTACHMENT_MAX_WIDTH)
if (bitmap == null) {
@@ -67,7 +67,7 @@ internal fun loadSizedImageAttachment(
},
)
val base64 = Base64.encodeToString(encoded.bytes, Base64.NO_WRAP)
return PendingImageAttachment(
return PendingAttachment(
id = uri.toString() + "#" + System.currentTimeMillis().toString(),
fileName = fileName,
mimeType = "image/jpeg",

View File

@@ -89,7 +89,7 @@ internal fun ChatMessageBubble(
when (part.type) {
"text" -> !part.text.isNullOrBlank()
"image" -> !part.base64.isNullOrBlank()
else -> false
else -> part.isAudioAttachment()
}
}
@@ -200,11 +200,12 @@ private fun ChatMessageBody(
) {
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
for (part in content) {
when (part.type) {
"text" -> {
when {
part.type == "text" -> {
val text = part.text ?: continue
ChatMarkdown(text = text, textColor = textColor)
}
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
else -> {
val b64 = part.base64 ?: continue
ChatBase64Image(base64 = b64, mimeType = part.mimeType)

View File

@@ -11,7 +11,7 @@ import ai.openclaw.app.chat.ChatPendingToolCall
import ai.openclaw.app.chat.ChatSessionEntry
import ai.openclaw.app.chat.MessageSpeechPhase
import ai.openclaw.app.chat.MessageSpeechState
import ai.openclaw.app.chat.OutgoingAttachment
import ai.openclaw.app.chat.VoiceNoteRecorderState
import ai.openclaw.app.resolveAgentIdFromMainSessionKey
import ai.openclaw.app.ui.copyGatewayDiagnosticsReport
import ai.openclaw.app.ui.design.ClawListItem
@@ -58,6 +58,7 @@ import androidx.compose.material.icons.filled.AttachFile
import androidx.compose.material.icons.filled.Close
import androidx.compose.material.icons.filled.Cloud
import androidx.compose.material.icons.filled.ContentCopy
import androidx.compose.material.icons.filled.GraphicEq
import androidx.compose.material.icons.filled.HourglassEmpty
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material.icons.filled.MoreHoriz
@@ -139,6 +140,11 @@ fun ChatScreen(
val modelFavorites by viewModel.modelFavorites.collectAsState()
val modelRecents by viewModel.modelRecents.collectAsState()
val selectedModelRef by viewModel.chatSelectedModelRef.collectAsState()
val micEnabled by viewModel.micEnabled.collectAsState()
val micIsListening by viewModel.micIsListening.collectAsState()
val micCooldown by viewModel.micCooldown.collectAsState()
val talkModeEnabled by viewModel.talkModeEnabled.collectAsState()
val talkModeListening by viewModel.talkModeListening.collectAsState()
val contextUsage = resolveChatContextUsage(sessionKey = sessionKey, mainSessionKey = mainSessionKey, sessions = sessions)
val gatewayAddress = gatewayDiagnosticsEndpoint(remoteAddress = remoteAddress, manualHost = manualHost, manualPort = manualPort, manualTls = manualTls)
val gatewayProblemMessage = gatewayConnectionDisplay.problem?.message?.takeIf { it.isNotBlank() }
@@ -149,7 +155,7 @@ fun ChatScreen(
val context = LocalContext.current
val resolver = context.contentResolver
val scope = rememberCoroutineScope()
val attachments = remember { mutableStateListOf<PendingImageAttachment>() }
val attachments = remember { mutableStateListOf<PendingAttachment>() }
var showModelPicker by rememberSaveable { mutableStateOf(false) }
DisposableEffect(viewModel) {
@@ -168,6 +174,14 @@ fun ChatScreen(
modelCatalog.firstOrNull { it.providerQualifiedRef() == selected }?.name?.takeIf { it.isNotBlank() }
?: selected.substringAfterLast('/')
} ?: "Model"
val micCaptureActive = micEnabled || micIsListening || micCooldown || talkModeEnabled || talkModeListening
val voiceNoteRecorder =
rememberVoiceNoteRecorderController(
viewModel = viewModel,
onFinished = attachments::add,
)
val voiceNoteState by voiceNoteRecorder.state.collectAsState()
val voiceNoteElapsedMs by voiceNoteRecorder.elapsedMs.collectAsState()
val pickImages =
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult
@@ -319,6 +333,12 @@ fun ChatScreen(
onOpenModelPicker = { showModelPicker = true },
onPickImages = { pickImages.launch("image/*") },
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
voiceNoteState = voiceNoteState,
voiceNoteElapsedMs = voiceNoteElapsedMs,
recordVoiceNoteEnabled = pendingRunCount == 0 && !micCaptureActive,
onStartVoiceNote = { scope.launch { voiceNoteRecorder.start() } },
onCancelVoiceNote = voiceNoteRecorder::cancel,
onFinishVoiceNote = voiceNoteRecorder::finish,
onVoice = onVoice,
onFixConnection = onOpenGatewaySettings,
onCopyDiagnostics = {
@@ -333,15 +353,7 @@ fun ChatScreen(
onSend = {
val message = input.trim()
if (message.isEmpty() && attachments.isEmpty()) return@ChatComposer
val outgoing =
attachments.map { attachment ->
OutgoingAttachment(
type = "image",
mimeType = attachment.mimeType,
fileName = attachment.fileName,
base64 = attachment.base64,
)
}
val outgoing = attachments.map(PendingAttachment::toOutgoingAttachment)
val pendingAttachments = attachments.toList()
input = ""
attachments.clear()
@@ -847,7 +859,7 @@ private fun ChatBubble(
when (part.type) {
"text" -> !part.text.isNullOrBlank()
"image" -> !part.base64.isNullOrBlank()
else -> false
else -> part.isAudioAttachment()
}
}
if (displayableContent.isEmpty()) return
@@ -896,10 +908,10 @@ private fun ChatBubble(
color = ClawTheme.colors.text,
)
displayableContent.forEach { part ->
if (part.type == "text") {
ChatText(text = part.text.orEmpty(), textColor = ClawTheme.colors.text, isStreaming = live)
} else {
Text(text = part.fileName ?: "Attachment", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
when {
part.type == "text" -> ChatText(text = part.text.orEmpty(), textColor = ClawTheme.colors.text, isStreaming = live)
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
else -> Text(text = part.fileName ?: "Attachment", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
}
}
if (messageId != null) {
@@ -1024,7 +1036,7 @@ private fun ChatNotice(
private fun ChatComposer(
value: String,
onValueChange: (String) -> Unit,
attachments: List<PendingImageAttachment>,
attachments: List<PendingAttachment>,
thinkingLevel: String,
contextUsage: ChatContextUsage,
selectedModelLabel: String,
@@ -1038,6 +1050,12 @@ private fun ChatComposer(
onOpenModelPicker: () -> Unit,
onPickImages: () -> Unit,
onRemoveAttachment: (String) -> Unit,
voiceNoteState: VoiceNoteRecorderState,
voiceNoteElapsedMs: Long,
recordVoiceNoteEnabled: Boolean,
onStartVoiceNote: () -> Unit,
onCancelVoiceNote: () -> Unit,
onFinishVoiceNote: () -> Unit,
onVoice: () -> Unit,
onFixConnection: () -> Unit,
onCopyDiagnostics: () -> Unit,
@@ -1080,17 +1098,32 @@ private fun ChatComposer(
}
Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(6.dp)) {
ChatInputPill(
value = value,
onValueChange = onValueChange,
onPickImages = onPickImages,
onVoice = onVoice,
modifier = Modifier.weight(1f),
)
if (voiceNoteState is VoiceNoteRecorderState.Recording) {
VoiceNoteRecordingControls(
elapsedMs = voiceNoteElapsedMs,
onCancel = onCancelVoiceNote,
onDone = onFinishVoiceNote,
modifier = Modifier.weight(1f),
)
} else if (voiceNoteState is VoiceNoteRecorderState.Preparing) {
VoiceNotePreparing(modifier = Modifier.weight(1f))
} else {
ChatInputPill(
value = value,
onValueChange = onValueChange,
onPickImages = onPickImages,
onStartVoiceNote = onStartVoiceNote,
recordVoiceNoteEnabled = recordVoiceNoteEnabled,
onVoice = onVoice,
modifier = Modifier.weight(1f),
)
}
SendButton(
// Offline, only text sends are enabled: they queue durably (text-only v1).
enabled =
pendingRunCount == 0 &&
voiceNoteState !is VoiceNoteRecorderState.Recording &&
voiceNoteState !is VoiceNoteRecorderState.Preparing &&
pendingRunCount == 0 &&
if (healthOk) {
value.trim().isNotEmpty() || attachments.isNotEmpty()
} else {
@@ -1100,6 +1133,8 @@ private fun ChatComposer(
)
}
VoiceNoteRecorderError(voiceNoteState)
if (!healthOk && gatewayOffline) {
ChatOfflineNotice(
status = offlineStatus,
@@ -1407,6 +1442,8 @@ private fun ChatInputPill(
value: String,
onValueChange: (String) -> Unit,
onPickImages: () -> Unit,
onStartVoiceNote: () -> Unit,
recordVoiceNoteEnabled: Boolean,
onVoice: () -> Unit,
modifier: Modifier = Modifier,
) {
@@ -1427,6 +1464,10 @@ private fun ChatInputPill(
Icon(imageVector = Icons.Default.AttachFile, contentDescription = "Attach image", modifier = Modifier.size(16.dp))
}
}
VoiceNoteRecordButton(
enabled = recordVoiceNoteEnabled,
onClick = onStartVoiceNote,
)
Box(modifier = Modifier.weight(1f)) {
BasicTextField(
value = value,
@@ -1454,7 +1495,7 @@ private fun ChatInputPill(
contentColor = ClawTheme.colors.text,
) {
Box(contentAlignment = Alignment.Center) {
Icon(imageVector = Icons.Default.Mic, contentDescription = "Voice", modifier = Modifier.size(18.dp))
Icon(imageVector = Icons.Default.GraphicEq, contentDescription = "Open voice", modifier = Modifier.size(18.dp))
}
}
}
@@ -1463,19 +1504,19 @@ private fun ChatInputPill(
@Composable
private fun AttachmentStrip(
attachments: List<PendingImageAttachment>,
attachments: List<PendingAttachment>,
onRemoveAttachment: (String) -> Unit,
) {
Row(modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), horizontalArrangement = Arrangement.spacedBy(6.dp)) {
attachments.forEach { attachment ->
AttachmentChip(fileName = attachment.fileName, onRemove = { onRemoveAttachment(attachment.id) })
AttachmentChip(attachment = attachment, onRemove = { onRemoveAttachment(attachment.id) })
}
}
}
@Composable
private fun AttachmentChip(
fileName: String,
attachment: PendingAttachment,
onRemove: () -> Unit,
) {
Surface(
@@ -1489,7 +1530,18 @@ private fun AttachmentChip(
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(6.dp),
) {
Text(text = fileName, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted, maxLines = 1, overflow = TextOverflow.Ellipsis)
if (attachment.mimeType.startsWith("audio/")) {
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(14.dp), tint = ClawTheme.colors.textMuted)
}
Text(
text =
attachment.durationMs?.let { duration -> "Voice note · ${formatVoiceNoteDuration(duration)}" }
?: attachment.fileName,
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
Surface(onClick = onRemove, modifier = Modifier.size(22.dp), shape = CircleShape, color = ClawTheme.colors.canvas, contentColor = ClawTheme.colors.text) {
Box(contentAlignment = Alignment.Center) {
Icon(imageVector = Icons.Default.Close, contentDescription = "Remove attachment", modifier = Modifier.size(13.dp))

View File

@@ -2,7 +2,6 @@ package ai.openclaw.app.ui.chat
import ai.openclaw.app.MainViewModel
import ai.openclaw.app.chat.ChatSessionEntry
import ai.openclaw.app.chat.OutgoingAttachment
import ai.openclaw.app.ui.mobileAccent
import ai.openclaw.app.ui.mobileAccentBorderStrong
import ai.openclaw.app.ui.mobileBorderStrong
@@ -91,6 +90,11 @@ fun ChatSheetContent(viewModel: MainViewModel) {
val messageSpeechState by viewModel.chatMessageSpeech.collectAsState()
val gatewayConnectionDisplay by viewModel.gatewayConnectionDisplay.collectAsState()
val gatewayOffline = !gatewayConnectionDisplay.isConnected
val micEnabled by viewModel.micEnabled.collectAsState()
val micIsListening by viewModel.micIsListening.collectAsState()
val micCooldown by viewModel.micCooldown.collectAsState()
val talkModeEnabled by viewModel.talkModeEnabled.collectAsState()
val talkModeListening by viewModel.talkModeListening.collectAsState()
DisposableEffect(viewModel) {
onDispose(viewModel::stopChatMessageSpeech)
@@ -124,7 +128,15 @@ fun ChatSheetContent(viewModel: MainViewModel) {
val resolver = context.contentResolver
val scope = rememberCoroutineScope()
val attachments = remember { mutableStateListOf<PendingImageAttachment>() }
val attachments = remember { mutableStateListOf<PendingAttachment>() }
val micCaptureActive = micEnabled || micIsListening || micCooldown || talkModeEnabled || talkModeListening
val voiceNoteRecorder =
rememberVoiceNoteRecorderController(
viewModel = viewModel,
onFinished = attachments::add,
)
val voiceNoteState by voiceNoteRecorder.state.collectAsState()
val voiceNoteElapsedMs by voiceNoteRecorder.elapsedMs.collectAsState()
val pickImages =
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
@@ -198,6 +210,12 @@ fun ChatSheetContent(viewModel: MainViewModel) {
onDraftApplied = viewModel::clearChatDraft,
onPickImages = { pickImages.launch("image/*") },
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
voiceNoteState = voiceNoteState,
voiceNoteElapsedMs = voiceNoteElapsedMs,
recordVoiceNoteEnabled = pendingRunCount == 0 && !micCaptureActive,
onStartVoiceNote = { scope.launch { voiceNoteRecorder.start() } },
onCancelVoiceNote = voiceNoteRecorder::cancel,
onFinishVoiceNote = voiceNoteRecorder::finish,
onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) },
onRefresh = {
viewModel.refreshChat()
@@ -206,15 +224,7 @@ fun ChatSheetContent(viewModel: MainViewModel) {
},
onAbort = { viewModel.abortChat() },
onSend = { text ->
val outgoing =
attachments.map { att ->
OutgoingAttachment(
type = "image",
mimeType = att.mimeType,
fileName = att.fileName,
base64 = att.base64,
)
}
val outgoing = attachments.map(PendingAttachment::toOutgoingAttachment)
val pendingAttachments = attachments.toList()
attachments.clear()
val accepted = viewModel.sendChatAwaitAcceptance(message = text, thinking = thinkingLevel, attachments = outgoing)
@@ -286,13 +296,3 @@ private fun ChatErrorRail(errorText: String) {
}
}
}
/**
* Image selected in the composer and held in memory until the next chat.send call.
*/
data class PendingImageAttachment(
val id: String,
val fileName: String,
val mimeType: String,
val base64: String,
)

View File

@@ -124,6 +124,8 @@ private fun stableMessageVersion(message: ChatMessage): String {
append(content.fileName.orEmpty())
append(',')
append(content.base64?.length ?: 0)
append(',')
append(content.durationMs ?: "")
}
}
}
@@ -163,6 +165,8 @@ private fun latestContentVersion(
append(content.fileName.orEmpty())
append(',')
append(content.base64?.length ?: 0)
append(',')
append(content.durationMs ?: "")
}
append(":runs=")
append(pendingRunCount)

View File

@@ -0,0 +1,51 @@
package ai.openclaw.app.ui.chat
import ai.openclaw.app.chat.OutgoingAttachment
import ai.openclaw.app.chat.VOICE_NOTE_MIME_TYPE
import ai.openclaw.app.chat.VoiceNoteRecording
import java.util.Base64
/** Attachment staged in a composer until the next chat.send call. */
data class PendingAttachment(
val id: String,
val fileName: String,
val mimeType: String,
val base64: String,
val durationMs: Long? = null,
)
internal fun PendingAttachment.toOutgoingAttachment(): OutgoingAttachment =
OutgoingAttachment(
type = attachmentTypeForMimeType(mimeType),
mimeType = mimeType,
fileName = fileName,
base64 = base64,
durationMs = durationMs,
)
internal fun attachmentTypeForMimeType(mimeType: String): String =
when {
mimeType.startsWith("audio/") -> "audio"
mimeType.startsWith("image/") -> "image"
else -> "file"
}
internal fun stageVoiceNoteAttachment(recording: VoiceNoteRecording): PendingAttachment =
try {
PendingAttachment(
id = recording.file.absolutePath + "#" + recording.durationMs,
fileName = recording.file.name,
mimeType = VOICE_NOTE_MIME_TYPE,
base64 = Base64.getEncoder().encodeToString(recording.file.readBytes()),
durationMs = recording.durationMs,
)
} finally {
recording.file.delete()
}
internal fun formatVoiceNoteDuration(durationMs: Long): String {
val totalSeconds = durationMs.coerceAtLeast(0L) / 1_000L
val minutes = totalSeconds / 60L
val seconds = totalSeconds % 60L
return "$minutes:${seconds.toString().padStart(2, '0')}"
}

View File

@@ -0,0 +1,214 @@
package ai.openclaw.app.ui.chat
import ai.openclaw.app.MainViewModel
import ai.openclaw.app.chat.AndroidVoiceNoteRecordingEngine
import ai.openclaw.app.chat.ChatMessageContent
import ai.openclaw.app.chat.VoiceNoteRecorderController
import ai.openclaw.app.chat.VoiceNoteRecorderState
import ai.openclaw.app.ui.design.ClawTheme
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Check
import androidx.compose.material.icons.filled.Close
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material3.Icon
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleEventObserver
import androidx.lifecycle.compose.LocalLifecycleOwner
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
@Composable
internal fun rememberVoiceNoteRecorderController(
viewModel: MainViewModel,
onFinished: (PendingAttachment) -> Unit,
): VoiceNoteRecorderController {
val context = LocalContext.current.applicationContext
val lifecycleOwner = LocalLifecycleOwner.current
val scope = rememberCoroutineScope()
val currentOnFinished by rememberUpdatedState(onFinished)
lateinit var controller: VoiceNoteRecorderController
controller =
remember(context, viewModel, scope) {
VoiceNoteRecorderController(
scope = scope,
outputDirectory = context.cacheDir,
engine = AndroidVoiceNoteRecordingEngine(context),
requestPermission = viewModel::requestVoiceNotePermission,
acquireMic = viewModel::tryAcquireVoiceNoteMic,
releaseMic = viewModel::releaseVoiceNoteMic,
onFinished = { recording ->
scope.launch(Dispatchers.IO) {
val attachment = runCatching { stageVoiceNoteAttachment(recording) }
withContext(Dispatchers.Main) {
attachment.fold(
onSuccess = {
currentOnFinished(it)
controller.completePreparation()
},
onFailure = { controller.reportFailure("Could not prepare voice note.") },
)
}
}
},
)
}
DisposableEffect(controller, lifecycleOwner) {
val observer =
LifecycleEventObserver { _, event ->
if (event == Lifecycle.Event.ON_STOP) controller.cancel()
}
lifecycleOwner.lifecycle.addObserver(observer)
onDispose {
lifecycleOwner.lifecycle.removeObserver(observer)
controller.cancel()
}
}
return controller
}
@Composable
internal fun VoiceNotePreparing(modifier: Modifier = Modifier) {
Surface(
modifier = modifier.fillMaxWidth().heightIn(min = ClawTheme.spacing.touchTarget),
shape = RoundedCornerShape(ClawTheme.radii.control),
color = ClawTheme.colors.surfaceRaised,
contentColor = ClawTheme.colors.textSubtle,
border = BorderStroke(1.dp, ClawTheme.colors.border),
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(10.dp),
) {
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(18.dp))
Text(text = "Preparing voice note…", style = ClawTheme.type.label)
}
}
}
@Composable
internal fun VoiceNoteRecordButton(
enabled: Boolean,
onClick: () -> Unit,
modifier: Modifier = Modifier,
) {
Surface(
onClick = onClick,
enabled = enabled,
modifier = modifier.size(ClawTheme.spacing.touchTarget),
shape = CircleShape,
color = ClawTheme.colors.surfaceRaised,
contentColor = if (enabled) ClawTheme.colors.text else ClawTheme.colors.textSubtle,
) {
Box(contentAlignment = Alignment.Center) {
Icon(imageVector = Icons.Default.Mic, contentDescription = "Record voice note", modifier = Modifier.size(18.dp))
}
}
}
@Composable
internal fun VoiceNoteRecordingControls(
elapsedMs: Long,
onCancel: () -> Unit,
onDone: () -> Unit,
modifier: Modifier = Modifier,
) {
Surface(
modifier = modifier.fillMaxWidth().heightIn(min = ClawTheme.spacing.touchTarget),
shape = RoundedCornerShape(ClawTheme.radii.control),
color = ClawTheme.colors.surfaceRaised,
contentColor = ClawTheme.colors.text,
border = BorderStroke(1.dp, ClawTheme.colors.border),
) {
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(10.dp),
) {
Box(modifier = Modifier.size(8.dp).background(ClawTheme.colors.danger, CircleShape))
Text(
text = formatVoiceNoteDuration(elapsedMs),
style = ClawTheme.type.label.copy(fontWeight = FontWeight.SemiBold),
modifier = Modifier.weight(1f),
)
Surface(
onClick = onCancel,
modifier = Modifier.size(36.dp),
shape = CircleShape,
color = ClawTheme.colors.canvas,
contentColor = ClawTheme.colors.text,
) {
Box(contentAlignment = Alignment.Center) {
Icon(imageVector = Icons.Default.Close, contentDescription = "Cancel voice note", modifier = Modifier.size(17.dp))
}
}
Surface(
onClick = onDone,
modifier = Modifier.size(36.dp),
shape = CircleShape,
color = ClawTheme.colors.primary,
contentColor = ClawTheme.colors.primaryText,
) {
Box(contentAlignment = Alignment.Center) {
Icon(imageVector = Icons.Default.Check, contentDescription = "Finish voice note", modifier = Modifier.size(17.dp))
}
}
}
}
}
@Composable
internal fun VoiceNoteRecorderError(state: VoiceNoteRecorderState) {
val message = (state as? VoiceNoteRecorderState.Failure)?.message ?: return
Text(text = message, style = ClawTheme.type.caption, color = ClawTheme.colors.danger)
}
internal fun ChatMessageContent.isAudioAttachment(): Boolean = type == "audio" || mimeType?.startsWith("audio/") == true
@Composable
internal fun VoiceNoteMessageRow(durationMs: Long?) {
Row(
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Icon(
imageVector = Icons.Default.Mic,
contentDescription = null,
modifier = Modifier.size(16.dp),
tint = ClawTheme.colors.textMuted,
)
Text(text = "Voice note", style = ClawTheme.type.body, color = ClawTheme.colors.text)
durationMs?.let { duration ->
Text(
text = formatVoiceNoteDuration(duration),
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
)
}
}
}

View File

@@ -847,6 +847,32 @@ class GatewayBootstrapAuthTest {
assertFalse(runtime.prefs.voiceMicEnabled.value)
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun voiceNoteMicOwnershipBlocksLocalVoiceAndGatewayPtt() =
runBlocking {
val app = RuntimeEnvironment.getApplication()
shadowOf(app).grantPermissions(Manifest.permission.RECORD_AUDIO)
val runtime = createTestRuntime(app)
val dispatcher = readField<InvokeDispatcher>(runtime, "invokeDispatcher")
Dispatchers.setMain(Dispatchers.Unconfined)
try {
assertTrue(runtime.tryAcquireVoiceNoteMic())
runtime.setMicEnabled(true)
runtime.setTalkModeEnabled(true)
val ptt = dispatcher.handleInvoke(OpenClawTalkCommand.PttStart.rawValue, null)
assertEquals(VoiceCaptureMode.Off, runtime.voiceCaptureMode.value)
assertEquals("MIC_BUSY", ptt.error?.code)
assertEquals("MIC_BUSY: voice note recording is active", ptt.error?.message)
assertFalse(readField<MutableStateFlow<Boolean>>(runtime, "externalAudioCaptureActive").value)
} finally {
runtime.releaseVoiceNoteMic()
Dispatchers.resetMain()
}
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun talkPttStart_cleansPreparedCaptureWhenBeginFails() =

View File

@@ -153,6 +153,63 @@ class ChatControllerMessageIdentityTest {
assertNotEquals(reconciled[0].id, reconciled[1].id)
}
@Test
fun reconcileMessageIdsPreservesOptimisticVoiceNoteDuration() {
val previous =
ChatMessage(
id = "local-user",
role = "user",
content =
listOf(
ChatMessageContent(type = "text", text = "See attached."),
ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "voice-note.m4a", durationMs = 4_321L),
),
timestampMs = 1_000L,
idempotencyKey = "run:user",
)
val incoming =
previous.copy(
id = "gateway-user",
content = previous.content.map { it.copy(durationMs = null) },
)
val reconciled = reconcileMessageIds(previous = listOf(previous), incoming = listOf(incoming)).single()
assertEquals("local-user", reconciled.id)
assertEquals(4_321L, reconciled.content[1].durationMs)
}
@Test
fun reconcileMessageIdsPreservesMultipleVoiceNoteDurationsInOrder() {
val previous =
ChatMessage(
id = "local-user",
role = "user",
content =
listOf(
ChatMessageContent(type = "text", text = "See attached."),
ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "first.m4a", durationMs = 1_000L),
ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "second.m4a", durationMs = 2_000L),
),
timestampMs = 1_000L,
idempotencyKey = "run:user",
)
val incoming =
previous.copy(
id = "gateway-user",
content =
listOf(
ChatMessageContent(type = "text", text = "See attached."),
ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "stored-a.m4a"),
ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "stored-b.m4a"),
),
)
val reconciled = reconcileMessageIds(previous = listOf(previous), incoming = listOf(incoming)).single()
assertEquals(listOf(1_000L, 2_000L), reconciled.content.drop(1).map { it.durationMs })
}
@Test
fun mergeOptimisticMessagesKeepsOutgoingUserTurnWhenHistoryOmitsIt() {
val optimistic =

View File

@@ -1,6 +1,7 @@
package ai.openclaw.app.chat
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
@@ -46,4 +47,42 @@ class ChatMessageContentParsingTest {
parseChatMessageContent(managedImage),
)
}
@Test
fun parsesDirectAndAttachmentAudioBlocks() {
val direct =
Json.parseToJsonElement(
"""{"type":"audio","mimeType":"audio/mp4","fileName":"voice.m4a"}""",
)
val attachment =
Json.parseToJsonElement(
"""{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mpeg","label":"reply.mp3"}}""",
)
assertEquals(
ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "voice.m4a"),
parseChatMessageContent(direct),
)
assertEquals(
ChatMessageContent(type = "audio", mimeType = "audio/mpeg", fileName = "reply.mp3"),
parseChatMessageContent(attachment),
)
}
@Test
fun parsesTranscriptAudioMediaFieldsAlongsideCaption() {
val message =
Json
.parseToJsonElement(
"""{"content":[{"type":"text","text":"See attached."}],"MediaPaths":["media/inbound/voice.m4a"],"MediaTypes":["audio/x-m4a"]}""",
).jsonObject
assertEquals(
listOf(
ChatMessageContent(type = "text", text = "See attached."),
ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "voice.m4a"),
),
parseChatMessageContents(message),
)
}
}

View File

@@ -0,0 +1,74 @@
package ai.openclaw.app.chat
import ai.openclaw.app.ui.chat.stageVoiceNoteAttachment
import ai.openclaw.app.ui.chat.toOutgoingAttachment
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.runCurrent
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import java.nio.file.Files
@OptIn(ExperimentalCoroutinesApi::class)
class ChatVoiceNoteAttachmentTest {
@Test
fun stagedVoiceNoteUsesAudioPayloadAndDurationInLocalEcho() =
runTest {
val file = Files.createTempFile("voice-note-", ".m4a").toFile()
file.writeBytes("voice-bytes".encodeToByteArray())
val pending = stageVoiceNoteAttachment(VoiceNoteRecording(file = file, durationMs = 12_345L))
val outgoing = pending.toOutgoingAttachment()
var sentParams: JsonObject? = null
val json = Json { ignoreUnknownKeys = true }
val chat =
ChatController(
scope = backgroundScope,
json = json,
requestGateway = { method, paramsJson ->
when (method) {
"chat.send" -> {
sentParams = json.parseToJsonElement(paramsJson.orEmpty()) as JsonObject
"""{"runId":"voice-run","status":"started"}"""
}
else -> "{}"
}
},
)
chat.handleGatewayEvent("health", null)
runCurrent()
val accepted =
chat.sendMessageAwaitAcceptance(
message = "listen",
thinkingLevel = "off",
attachments = listOf(outgoing),
)
assertTrue(accepted)
assertFalse(file.exists())
assertEquals("audio", outgoing.type)
assertEquals(VOICE_NOTE_MIME_TYPE, outgoing.mimeType)
assertTrue(outgoing.fileName.endsWith(".m4a"))
assertEquals("dm9pY2UtYnl0ZXM=", outgoing.base64)
val sentAttachment = ((sentParams?.get("attachments") as JsonArray).single() as JsonObject)
assertEquals("audio", (sentAttachment["type"] as JsonPrimitive).content)
assertEquals(VOICE_NOTE_MIME_TYPE, (sentAttachment["mimeType"] as JsonPrimitive).content)
assertTrue((sentAttachment["fileName"] as JsonPrimitive).content.endsWith(".m4a"))
assertEquals("dm9pY2UtYnl0ZXM=", (sentAttachment["content"] as JsonPrimitive).content)
val echoedAudio =
chat.messages.value
.single()
.content
.single { it.type == "audio" }
assertEquals(VOICE_NOTE_MIME_TYPE, echoedAudio.mimeType)
assertEquals(12_345L, echoedAudio.durationMs)
}
}

View File

@@ -0,0 +1,294 @@
package ai.openclaw.app.chat
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.advanceTimeBy
import kotlinx.coroutines.test.runCurrent
import kotlinx.coroutines.test.runTest
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import java.io.File
import java.nio.file.Files
@OptIn(ExperimentalCoroutinesApi::class)
class VoiceNoteRecorderControllerTest {
private class FakeEngine(
var durationMs: Long = 1_200L,
var outputBytes: ByteArray = byteArrayOf(1, 2, 3),
) : VoiceNoteRecordingEngine {
var startCount = 0
var stopCount = 0
var cancelCount = 0
var outputFile: File? = null
override fun start(outputFile: File) {
startCount += 1
this.outputFile = outputFile
outputFile.writeBytes(outputBytes)
}
override fun stop(): Long {
stopCount += 1
return durationMs
}
override fun cancel() {
cancelCount += 1
}
}
@Test
fun startTransitionsToRecordingAndPublishesElapsedTime() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
var now = 1_000L
val engine = FakeEngine()
val controller = controller(directory, engine, elapsedRealtimeMillis = { now })
assertTrue(controller.start())
assertEquals(VoiceNoteRecorderState.Recording(startedAtMillis = 1_000L), controller.state.value)
now = 3_500L
advanceTimeBy(250L)
runCurrent()
assertEquals(2_500L, controller.elapsedMs.value)
controller.cancel()
directory.deleteRecursively()
}
@Test
fun stopReturnsRetainedFileAndDuration() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine(durationMs = 4_321L)
val finished = mutableListOf<VoiceNoteRecording>()
val controller = controller(directory, engine, onFinished = finished::add)
controller.start()
assertTrue(controller.finish())
val recording = finished.single()
assertEquals(4_321L, recording.durationMs)
assertTrue(recording.file.exists())
assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value)
controller.completePreparation()
assertEquals(VoiceNoteRecorderState.Idle, controller.state.value)
recording.file.delete()
directory.deleteRecursively()
}
@Test
fun stopMarksMpeg4ContainerAsM4aForGatewaySniffing() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val mpeg4 = ByteArray(24)
"ftypmp42".toByteArray(Charsets.US_ASCII).copyInto(mpeg4, destinationOffset = 4)
val engine = FakeEngine(outputBytes = mpeg4)
val finished = mutableListOf<VoiceNoteRecording>()
val controller = controller(directory, engine, onFinished = finished::add)
controller.start()
controller.finish()
val recording = finished.single()
val majorBrand =
recording.file
.readBytes()
.copyOfRange(8, 12)
.toString(Charsets.US_ASCII)
assertEquals("M4A ", majorBrand)
recording.file.delete()
directory.deleteRecursively()
}
@Test
fun cancelDeletesTemporaryFileAndReturnsIdle() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine()
val controller = controller(directory, engine)
controller.start()
val file = requireNotNull(engine.outputFile)
controller.cancel()
assertFalse(file.exists())
assertEquals(1, engine.cancelCount)
assertEquals(VoiceNoteRecorderState.Idle, controller.state.value)
directory.deleteRecursively()
}
@Test
fun durationCapUsesNormalFinishPath() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
var now = 1_000L
val engine = FakeEngine(durationMs = VOICE_NOTE_MAX_DURATION_MS)
val finished = mutableListOf<VoiceNoteRecording>()
val controller = controller(directory, engine, onFinished = finished::add, elapsedRealtimeMillis = { now })
controller.start()
now += VOICE_NOTE_MAX_DURATION_MS
advanceTimeBy(250L)
runCurrent()
assertEquals(1, engine.stopCount)
assertEquals(VOICE_NOTE_MAX_DURATION_MS, finished.single().durationMs)
finished.single().file.delete()
directory.deleteRecursively()
}
@Test
fun oversizeRecordingFailsAndDeletesFile() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine(outputBytes = ByteArray(VOICE_NOTE_MAX_BYTES.toInt() + 1))
val finished = mutableListOf<VoiceNoteRecording>()
val controller = controller(directory, engine, onFinished = finished::add)
controller.start()
val file = requireNotNull(engine.outputFile)
assertFalse(controller.finish())
assertFalse(file.exists())
assertTrue(finished.isEmpty())
assertEquals(
VoiceNoteRecorderState.Failure("Voice note is too large. Record a shorter message."),
controller.state.value,
)
directory.deleteRecursively()
}
@Test
fun startIsRefusedWhileAlreadyRecording() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine()
val controller = controller(directory, engine)
assertTrue(controller.start())
assertFalse(controller.start())
assertEquals(1, engine.startCount)
controller.cancel()
directory.deleteRecursively()
}
@Test
fun startIsRefusedWhilePreparingAttachment() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine()
val finished = mutableListOf<VoiceNoteRecording>()
val controller = controller(directory, engine, onFinished = finished::add)
controller.start()
controller.finish()
assertFalse(controller.start())
assertEquals(1, engine.startCount)
assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value)
finished.single().file.delete()
directory.deleteRecursively()
}
@Test
fun cancelDuringPreparingDeletesHandedOffFile() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine()
val finished = mutableListOf<VoiceNoteRecording>()
val controller = controller(directory, engine, onFinished = finished::add)
controller.start()
controller.finish()
assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value)
assertTrue(finished.single().file.exists())
// Composition-scoped staging may be cancelled before it runs; cancel()
// must still delete the handed-off recording.
controller.cancel()
assertEquals(VoiceNoteRecorderState.Idle, controller.state.value)
assertFalse(finished.single().file.exists())
directory.deleteRecursively()
}
@Test
fun reportFailureDuringPreparingDeletesHandedOffFile() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine()
val finished = mutableListOf<VoiceNoteRecording>()
val controller = controller(directory, engine, onFinished = finished::add)
controller.start()
controller.finish()
controller.reportFailure("Could not prepare voice note.")
assertEquals(
VoiceNoteRecorderState.Failure("Could not prepare voice note."),
controller.state.value,
)
assertFalse(finished.single().file.exists())
directory.deleteRecursively()
}
@Test
fun startIsRefusedWhileVoiceCaptureOwnsMic() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine()
val controller = controller(directory, engine, acquireMic = { false })
assertFalse(controller.start())
assertEquals(0, engine.startCount)
assertEquals(
VoiceNoteRecorderState.Failure("Voice capture is already using the microphone."),
controller.state.value,
)
directory.deleteRecursively()
}
@Test
fun permissionDeniedIsUserVisibleAndDoesNotStartEngine() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine()
val controller = controller(directory, engine, requestPermission = { false })
assertFalse(controller.start())
assertEquals(0, engine.startCount)
assertEquals(
VoiceNoteRecorderState.Failure("Microphone permission is required to record a voice note."),
controller.state.value,
)
directory.deleteRecursively()
}
private fun kotlinx.coroutines.test.TestScope.controller(
directory: File,
engine: FakeEngine,
requestPermission: suspend () -> Boolean = { true },
acquireMic: () -> Boolean = { true },
releaseMic: () -> Unit = {},
onFinished: (VoiceNoteRecording) -> Unit = {},
elapsedRealtimeMillis: () -> Long = { 1_000L },
): VoiceNoteRecorderController =
VoiceNoteRecorderController(
scope = this,
outputDirectory = directory,
engine = engine,
requestPermission = requestPermission,
acquireMic = acquireMic,
releaseMic = releaseMic,
onFinished = onFinished,
elapsedRealtimeMillis = elapsedRealtimeMillis,
)
}