mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 02:43:57 +00:00
feat(android): record and send voice notes from the chat composer (#101193)
* feat(android): record and send voice notes from the chat composer Adds tap-to-record voice notes to both Android chat composers: MediaRecorder AAC/m4a mono capture with a 3-minute cap, lazy RECORD_AUDIO permission, cancel/finish controls, audio attachment chips with duration, and voice-note transcript rows. Recording is mutually exclusive with MicCaptureManager voice capture (no mic arbiter exists), and the MPEG-4 container brand is normalized to M4A so gateway byte sniffing classifies the upload as audio, not video. Attachments stay online-only per the outbox contract. Related: #100709 * fix(android): coordinate voice note microphone ownership * chore(android): refresh native localization inventory
This commit is contained in:
committed by
GitHub
parent
66c764b269
commit
ef1767baca
@@ -139,7 +139,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-state-text",
|
||||
"line": 121,
|
||||
"line": 125,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
|
||||
"source": "Searching…",
|
||||
"surface": "android",
|
||||
@@ -147,7 +147,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-state-text",
|
||||
"line": 212,
|
||||
"line": 216,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
|
||||
"source": "Mic off",
|
||||
"surface": "android",
|
||||
@@ -155,7 +155,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-state-text",
|
||||
"line": 222,
|
||||
"line": 226,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
|
||||
"source": "Off",
|
||||
"surface": "android",
|
||||
@@ -203,7 +203,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 2683,
|
||||
"line": 2701,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt",
|
||||
"source": "Failed: this host requires wss:// or Tailscale Serve. No TLS endpoint detected.",
|
||||
"surface": "android",
|
||||
@@ -211,7 +211,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 2685,
|
||||
"line": 2703,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt",
|
||||
"source": "Failed: secure endpoint reached, but TLS fingerprint verification timed out. Check Tailscale Serve or gateway TLS and retry.",
|
||||
"surface": "android",
|
||||
@@ -219,7 +219,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 2687,
|
||||
"line": 2705,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt",
|
||||
"source": "Failed: couldn't reach the secure gateway endpoint for this host.",
|
||||
"surface": "android",
|
||||
@@ -5763,7 +5763,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 199,
|
||||
"line": 218,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
|
||||
"source": "Type a message…",
|
||||
"surface": "android",
|
||||
@@ -5771,7 +5771,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 209,
|
||||
"line": 231,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
|
||||
"source": "Gateway is offline. Text messages are queued and sent after reconnecting.",
|
||||
"surface": "android",
|
||||
@@ -5779,7 +5779,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 238,
|
||||
"line": 260,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
|
||||
"source": "Select thinking level",
|
||||
"surface": "android",
|
||||
@@ -5787,7 +5787,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 262,
|
||||
"line": 284,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
|
||||
"source": "Attach",
|
||||
"surface": "android",
|
||||
@@ -5795,7 +5795,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 270,
|
||||
"line": 299,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
|
||||
"source": "Refresh",
|
||||
"surface": "android",
|
||||
@@ -5803,7 +5803,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 278,
|
||||
"line": 307,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
|
||||
"source": "Abort",
|
||||
"surface": "android",
|
||||
@@ -5811,7 +5811,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 323,
|
||||
"line": 356,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
|
||||
"source": "Send",
|
||||
"surface": "android",
|
||||
@@ -5819,7 +5819,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 349,
|
||||
"line": 382,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatComposer.kt",
|
||||
"source": "No commands found",
|
||||
"surface": "android",
|
||||
@@ -5979,7 +5979,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 260,
|
||||
"line": 261,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Preview · $domain",
|
||||
"surface": "android",
|
||||
@@ -5987,7 +5987,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 269,
|
||||
"line": 270,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Expand link preview",
|
||||
"surface": "android",
|
||||
@@ -5995,7 +5995,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 293,
|
||||
"line": 294,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Loading preview…",
|
||||
"surface": "android",
|
||||
@@ -6003,7 +6003,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 294,
|
||||
"line": 295,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "No preview available",
|
||||
"surface": "android",
|
||||
@@ -6011,7 +6011,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 339,
|
||||
"line": 340,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Thinking...",
|
||||
"surface": "android",
|
||||
@@ -6019,7 +6019,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 355,
|
||||
"line": 356,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Tools",
|
||||
"surface": "android",
|
||||
@@ -6027,7 +6027,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 358,
|
||||
"line": 359,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Running tools...",
|
||||
"surface": "android",
|
||||
@@ -6035,7 +6035,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 361,
|
||||
"line": 362,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "${display.emoji} ${display.label}",
|
||||
"surface": "android",
|
||||
@@ -6043,7 +6043,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 379,
|
||||
"line": 380,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "... +${toolCalls.size - 6} more",
|
||||
"surface": "android",
|
||||
@@ -6051,7 +6051,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 410,
|
||||
"line": 411,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "You",
|
||||
"surface": "android",
|
||||
@@ -6059,7 +6059,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 424,
|
||||
"line": 425,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Retry",
|
||||
"surface": "android",
|
||||
@@ -6067,7 +6067,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 427,
|
||||
"line": 428,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Delete",
|
||||
"surface": "android",
|
||||
@@ -6075,7 +6075,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 459,
|
||||
"line": 460,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "OpenClaw · Live",
|
||||
"surface": "android",
|
||||
@@ -6083,7 +6083,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 496,
|
||||
"line": 497,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "System",
|
||||
"surface": "android",
|
||||
@@ -6091,7 +6091,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 497,
|
||||
"line": 498,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "OpenClaw",
|
||||
"surface": "android",
|
||||
@@ -6099,7 +6099,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 523,
|
||||
"line": 524,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt",
|
||||
"source": "Unsupported attachment",
|
||||
"surface": "android",
|
||||
@@ -6107,7 +6107,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 276,
|
||||
"line": 290,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Chat needs attention",
|
||||
"surface": "android",
|
||||
@@ -6115,7 +6115,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 436,
|
||||
"line": 448,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "All",
|
||||
"surface": "android",
|
||||
@@ -6123,7 +6123,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 494,
|
||||
"line": 506,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "OpenClaw",
|
||||
"surface": "android",
|
||||
@@ -6131,7 +6131,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 515,
|
||||
"line": 527,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "New chat",
|
||||
"surface": "android",
|
||||
@@ -6139,7 +6139,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 520,
|
||||
"line": 532,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "More new chat options",
|
||||
"surface": "android",
|
||||
@@ -6147,7 +6147,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 535,
|
||||
"line": 547,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Refresh chat",
|
||||
"surface": "android",
|
||||
@@ -6155,7 +6155,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 538,
|
||||
"line": 550,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Chat",
|
||||
"surface": "android",
|
||||
@@ -6163,7 +6163,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 686,
|
||||
"line": 698,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Loading session",
|
||||
"surface": "android",
|
||||
@@ -6171,7 +6171,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 713,
|
||||
"line": 725,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Jump to latest",
|
||||
"surface": "android",
|
||||
@@ -6179,7 +6179,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 739,
|
||||
"line": 751,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Ready when you are",
|
||||
"surface": "android",
|
||||
@@ -6187,7 +6187,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 770,
|
||||
"line": 782,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Fix connection",
|
||||
"surface": "android",
|
||||
@@ -6195,7 +6195,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 771,
|
||||
"line": 783,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Copy diagnostics",
|
||||
"surface": "android",
|
||||
@@ -6203,7 +6203,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 955,
|
||||
"line": 967,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Preparing audio…",
|
||||
"surface": "android",
|
||||
@@ -6211,7 +6211,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 955,
|
||||
"line": 967,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Speaking…",
|
||||
"surface": "android",
|
||||
@@ -6219,7 +6219,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 976,
|
||||
"line": 988,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Tools running",
|
||||
"surface": "android",
|
||||
@@ -6227,7 +6227,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 978,
|
||||
"line": 990,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "OpenClaw is working",
|
||||
"surface": "android",
|
||||
@@ -6235,7 +6235,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 981,
|
||||
"line": 993,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "+${toolCalls.size - 4} more",
|
||||
"surface": "android",
|
||||
@@ -6243,7 +6243,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 991,
|
||||
"line": 1003,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Thinking",
|
||||
"surface": "android",
|
||||
@@ -6251,7 +6251,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 992,
|
||||
"line": 1004,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "OpenClaw is preparing a response.",
|
||||
"surface": "android",
|
||||
@@ -6259,7 +6259,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1126,
|
||||
"line": 1161,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Stop",
|
||||
"surface": "android",
|
||||
@@ -6267,7 +6267,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1192,
|
||||
"line": 1227,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Default",
|
||||
"surface": "android",
|
||||
@@ -6275,7 +6275,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 1258,
|
||||
"line": 1293,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Pin model",
|
||||
"surface": "android",
|
||||
@@ -6283,7 +6283,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 1258,
|
||||
"line": 1293,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Unpin model",
|
||||
"surface": "android",
|
||||
@@ -6291,7 +6291,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1275,
|
||||
"line": 1310,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "No commands found",
|
||||
"surface": "android",
|
||||
@@ -6299,7 +6299,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1337,
|
||||
"line": 1372,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Gateway offline",
|
||||
"surface": "android",
|
||||
@@ -6307,7 +6307,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1427,
|
||||
"line": 1464,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Attach image",
|
||||
"surface": "android",
|
||||
@@ -6315,7 +6315,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1442,
|
||||
"line": 1483,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Message OpenClaw",
|
||||
"surface": "android",
|
||||
@@ -6323,15 +6323,15 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1457,
|
||||
"line": 1498,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Voice",
|
||||
"source": "Open voice",
|
||||
"surface": "android",
|
||||
"id": "native.android.19396391033f4fa8"
|
||||
"id": "native.android.97602a28629a331f"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1495,
|
||||
"line": 1547,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Remove attachment",
|
||||
"surface": "android",
|
||||
@@ -6339,7 +6339,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 1516,
|
||||
"line": 1568,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Main",
|
||||
"surface": "android",
|
||||
@@ -6347,7 +6347,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 1572,
|
||||
"line": 1624,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "Send",
|
||||
"surface": "android",
|
||||
@@ -6355,7 +6355,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 1611,
|
||||
"line": 1663,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt",
|
||||
"source": "$contextLabel · ${contextMeterThinkingLabel(thinkingLevel)}",
|
||||
"surface": "android",
|
||||
@@ -6363,12 +6363,52 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 281,
|
||||
"line": 291,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatSheetContent.kt",
|
||||
"source": "CHAT ERROR",
|
||||
"surface": "android",
|
||||
"id": "native.android.826804c0189e03d0"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 109,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
|
||||
"source": "Preparing voice note…",
|
||||
"surface": "android",
|
||||
"id": "native.android.3df4e24a0fc8810b"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 129,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
|
||||
"source": "Record voice note",
|
||||
"surface": "android",
|
||||
"id": "native.android.06ab95879f652d3e"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 167,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
|
||||
"source": "Cancel voice note",
|
||||
"surface": "android",
|
||||
"id": "native.android.8a7653317387033f"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 178,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
|
||||
"source": "Finish voice note",
|
||||
"surface": "android",
|
||||
"id": "native.android.fb327cd3030aae44"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 205,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/VoiceNoteComposer.kt",
|
||||
"source": "Voice note",
|
||||
"surface": "android",
|
||||
"id": "native.android.e9d238668cb46e71"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 537,
|
||||
|
||||
@@ -17,10 +17,12 @@ import ai.openclaw.app.node.SmsManager
|
||||
import ai.openclaw.app.ui.GatewayConnectPlan
|
||||
import ai.openclaw.app.ui.GatewaySavedAuthAction
|
||||
import ai.openclaw.app.voice.VoiceConversationEntry
|
||||
import android.Manifest
|
||||
import android.app.Application
|
||||
import androidx.lifecycle.AndroidViewModel
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import androidx.lifecycle.viewModelScope
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
@@ -57,6 +59,8 @@ class MainViewModel(
|
||||
private val gatewayConfigOperationSeq = AtomicLong()
|
||||
private val gatewayConfigOperationMutex = Mutex()
|
||||
|
||||
@Volatile private var permissionRequester: PermissionRequester? = null
|
||||
|
||||
@Volatile private var foreground = false
|
||||
|
||||
@Volatile private var runtimeStartupQueued = false
|
||||
@@ -264,6 +268,7 @@ class MainViewModel(
|
||||
runtime.camera.attachLifecycleOwner(owner)
|
||||
runtime.camera.attachPermissionRequester(permissionRequester)
|
||||
runtime.sms.attachPermissionRequester(permissionRequester)
|
||||
this.permissionRequester = permissionRequester
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -536,6 +541,23 @@ class MainViewModel(
|
||||
ensureRuntime().setTalkModeEnabled(enabled)
|
||||
}
|
||||
|
||||
suspend fun requestVoiceNotePermission(): Boolean {
|
||||
val requester = permissionRequester ?: return false
|
||||
return try {
|
||||
requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO))[Manifest.permission.RECORD_AUDIO] == true
|
||||
} catch (error: CancellationException) {
|
||||
throw error
|
||||
} catch (_: Throwable) {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
internal fun tryAcquireVoiceNoteMic(): Boolean = runtimeRef.value?.tryAcquireVoiceNoteMic() == true
|
||||
|
||||
internal fun releaseVoiceNoteMic() {
|
||||
runtimeRef.value?.releaseVoiceNoteMic()
|
||||
}
|
||||
|
||||
fun setSpeakerEnabled(enabled: Boolean) {
|
||||
ensureRuntime().setSpeakerEnabled(enabled)
|
||||
}
|
||||
|
||||
@@ -742,6 +742,7 @@ class NodeRuntime private constructor(
|
||||
// Keep ownership epochs and their service/capture state transitions atomic.
|
||||
// Otherwise stale PTT cleanup can pass its epoch check before a UI mode change.
|
||||
private val voiceCaptureOwnershipLock = Any()
|
||||
private var voiceNoteOwnsMic = false
|
||||
private val voiceCapturePreparationMutex = Mutex()
|
||||
|
||||
private var didAutoRequestCanvasRehydrate = false
|
||||
@@ -1865,6 +1866,19 @@ class NodeRuntime private constructor(
|
||||
setVoiceCaptureMode(if (value) VoiceCaptureMode.ManualMic else VoiceCaptureMode.Off)
|
||||
}
|
||||
|
||||
internal fun tryAcquireVoiceNoteMic(): Boolean =
|
||||
synchronized(voiceCaptureOwnershipLock) {
|
||||
if (voiceNoteOwnsMic || !isVoiceCaptureModeActive(VoiceCaptureMode.Off)) return@synchronized false
|
||||
voiceNoteOwnsMic = true
|
||||
true
|
||||
}
|
||||
|
||||
internal fun releaseVoiceNoteMic() {
|
||||
synchronized(voiceCaptureOwnershipLock) {
|
||||
voiceNoteOwnsMic = false
|
||||
}
|
||||
}
|
||||
|
||||
fun cancelMicCapture() {
|
||||
micCapture.cancelMicCapture()
|
||||
setVoiceCaptureMode(VoiceCaptureMode.Off, persistManualMic = false)
|
||||
@@ -2024,6 +2038,9 @@ class NodeRuntime private constructor(
|
||||
) {
|
||||
throw IllegalStateException("NODE_BACKGROUND_UNAVAILABLE: command requires foreground")
|
||||
}
|
||||
if (voiceNoteOwnsMic) {
|
||||
throw IllegalStateException("MIC_BUSY: voice note recording is active")
|
||||
}
|
||||
if (!hasRecordAudioPermission()) {
|
||||
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
|
||||
}
|
||||
@@ -2251,6 +2268,7 @@ class NodeRuntime private constructor(
|
||||
persistManualMic: Boolean = true,
|
||||
) {
|
||||
synchronized(voiceCaptureOwnershipLock) {
|
||||
if (mode != VoiceCaptureMode.Off && voiceNoteOwnsMic) return
|
||||
talkPttCommandEpoch.incrementAndGet()
|
||||
voiceCaptureOwnershipEpoch.incrementAndGet()
|
||||
val permissionDenied = mode.requiresMicrophonePermission && !hasRecordAudioPermission()
|
||||
|
||||
@@ -682,6 +682,7 @@ class ChatController internal constructor(
|
||||
mimeType = att.mimeType,
|
||||
fileName = att.fileName,
|
||||
base64 = att.base64,
|
||||
durationMs = att.durationMs,
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -2221,29 +2222,60 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
text = obj["text"].asStringOrNull() ?: obj["content"].asStringOrNull(),
|
||||
)
|
||||
|
||||
"image" ->
|
||||
"image", "audio" ->
|
||||
ChatMessageContent(
|
||||
type = "image",
|
||||
type = obj["type"].asStringOrNull() ?: "image",
|
||||
mimeType = obj["mimeType"].asStringOrNull(),
|
||||
fileName = obj["fileName"].asStringOrNull(),
|
||||
base64 = obj["content"].asStringOrNull()?.takeIf { it.isNotBlank() },
|
||||
)
|
||||
|
||||
"attachment" -> {
|
||||
val attachment = obj["attachment"].asObjectOrNull() ?: return null
|
||||
val mimeType = attachment["mimeType"].asStringOrNull()
|
||||
if (attachment["kind"].asStringOrNull() != "audio" && mimeType?.startsWith("audio/") != true) return null
|
||||
ChatMessageContent(
|
||||
type = "audio",
|
||||
mimeType = mimeType,
|
||||
fileName = attachment["label"].asStringOrNull(),
|
||||
)
|
||||
}
|
||||
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
internal fun parseChatMessageContents(obj: JsonObject): List<ChatMessageContent> {
|
||||
obj["content"].asArrayOrNull()?.let { content ->
|
||||
return content.mapNotNull(::parseChatMessageContent)
|
||||
val content =
|
||||
obj["content"].asArrayOrNull()?.mapNotNull(::parseChatMessageContent)
|
||||
?: obj["content"].asStringOrNull()?.let { listOf(ChatMessageContent(type = "text", text = it)) }
|
||||
?: obj["text"].asStringOrNull()?.let { listOf(ChatMessageContent(type = "text", text = it)) }
|
||||
?: emptyList()
|
||||
val transcriptAudio = parseTranscriptAudioContents(obj)
|
||||
if (transcriptAudio.isEmpty()) return content
|
||||
return content +
|
||||
transcriptAudio.filterNot { audio ->
|
||||
content.any { it.mimeType == audio.mimeType && it.fileName == audio.fileName }
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseTranscriptAudioContents(obj: JsonObject): List<ChatMessageContent> {
|
||||
val paths =
|
||||
obj["MediaPaths"].asArrayOrNull()?.mapNotNull { it.asStringOrNull() }
|
||||
?: obj["MediaPath"].asStringOrNull()?.let { listOf(it) }
|
||||
?: return emptyList()
|
||||
val types =
|
||||
obj["MediaTypes"].asArrayOrNull()?.map { it.asStringOrNull().orEmpty() }
|
||||
?: obj["MediaType"].asStringOrNull()?.let { listOf(it) }
|
||||
?: emptyList()
|
||||
return paths.mapIndexedNotNull { index, path ->
|
||||
val mimeType = types.getOrNull(index)?.takeIf { it.startsWith("audio/") } ?: return@mapIndexedNotNull null
|
||||
ChatMessageContent(
|
||||
type = "audio",
|
||||
mimeType = mimeType,
|
||||
fileName = path.substringAfterLast('/').takeIf(String::isNotBlank),
|
||||
)
|
||||
}
|
||||
obj["content"].asStringOrNull()?.let { text ->
|
||||
return listOf(ChatMessageContent(type = "text", text = text))
|
||||
}
|
||||
obj["text"].asStringOrNull()?.let { text ->
|
||||
return listOf(ChatMessageContent(type = "text", text = text))
|
||||
}
|
||||
return emptyList()
|
||||
}
|
||||
|
||||
private fun parseCreatedSessionKey(
|
||||
@@ -2330,21 +2362,48 @@ internal fun reconcileMessageIds(
|
||||
): List<ChatMessage> {
|
||||
if (previous.isEmpty() || incoming.isEmpty()) return incoming
|
||||
|
||||
val idsByKey = LinkedHashMap<String, ArrayDeque<String>>()
|
||||
val messagesByKey = LinkedHashMap<String, ArrayDeque<ChatMessage>>()
|
||||
for (message in previous) {
|
||||
val key = messageIdentityKey(message) ?: continue
|
||||
idsByKey.getOrPut(key) { ArrayDeque() }.addLast(message.id)
|
||||
messagesByKey.getOrPut(key) { ArrayDeque() }.addLast(message)
|
||||
}
|
||||
|
||||
return incoming.map { message ->
|
||||
val key = messageIdentityKey(message) ?: return@map message
|
||||
val ids = idsByKey[key] ?: return@map message
|
||||
val reusedId = ids.removeFirstOrNull() ?: return@map message
|
||||
if (ids.isEmpty()) {
|
||||
idsByKey.remove(key)
|
||||
val matches = messagesByKey[key] ?: return@map message
|
||||
val previousMessage = matches.removeFirstOrNull() ?: return@map message
|
||||
if (matches.isEmpty()) {
|
||||
messagesByKey.remove(key)
|
||||
}
|
||||
if (reusedId == message.id) return@map message
|
||||
message.copy(id = reusedId)
|
||||
message.copy(
|
||||
id = previousMessage.id,
|
||||
content = preserveOptimisticAudioDuration(previous = previousMessage, incoming = message),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun preserveOptimisticAudioDuration(
|
||||
previous: ChatMessage,
|
||||
incoming: ChatMessage,
|
||||
): List<ChatMessageContent> {
|
||||
val idempotencyKey = incoming.idempotencyKey?.trim().orEmpty()
|
||||
if (idempotencyKey.isEmpty() || idempotencyKey != previous.idempotencyKey?.trim()) return incoming.content
|
||||
|
||||
val remainingAudio =
|
||||
previous.content
|
||||
.filter { it.mimeType?.startsWith("audio/") == true && it.durationMs != null }
|
||||
.toMutableList()
|
||||
if (remainingAudio.isEmpty()) return incoming.content
|
||||
|
||||
return incoming.content.map { part ->
|
||||
if (part.durationMs != null || part.mimeType?.startsWith("audio/") != true) return@map part
|
||||
if (remainingAudio.isEmpty()) return@map part
|
||||
val exactIndex =
|
||||
remainingAudio.indexOfFirst {
|
||||
it.mimeType == part.mimeType && it.fileName == part.fileName
|
||||
}
|
||||
val match = remainingAudio.removeAt(if (exactIndex >= 0) exactIndex else 0)
|
||||
part.copy(durationMs = match.durationMs)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ data class ChatMessageContent(
|
||||
val mimeType: String? = null,
|
||||
val fileName: String? = null,
|
||||
val base64: String? = null,
|
||||
val durationMs: Long? = null,
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -106,4 +107,5 @@ data class OutgoingAttachment(
|
||||
val mimeType: String,
|
||||
val fileName: String,
|
||||
val base64: String,
|
||||
val durationMs: Long? = null,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
package ai.openclaw.app.chat
|
||||
|
||||
import android.content.Context
|
||||
import android.media.MediaRecorder
|
||||
import android.os.SystemClock
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import java.io.File
|
||||
import java.io.RandomAccessFile
|
||||
import java.util.UUID
|
||||
|
||||
internal const val VOICE_NOTE_MAX_DURATION_MS = 180_000L
|
||||
internal const val VOICE_NOTE_MAX_BYTES = 3_500_000L
|
||||
internal const val VOICE_NOTE_MIME_TYPE = "audio/mp4"
|
||||
|
||||
internal sealed interface VoiceNoteRecorderState {
|
||||
data object Idle : VoiceNoteRecorderState
|
||||
|
||||
data class Recording(
|
||||
val startedAtMillis: Long,
|
||||
) : VoiceNoteRecorderState
|
||||
|
||||
data object Preparing : VoiceNoteRecorderState
|
||||
|
||||
data class Failure(
|
||||
val message: String,
|
||||
) : VoiceNoteRecorderState
|
||||
}
|
||||
|
||||
internal data class VoiceNoteRecording(
|
||||
val file: File,
|
||||
val durationMs: Long,
|
||||
)
|
||||
|
||||
internal interface VoiceNoteRecordingEngine {
|
||||
fun start(outputFile: File)
|
||||
|
||||
fun stop(): Long
|
||||
|
||||
fun cancel()
|
||||
}
|
||||
|
||||
/** Owns voice-note recording state and temporary-file cleanup. */
|
||||
internal class VoiceNoteRecorderController(
|
||||
private val scope: CoroutineScope,
|
||||
private val outputDirectory: File,
|
||||
private val engine: VoiceNoteRecordingEngine,
|
||||
private val requestPermission: suspend () -> Boolean,
|
||||
private val acquireMic: () -> Boolean,
|
||||
private val releaseMic: () -> Unit,
|
||||
private val onFinished: (VoiceNoteRecording) -> Unit,
|
||||
private val elapsedRealtimeMillis: () -> Long = SystemClock::elapsedRealtime,
|
||||
) {
|
||||
private val lock = Any()
|
||||
private val _state = MutableStateFlow<VoiceNoteRecorderState>(VoiceNoteRecorderState.Idle)
|
||||
val state: StateFlow<VoiceNoteRecorderState> = _state.asStateFlow()
|
||||
|
||||
private val _elapsedMs = MutableStateFlow(0L)
|
||||
val elapsedMs: StateFlow<Long> = _elapsedMs.asStateFlow()
|
||||
|
||||
private var outputFile: File? = null
|
||||
private var elapsedJob: Job? = null
|
||||
private var ownsMic = false
|
||||
|
||||
suspend fun start(): Boolean {
|
||||
synchronized(lock) {
|
||||
if (_state.value !is VoiceNoteRecorderState.Idle && _state.value !is VoiceNoteRecorderState.Failure) return false
|
||||
}
|
||||
if (!requestPermission()) {
|
||||
fail("Microphone permission is required to record a voice note.")
|
||||
return false
|
||||
}
|
||||
|
||||
return synchronized(lock) {
|
||||
if (_state.value !is VoiceNoteRecorderState.Idle && _state.value !is VoiceNoteRecorderState.Failure) {
|
||||
return@synchronized false
|
||||
}
|
||||
if (!acquireMic()) {
|
||||
failLocked("Voice capture is already using the microphone.")
|
||||
return@synchronized false
|
||||
}
|
||||
ownsMic = true
|
||||
|
||||
val startedAt = elapsedRealtimeMillis()
|
||||
val file = File(outputDirectory, "voice-note-${UUID.randomUUID()}.m4a")
|
||||
try {
|
||||
engine.start(file)
|
||||
} catch (_: Throwable) {
|
||||
engine.cancel()
|
||||
releaseMicLocked()
|
||||
file.delete()
|
||||
failLocked("Could not start voice-note recording.")
|
||||
return@synchronized false
|
||||
}
|
||||
|
||||
outputFile = file
|
||||
_elapsedMs.value = 0L
|
||||
_state.value = VoiceNoteRecorderState.Recording(startedAtMillis = startedAt)
|
||||
startElapsedUpdates(startedAt)
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
fun finish(): Boolean {
|
||||
val recording =
|
||||
synchronized(lock) {
|
||||
if (_state.value !is VoiceNoteRecorderState.Recording) return false
|
||||
val file = outputFile ?: return false
|
||||
elapsedJob?.cancel()
|
||||
elapsedJob = null
|
||||
|
||||
val durationMs =
|
||||
try {
|
||||
engine.stop().coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS)
|
||||
} catch (_: Throwable) {
|
||||
engine.cancel()
|
||||
file.delete()
|
||||
finishFailureLocked("Could not finish voice-note recording.")
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
normalizeM4aContainerBrand(file)
|
||||
} catch (_: Throwable) {
|
||||
file.delete()
|
||||
finishFailureLocked("Could not finish voice-note recording.")
|
||||
return false
|
||||
}
|
||||
|
||||
if (file.length() > VOICE_NOTE_MAX_BYTES) {
|
||||
file.delete()
|
||||
finishFailureLocked("Voice note is too large. Record a shorter message.")
|
||||
return false
|
||||
}
|
||||
|
||||
// The file stays owned by the controller through Preparing: the staging
|
||||
// coroutine is composition-scoped and may be cancelled before it runs,
|
||||
// so cancel() must still be able to delete the handed-off recording.
|
||||
_elapsedMs.value = 0L
|
||||
_state.value = VoiceNoteRecorderState.Preparing
|
||||
releaseMicLocked()
|
||||
VoiceNoteRecording(file = file, durationMs = durationMs)
|
||||
}
|
||||
onFinished(recording)
|
||||
return true
|
||||
}
|
||||
|
||||
fun completePreparation() {
|
||||
synchronized(lock) {
|
||||
if (_state.value is VoiceNoteRecorderState.Preparing) {
|
||||
outputFile = null
|
||||
_state.value = VoiceNoteRecorderState.Idle
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun cancel() {
|
||||
synchronized(lock) {
|
||||
elapsedJob?.cancel()
|
||||
elapsedJob = null
|
||||
if (_state.value is VoiceNoteRecorderState.Recording) {
|
||||
engine.cancel()
|
||||
}
|
||||
releaseMicLocked()
|
||||
outputFile?.delete()
|
||||
outputFile = null
|
||||
_elapsedMs.value = 0L
|
||||
_state.value = VoiceNoteRecorderState.Idle
|
||||
}
|
||||
}
|
||||
|
||||
fun reportFailure(message: String) {
|
||||
synchronized(lock) {
|
||||
outputFile?.delete()
|
||||
outputFile = null
|
||||
failLocked(message)
|
||||
}
|
||||
}
|
||||
|
||||
private fun startElapsedUpdates(startedAt: Long) {
|
||||
elapsedJob?.cancel()
|
||||
elapsedJob =
|
||||
scope.launch {
|
||||
while (isActive && state.value is VoiceNoteRecorderState.Recording) {
|
||||
val elapsed = (elapsedRealtimeMillis() - startedAt).coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS)
|
||||
_elapsedMs.value = elapsed
|
||||
// MediaRecorder's duration callback races its asynchronous auto-stop.
|
||||
// Own the cap here so every successful finish calls stop() exactly once.
|
||||
if (elapsed >= VOICE_NOTE_MAX_DURATION_MS) {
|
||||
finish()
|
||||
return@launch
|
||||
}
|
||||
delay(250L)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun fail(message: String) {
|
||||
synchronized(lock) { failLocked(message) }
|
||||
}
|
||||
|
||||
private fun failLocked(message: String) {
|
||||
_state.value = VoiceNoteRecorderState.Failure(message)
|
||||
}
|
||||
|
||||
private fun finishFailureLocked(message: String) {
|
||||
releaseMicLocked()
|
||||
outputFile = null
|
||||
_elapsedMs.value = 0L
|
||||
_state.value = VoiceNoteRecorderState.Failure(message)
|
||||
}
|
||||
|
||||
private fun releaseMicLocked() {
|
||||
if (!ownsMic) return
|
||||
ownsMic = false
|
||||
releaseMic()
|
||||
}
|
||||
}
|
||||
|
||||
/** Marks AAC-only MPEG-4 output as audio so gateway byte sniffing cannot classify it as video. */
|
||||
internal fun normalizeM4aContainerBrand(file: File) {
|
||||
RandomAccessFile(file, "rw").use { output ->
|
||||
if (output.length() < 12L) return
|
||||
output.seek(4L)
|
||||
val boxType = ByteArray(4)
|
||||
output.readFully(boxType)
|
||||
if (!boxType.contentEquals("ftyp".toByteArray(Charsets.US_ASCII))) return
|
||||
output.seek(8L)
|
||||
output.write("M4A ".toByteArray(Charsets.US_ASCII))
|
||||
}
|
||||
}
|
||||
|
||||
/** Android AAC/m4a engine kept behind [VoiceNoteRecordingEngine] for JVM tests. */
|
||||
internal class AndroidVoiceNoteRecordingEngine(
|
||||
private val context: Context,
|
||||
private val elapsedRealtime: () -> Long = SystemClock::elapsedRealtime,
|
||||
) : VoiceNoteRecordingEngine {
|
||||
private var recorder: MediaRecorder? = null
|
||||
private var startedAtElapsedMs = 0L
|
||||
|
||||
override fun start(outputFile: File) {
|
||||
check(recorder == null)
|
||||
val next = MediaRecorder(context)
|
||||
try {
|
||||
next.setAudioSource(MediaRecorder.AudioSource.MIC)
|
||||
next.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
|
||||
next.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
|
||||
next.setAudioChannels(1)
|
||||
next.setAudioEncodingBitRate(32_000)
|
||||
next.setAudioSamplingRate(44_100)
|
||||
next.setOutputFile(outputFile.absolutePath)
|
||||
next.prepare()
|
||||
next.start()
|
||||
startedAtElapsedMs = elapsedRealtime()
|
||||
recorder = next
|
||||
} catch (error: Throwable) {
|
||||
next.release()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
override fun stop(): Long {
|
||||
val active = checkNotNull(recorder)
|
||||
recorder = null
|
||||
return try {
|
||||
active.stop()
|
||||
(elapsedRealtime() - startedAtElapsedMs).coerceAtLeast(0L)
|
||||
} finally {
|
||||
active.release()
|
||||
}
|
||||
}
|
||||
|
||||
override fun cancel() {
|
||||
val active = recorder ?: return
|
||||
recorder = null
|
||||
runCatching { active.stop() }
|
||||
active.release()
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package ai.openclaw.app.ui.chat
|
||||
import ai.openclaw.app.ChatDraft
|
||||
import ai.openclaw.app.ChatDraftPlacement
|
||||
import ai.openclaw.app.chat.ChatCommandEntry
|
||||
import ai.openclaw.app.chat.VoiceNoteRecorderState
|
||||
import ai.openclaw.app.ui.mobileAccent
|
||||
import ai.openclaw.app.ui.mobileAccentBorderStrong
|
||||
import ai.openclaw.app.ui.mobileAccentSoft
|
||||
@@ -35,6 +36,7 @@ import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.automirrored.filled.Send
|
||||
import androidx.compose.material.icons.filled.ArrowDropDown
|
||||
import androidx.compose.material.icons.filled.AttachFile
|
||||
import androidx.compose.material.icons.filled.Mic
|
||||
import androidx.compose.material.icons.filled.Refresh
|
||||
import androidx.compose.material.icons.filled.Stop
|
||||
import androidx.compose.material3.Button
|
||||
@@ -130,16 +132,22 @@ internal fun applyDraftText(
|
||||
|
||||
/** Chat input surface for text, image attachments, thinking level, and run controls. */
|
||||
@Composable
|
||||
fun ChatComposer(
|
||||
internal fun ChatComposer(
|
||||
draftText: ChatDraft?,
|
||||
healthOk: Boolean,
|
||||
thinkingLevel: String,
|
||||
pendingRunCount: Int,
|
||||
commands: List<ChatCommandEntry>,
|
||||
attachments: List<PendingImageAttachment>,
|
||||
attachments: List<PendingAttachment>,
|
||||
onDraftApplied: () -> Unit,
|
||||
onPickImages: () -> Unit,
|
||||
onRemoveAttachment: (id: String) -> Unit,
|
||||
voiceNoteState: VoiceNoteRecorderState,
|
||||
voiceNoteElapsedMs: Long,
|
||||
recordVoiceNoteEnabled: Boolean,
|
||||
onStartVoiceNote: () -> Unit,
|
||||
onCancelVoiceNote: () -> Unit,
|
||||
onFinishVoiceNote: () -> Unit,
|
||||
onSetThinkingLevel: (level: String) -> Unit,
|
||||
onRefresh: () -> Unit,
|
||||
onAbort: () -> Unit,
|
||||
@@ -176,6 +184,8 @@ fun ChatComposer(
|
||||
input.trim().isNotEmpty() && attachments.isEmpty()
|
||||
}
|
||||
val sendBusy = pendingRunCount > 0
|
||||
val recordingVoiceNote = voiceNoteState is VoiceNoteRecorderState.Recording
|
||||
val preparingVoiceNote = voiceNoteState is VoiceNoteRecorderState.Preparing
|
||||
|
||||
Column(modifier = Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
if (attachments.isNotEmpty()) {
|
||||
@@ -192,17 +202,29 @@ fun ChatComposer(
|
||||
)
|
||||
}
|
||||
|
||||
OutlinedTextField(
|
||||
value = input,
|
||||
onValueChange = { input = it },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
placeholder = { Text("Type a message…", style = mobileBodyStyle(), color = mobileTextTertiary) },
|
||||
minLines = 2,
|
||||
maxLines = 5,
|
||||
textStyle = mobileBodyStyle().copy(color = mobileText),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors = chatTextFieldColors(),
|
||||
)
|
||||
if (recordingVoiceNote) {
|
||||
VoiceNoteRecordingControls(
|
||||
elapsedMs = voiceNoteElapsedMs,
|
||||
onCancel = onCancelVoiceNote,
|
||||
onDone = onFinishVoiceNote,
|
||||
)
|
||||
} else if (preparingVoiceNote) {
|
||||
VoiceNotePreparing()
|
||||
} else {
|
||||
OutlinedTextField(
|
||||
value = input,
|
||||
onValueChange = { input = it },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
placeholder = { Text("Type a message…", style = mobileBodyStyle(), color = mobileTextTertiary) },
|
||||
minLines = 2,
|
||||
maxLines = 5,
|
||||
textStyle = mobileBodyStyle().copy(color = mobileText),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors = chatTextFieldColors(),
|
||||
)
|
||||
}
|
||||
|
||||
VoiceNoteRecorderError(voiceNoteState)
|
||||
|
||||
if (!healthOk) {
|
||||
Text(
|
||||
@@ -266,6 +288,13 @@ fun ChatComposer(
|
||||
onClick = onPickImages,
|
||||
)
|
||||
|
||||
if (!recordingVoiceNote) {
|
||||
VoiceNoteRecordButton(
|
||||
enabled = recordVoiceNoteEnabled && !preparingVoiceNote,
|
||||
onClick = onStartVoiceNote,
|
||||
)
|
||||
}
|
||||
|
||||
SecondaryActionButton(
|
||||
label = "Refresh",
|
||||
icon = Icons.Default.Refresh,
|
||||
@@ -300,7 +329,7 @@ fun ChatComposer(
|
||||
}
|
||||
}
|
||||
},
|
||||
enabled = canSend,
|
||||
enabled = canSend && !recordingVoiceNote && !preparingVoiceNote,
|
||||
modifier = Modifier.height(44.dp),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
contentPadding = PaddingValues(horizontal = 20.dp),
|
||||
@@ -311,7 +340,11 @@ fun ChatComposer(
|
||||
disabledContainerColor = mobileBorderStrong,
|
||||
disabledContentColor = mobileTextTertiary,
|
||||
),
|
||||
border = BorderStroke(1.dp, if (canSend) mobileAccentBorderStrong else mobileBorderStrong),
|
||||
border =
|
||||
BorderStroke(
|
||||
1.dp,
|
||||
if (canSend && !recordingVoiceNote && !preparingVoiceNote) mobileAccentBorderStrong else mobileBorderStrong,
|
||||
),
|
||||
) {
|
||||
if (sendBusy) {
|
||||
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp, color = Color.White)
|
||||
@@ -456,7 +489,7 @@ private fun thinkingLabel(raw: String): String =
|
||||
|
||||
@Composable
|
||||
private fun AttachmentsStrip(
|
||||
attachments: List<PendingImageAttachment>,
|
||||
attachments: List<PendingAttachment>,
|
||||
onRemoveAttachment: (id: String) -> Unit,
|
||||
) {
|
||||
Row(
|
||||
@@ -465,7 +498,7 @@ private fun AttachmentsStrip(
|
||||
) {
|
||||
for (att in attachments) {
|
||||
AttachmentChip(
|
||||
fileName = att.fileName,
|
||||
attachment = att,
|
||||
onRemove = { onRemoveAttachment(att.id) },
|
||||
)
|
||||
}
|
||||
@@ -474,7 +507,7 @@ private fun AttachmentsStrip(
|
||||
|
||||
@Composable
|
||||
private fun AttachmentChip(
|
||||
fileName: String,
|
||||
attachment: PendingAttachment,
|
||||
onRemove: () -> Unit,
|
||||
) {
|
||||
Surface(
|
||||
@@ -487,8 +520,13 @@ private fun AttachmentChip(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
if (attachment.mimeType.startsWith("audio/")) {
|
||||
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(14.dp), tint = mobileTextSecondary)
|
||||
}
|
||||
Text(
|
||||
text = fileName,
|
||||
text =
|
||||
attachment.durationMs?.let { duration -> "Voice note · ${formatVoiceNoteDuration(duration)}" }
|
||||
?: attachment.fileName,
|
||||
style = mobileCaption1,
|
||||
color = mobileText,
|
||||
maxLines = 1,
|
||||
|
||||
@@ -30,7 +30,7 @@ private val decodedBitmapCache =
|
||||
internal fun loadSizedImageAttachment(
|
||||
resolver: ContentResolver,
|
||||
uri: Uri,
|
||||
): PendingImageAttachment {
|
||||
): PendingAttachment {
|
||||
val fileName = normalizeAttachmentFileName((uri.lastPathSegment ?: "image").substringAfterLast('/'))
|
||||
val bitmap = decodeScaledBitmap(resolver, uri, maxDimension = CHAT_ATTACHMENT_MAX_WIDTH)
|
||||
if (bitmap == null) {
|
||||
@@ -67,7 +67,7 @@ internal fun loadSizedImageAttachment(
|
||||
},
|
||||
)
|
||||
val base64 = Base64.encodeToString(encoded.bytes, Base64.NO_WRAP)
|
||||
return PendingImageAttachment(
|
||||
return PendingAttachment(
|
||||
id = uri.toString() + "#" + System.currentTimeMillis().toString(),
|
||||
fileName = fileName,
|
||||
mimeType = "image/jpeg",
|
||||
|
||||
@@ -89,7 +89,7 @@ internal fun ChatMessageBubble(
|
||||
when (part.type) {
|
||||
"text" -> !part.text.isNullOrBlank()
|
||||
"image" -> !part.base64.isNullOrBlank()
|
||||
else -> false
|
||||
else -> part.isAudioAttachment()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,11 +200,12 @@ private fun ChatMessageBody(
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
for (part in content) {
|
||||
when (part.type) {
|
||||
"text" -> {
|
||||
when {
|
||||
part.type == "text" -> {
|
||||
val text = part.text ?: continue
|
||||
ChatMarkdown(text = text, textColor = textColor)
|
||||
}
|
||||
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
|
||||
else -> {
|
||||
val b64 = part.base64 ?: continue
|
||||
ChatBase64Image(base64 = b64, mimeType = part.mimeType)
|
||||
|
||||
@@ -11,7 +11,7 @@ import ai.openclaw.app.chat.ChatPendingToolCall
|
||||
import ai.openclaw.app.chat.ChatSessionEntry
|
||||
import ai.openclaw.app.chat.MessageSpeechPhase
|
||||
import ai.openclaw.app.chat.MessageSpeechState
|
||||
import ai.openclaw.app.chat.OutgoingAttachment
|
||||
import ai.openclaw.app.chat.VoiceNoteRecorderState
|
||||
import ai.openclaw.app.resolveAgentIdFromMainSessionKey
|
||||
import ai.openclaw.app.ui.copyGatewayDiagnosticsReport
|
||||
import ai.openclaw.app.ui.design.ClawListItem
|
||||
@@ -58,6 +58,7 @@ import androidx.compose.material.icons.filled.AttachFile
|
||||
import androidx.compose.material.icons.filled.Close
|
||||
import androidx.compose.material.icons.filled.Cloud
|
||||
import androidx.compose.material.icons.filled.ContentCopy
|
||||
import androidx.compose.material.icons.filled.GraphicEq
|
||||
import androidx.compose.material.icons.filled.HourglassEmpty
|
||||
import androidx.compose.material.icons.filled.Mic
|
||||
import androidx.compose.material.icons.filled.MoreHoriz
|
||||
@@ -139,6 +140,11 @@ fun ChatScreen(
|
||||
val modelFavorites by viewModel.modelFavorites.collectAsState()
|
||||
val modelRecents by viewModel.modelRecents.collectAsState()
|
||||
val selectedModelRef by viewModel.chatSelectedModelRef.collectAsState()
|
||||
val micEnabled by viewModel.micEnabled.collectAsState()
|
||||
val micIsListening by viewModel.micIsListening.collectAsState()
|
||||
val micCooldown by viewModel.micCooldown.collectAsState()
|
||||
val talkModeEnabled by viewModel.talkModeEnabled.collectAsState()
|
||||
val talkModeListening by viewModel.talkModeListening.collectAsState()
|
||||
val contextUsage = resolveChatContextUsage(sessionKey = sessionKey, mainSessionKey = mainSessionKey, sessions = sessions)
|
||||
val gatewayAddress = gatewayDiagnosticsEndpoint(remoteAddress = remoteAddress, manualHost = manualHost, manualPort = manualPort, manualTls = manualTls)
|
||||
val gatewayProblemMessage = gatewayConnectionDisplay.problem?.message?.takeIf { it.isNotBlank() }
|
||||
@@ -149,7 +155,7 @@ fun ChatScreen(
|
||||
val context = LocalContext.current
|
||||
val resolver = context.contentResolver
|
||||
val scope = rememberCoroutineScope()
|
||||
val attachments = remember { mutableStateListOf<PendingImageAttachment>() }
|
||||
val attachments = remember { mutableStateListOf<PendingAttachment>() }
|
||||
var showModelPicker by rememberSaveable { mutableStateOf(false) }
|
||||
|
||||
DisposableEffect(viewModel) {
|
||||
@@ -168,6 +174,14 @@ fun ChatScreen(
|
||||
modelCatalog.firstOrNull { it.providerQualifiedRef() == selected }?.name?.takeIf { it.isNotBlank() }
|
||||
?: selected.substringAfterLast('/')
|
||||
} ?: "Model"
|
||||
val micCaptureActive = micEnabled || micIsListening || micCooldown || talkModeEnabled || talkModeListening
|
||||
val voiceNoteRecorder =
|
||||
rememberVoiceNoteRecorderController(
|
||||
viewModel = viewModel,
|
||||
onFinished = attachments::add,
|
||||
)
|
||||
val voiceNoteState by voiceNoteRecorder.state.collectAsState()
|
||||
val voiceNoteElapsedMs by voiceNoteRecorder.elapsedMs.collectAsState()
|
||||
val pickImages =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
|
||||
if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult
|
||||
@@ -319,6 +333,12 @@ fun ChatScreen(
|
||||
onOpenModelPicker = { showModelPicker = true },
|
||||
onPickImages = { pickImages.launch("image/*") },
|
||||
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
|
||||
voiceNoteState = voiceNoteState,
|
||||
voiceNoteElapsedMs = voiceNoteElapsedMs,
|
||||
recordVoiceNoteEnabled = pendingRunCount == 0 && !micCaptureActive,
|
||||
onStartVoiceNote = { scope.launch { voiceNoteRecorder.start() } },
|
||||
onCancelVoiceNote = voiceNoteRecorder::cancel,
|
||||
onFinishVoiceNote = voiceNoteRecorder::finish,
|
||||
onVoice = onVoice,
|
||||
onFixConnection = onOpenGatewaySettings,
|
||||
onCopyDiagnostics = {
|
||||
@@ -333,15 +353,7 @@ fun ChatScreen(
|
||||
onSend = {
|
||||
val message = input.trim()
|
||||
if (message.isEmpty() && attachments.isEmpty()) return@ChatComposer
|
||||
val outgoing =
|
||||
attachments.map { attachment ->
|
||||
OutgoingAttachment(
|
||||
type = "image",
|
||||
mimeType = attachment.mimeType,
|
||||
fileName = attachment.fileName,
|
||||
base64 = attachment.base64,
|
||||
)
|
||||
}
|
||||
val outgoing = attachments.map(PendingAttachment::toOutgoingAttachment)
|
||||
val pendingAttachments = attachments.toList()
|
||||
input = ""
|
||||
attachments.clear()
|
||||
@@ -847,7 +859,7 @@ private fun ChatBubble(
|
||||
when (part.type) {
|
||||
"text" -> !part.text.isNullOrBlank()
|
||||
"image" -> !part.base64.isNullOrBlank()
|
||||
else -> false
|
||||
else -> part.isAudioAttachment()
|
||||
}
|
||||
}
|
||||
if (displayableContent.isEmpty()) return
|
||||
@@ -896,10 +908,10 @@ private fun ChatBubble(
|
||||
color = ClawTheme.colors.text,
|
||||
)
|
||||
displayableContent.forEach { part ->
|
||||
if (part.type == "text") {
|
||||
ChatText(text = part.text.orEmpty(), textColor = ClawTheme.colors.text, isStreaming = live)
|
||||
} else {
|
||||
Text(text = part.fileName ?: "Attachment", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
|
||||
when {
|
||||
part.type == "text" -> ChatText(text = part.text.orEmpty(), textColor = ClawTheme.colors.text, isStreaming = live)
|
||||
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
|
||||
else -> Text(text = part.fileName ?: "Attachment", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
|
||||
}
|
||||
}
|
||||
if (messageId != null) {
|
||||
@@ -1024,7 +1036,7 @@ private fun ChatNotice(
|
||||
private fun ChatComposer(
|
||||
value: String,
|
||||
onValueChange: (String) -> Unit,
|
||||
attachments: List<PendingImageAttachment>,
|
||||
attachments: List<PendingAttachment>,
|
||||
thinkingLevel: String,
|
||||
contextUsage: ChatContextUsage,
|
||||
selectedModelLabel: String,
|
||||
@@ -1038,6 +1050,12 @@ private fun ChatComposer(
|
||||
onOpenModelPicker: () -> Unit,
|
||||
onPickImages: () -> Unit,
|
||||
onRemoveAttachment: (String) -> Unit,
|
||||
voiceNoteState: VoiceNoteRecorderState,
|
||||
voiceNoteElapsedMs: Long,
|
||||
recordVoiceNoteEnabled: Boolean,
|
||||
onStartVoiceNote: () -> Unit,
|
||||
onCancelVoiceNote: () -> Unit,
|
||||
onFinishVoiceNote: () -> Unit,
|
||||
onVoice: () -> Unit,
|
||||
onFixConnection: () -> Unit,
|
||||
onCopyDiagnostics: () -> Unit,
|
||||
@@ -1080,17 +1098,32 @@ private fun ChatComposer(
|
||||
}
|
||||
|
||||
Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(6.dp)) {
|
||||
ChatInputPill(
|
||||
value = value,
|
||||
onValueChange = onValueChange,
|
||||
onPickImages = onPickImages,
|
||||
onVoice = onVoice,
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
if (voiceNoteState is VoiceNoteRecorderState.Recording) {
|
||||
VoiceNoteRecordingControls(
|
||||
elapsedMs = voiceNoteElapsedMs,
|
||||
onCancel = onCancelVoiceNote,
|
||||
onDone = onFinishVoiceNote,
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
} else if (voiceNoteState is VoiceNoteRecorderState.Preparing) {
|
||||
VoiceNotePreparing(modifier = Modifier.weight(1f))
|
||||
} else {
|
||||
ChatInputPill(
|
||||
value = value,
|
||||
onValueChange = onValueChange,
|
||||
onPickImages = onPickImages,
|
||||
onStartVoiceNote = onStartVoiceNote,
|
||||
recordVoiceNoteEnabled = recordVoiceNoteEnabled,
|
||||
onVoice = onVoice,
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
}
|
||||
SendButton(
|
||||
// Offline, only text sends are enabled: they queue durably (text-only v1).
|
||||
enabled =
|
||||
pendingRunCount == 0 &&
|
||||
voiceNoteState !is VoiceNoteRecorderState.Recording &&
|
||||
voiceNoteState !is VoiceNoteRecorderState.Preparing &&
|
||||
pendingRunCount == 0 &&
|
||||
if (healthOk) {
|
||||
value.trim().isNotEmpty() || attachments.isNotEmpty()
|
||||
} else {
|
||||
@@ -1100,6 +1133,8 @@ private fun ChatComposer(
|
||||
)
|
||||
}
|
||||
|
||||
VoiceNoteRecorderError(voiceNoteState)
|
||||
|
||||
if (!healthOk && gatewayOffline) {
|
||||
ChatOfflineNotice(
|
||||
status = offlineStatus,
|
||||
@@ -1407,6 +1442,8 @@ private fun ChatInputPill(
|
||||
value: String,
|
||||
onValueChange: (String) -> Unit,
|
||||
onPickImages: () -> Unit,
|
||||
onStartVoiceNote: () -> Unit,
|
||||
recordVoiceNoteEnabled: Boolean,
|
||||
onVoice: () -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
@@ -1427,6 +1464,10 @@ private fun ChatInputPill(
|
||||
Icon(imageVector = Icons.Default.AttachFile, contentDescription = "Attach image", modifier = Modifier.size(16.dp))
|
||||
}
|
||||
}
|
||||
VoiceNoteRecordButton(
|
||||
enabled = recordVoiceNoteEnabled,
|
||||
onClick = onStartVoiceNote,
|
||||
)
|
||||
Box(modifier = Modifier.weight(1f)) {
|
||||
BasicTextField(
|
||||
value = value,
|
||||
@@ -1454,7 +1495,7 @@ private fun ChatInputPill(
|
||||
contentColor = ClawTheme.colors.text,
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(imageVector = Icons.Default.Mic, contentDescription = "Voice", modifier = Modifier.size(18.dp))
|
||||
Icon(imageVector = Icons.Default.GraphicEq, contentDescription = "Open voice", modifier = Modifier.size(18.dp))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1463,19 +1504,19 @@ private fun ChatInputPill(
|
||||
|
||||
@Composable
|
||||
private fun AttachmentStrip(
|
||||
attachments: List<PendingImageAttachment>,
|
||||
attachments: List<PendingAttachment>,
|
||||
onRemoveAttachment: (String) -> Unit,
|
||||
) {
|
||||
Row(modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), horizontalArrangement = Arrangement.spacedBy(6.dp)) {
|
||||
attachments.forEach { attachment ->
|
||||
AttachmentChip(fileName = attachment.fileName, onRemove = { onRemoveAttachment(attachment.id) })
|
||||
AttachmentChip(attachment = attachment, onRemove = { onRemoveAttachment(attachment.id) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun AttachmentChip(
|
||||
fileName: String,
|
||||
attachment: PendingAttachment,
|
||||
onRemove: () -> Unit,
|
||||
) {
|
||||
Surface(
|
||||
@@ -1489,7 +1530,18 @@ private fun AttachmentChip(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(6.dp),
|
||||
) {
|
||||
Text(text = fileName, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted, maxLines = 1, overflow = TextOverflow.Ellipsis)
|
||||
if (attachment.mimeType.startsWith("audio/")) {
|
||||
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(14.dp), tint = ClawTheme.colors.textMuted)
|
||||
}
|
||||
Text(
|
||||
text =
|
||||
attachment.durationMs?.let { duration -> "Voice note · ${formatVoiceNoteDuration(duration)}" }
|
||||
?: attachment.fileName,
|
||||
style = ClawTheme.type.caption,
|
||||
color = ClawTheme.colors.textMuted,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
)
|
||||
Surface(onClick = onRemove, modifier = Modifier.size(22.dp), shape = CircleShape, color = ClawTheme.colors.canvas, contentColor = ClawTheme.colors.text) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(imageVector = Icons.Default.Close, contentDescription = "Remove attachment", modifier = Modifier.size(13.dp))
|
||||
|
||||
@@ -2,7 +2,6 @@ package ai.openclaw.app.ui.chat
|
||||
|
||||
import ai.openclaw.app.MainViewModel
|
||||
import ai.openclaw.app.chat.ChatSessionEntry
|
||||
import ai.openclaw.app.chat.OutgoingAttachment
|
||||
import ai.openclaw.app.ui.mobileAccent
|
||||
import ai.openclaw.app.ui.mobileAccentBorderStrong
|
||||
import ai.openclaw.app.ui.mobileBorderStrong
|
||||
@@ -91,6 +90,11 @@ fun ChatSheetContent(viewModel: MainViewModel) {
|
||||
val messageSpeechState by viewModel.chatMessageSpeech.collectAsState()
|
||||
val gatewayConnectionDisplay by viewModel.gatewayConnectionDisplay.collectAsState()
|
||||
val gatewayOffline = !gatewayConnectionDisplay.isConnected
|
||||
val micEnabled by viewModel.micEnabled.collectAsState()
|
||||
val micIsListening by viewModel.micIsListening.collectAsState()
|
||||
val micCooldown by viewModel.micCooldown.collectAsState()
|
||||
val talkModeEnabled by viewModel.talkModeEnabled.collectAsState()
|
||||
val talkModeListening by viewModel.talkModeListening.collectAsState()
|
||||
|
||||
DisposableEffect(viewModel) {
|
||||
onDispose(viewModel::stopChatMessageSpeech)
|
||||
@@ -124,7 +128,15 @@ fun ChatSheetContent(viewModel: MainViewModel) {
|
||||
val resolver = context.contentResolver
|
||||
val scope = rememberCoroutineScope()
|
||||
|
||||
val attachments = remember { mutableStateListOf<PendingImageAttachment>() }
|
||||
val attachments = remember { mutableStateListOf<PendingAttachment>() }
|
||||
val micCaptureActive = micEnabled || micIsListening || micCooldown || talkModeEnabled || talkModeListening
|
||||
val voiceNoteRecorder =
|
||||
rememberVoiceNoteRecorderController(
|
||||
viewModel = viewModel,
|
||||
onFinished = attachments::add,
|
||||
)
|
||||
val voiceNoteState by voiceNoteRecorder.state.collectAsState()
|
||||
val voiceNoteElapsedMs by voiceNoteRecorder.elapsedMs.collectAsState()
|
||||
|
||||
val pickImages =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
|
||||
@@ -198,6 +210,12 @@ fun ChatSheetContent(viewModel: MainViewModel) {
|
||||
onDraftApplied = viewModel::clearChatDraft,
|
||||
onPickImages = { pickImages.launch("image/*") },
|
||||
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
|
||||
voiceNoteState = voiceNoteState,
|
||||
voiceNoteElapsedMs = voiceNoteElapsedMs,
|
||||
recordVoiceNoteEnabled = pendingRunCount == 0 && !micCaptureActive,
|
||||
onStartVoiceNote = { scope.launch { voiceNoteRecorder.start() } },
|
||||
onCancelVoiceNote = voiceNoteRecorder::cancel,
|
||||
onFinishVoiceNote = voiceNoteRecorder::finish,
|
||||
onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) },
|
||||
onRefresh = {
|
||||
viewModel.refreshChat()
|
||||
@@ -206,15 +224,7 @@ fun ChatSheetContent(viewModel: MainViewModel) {
|
||||
},
|
||||
onAbort = { viewModel.abortChat() },
|
||||
onSend = { text ->
|
||||
val outgoing =
|
||||
attachments.map { att ->
|
||||
OutgoingAttachment(
|
||||
type = "image",
|
||||
mimeType = att.mimeType,
|
||||
fileName = att.fileName,
|
||||
base64 = att.base64,
|
||||
)
|
||||
}
|
||||
val outgoing = attachments.map(PendingAttachment::toOutgoingAttachment)
|
||||
val pendingAttachments = attachments.toList()
|
||||
attachments.clear()
|
||||
val accepted = viewModel.sendChatAwaitAcceptance(message = text, thinking = thinkingLevel, attachments = outgoing)
|
||||
@@ -286,13 +296,3 @@ private fun ChatErrorRail(errorText: String) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Image selected in the composer and held in memory until the next chat.send call.
|
||||
*/
|
||||
data class PendingImageAttachment(
|
||||
val id: String,
|
||||
val fileName: String,
|
||||
val mimeType: String,
|
||||
val base64: String,
|
||||
)
|
||||
|
||||
@@ -124,6 +124,8 @@ private fun stableMessageVersion(message: ChatMessage): String {
|
||||
append(content.fileName.orEmpty())
|
||||
append(',')
|
||||
append(content.base64?.length ?: 0)
|
||||
append(',')
|
||||
append(content.durationMs ?: "")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -163,6 +165,8 @@ private fun latestContentVersion(
|
||||
append(content.fileName.orEmpty())
|
||||
append(',')
|
||||
append(content.base64?.length ?: 0)
|
||||
append(',')
|
||||
append(content.durationMs ?: "")
|
||||
}
|
||||
append(":runs=")
|
||||
append(pendingRunCount)
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import ai.openclaw.app.chat.OutgoingAttachment
|
||||
import ai.openclaw.app.chat.VOICE_NOTE_MIME_TYPE
|
||||
import ai.openclaw.app.chat.VoiceNoteRecording
|
||||
import java.util.Base64
|
||||
|
||||
/** Attachment staged in a composer until the next chat.send call. */
|
||||
data class PendingAttachment(
|
||||
val id: String,
|
||||
val fileName: String,
|
||||
val mimeType: String,
|
||||
val base64: String,
|
||||
val durationMs: Long? = null,
|
||||
)
|
||||
|
||||
internal fun PendingAttachment.toOutgoingAttachment(): OutgoingAttachment =
|
||||
OutgoingAttachment(
|
||||
type = attachmentTypeForMimeType(mimeType),
|
||||
mimeType = mimeType,
|
||||
fileName = fileName,
|
||||
base64 = base64,
|
||||
durationMs = durationMs,
|
||||
)
|
||||
|
||||
internal fun attachmentTypeForMimeType(mimeType: String): String =
|
||||
when {
|
||||
mimeType.startsWith("audio/") -> "audio"
|
||||
mimeType.startsWith("image/") -> "image"
|
||||
else -> "file"
|
||||
}
|
||||
|
||||
internal fun stageVoiceNoteAttachment(recording: VoiceNoteRecording): PendingAttachment =
|
||||
try {
|
||||
PendingAttachment(
|
||||
id = recording.file.absolutePath + "#" + recording.durationMs,
|
||||
fileName = recording.file.name,
|
||||
mimeType = VOICE_NOTE_MIME_TYPE,
|
||||
base64 = Base64.getEncoder().encodeToString(recording.file.readBytes()),
|
||||
durationMs = recording.durationMs,
|
||||
)
|
||||
} finally {
|
||||
recording.file.delete()
|
||||
}
|
||||
|
||||
internal fun formatVoiceNoteDuration(durationMs: Long): String {
|
||||
val totalSeconds = durationMs.coerceAtLeast(0L) / 1_000L
|
||||
val minutes = totalSeconds / 60L
|
||||
val seconds = totalSeconds % 60L
|
||||
return "$minutes:${seconds.toString().padStart(2, '0')}"
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import ai.openclaw.app.MainViewModel
|
||||
import ai.openclaw.app.chat.AndroidVoiceNoteRecordingEngine
|
||||
import ai.openclaw.app.chat.ChatMessageContent
|
||||
import ai.openclaw.app.chat.VoiceNoteRecorderController
|
||||
import ai.openclaw.app.chat.VoiceNoteRecorderState
|
||||
import ai.openclaw.app.ui.design.ClawTheme
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.heightIn
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Check
|
||||
import androidx.compose.material.icons.filled.Close
|
||||
import androidx.compose.material.icons.filled.Mic
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.runtime.rememberUpdatedState
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.LifecycleEventObserver
|
||||
import androidx.lifecycle.compose.LocalLifecycleOwner
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
@Composable
|
||||
internal fun rememberVoiceNoteRecorderController(
|
||||
viewModel: MainViewModel,
|
||||
onFinished: (PendingAttachment) -> Unit,
|
||||
): VoiceNoteRecorderController {
|
||||
val context = LocalContext.current.applicationContext
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
val scope = rememberCoroutineScope()
|
||||
val currentOnFinished by rememberUpdatedState(onFinished)
|
||||
lateinit var controller: VoiceNoteRecorderController
|
||||
controller =
|
||||
remember(context, viewModel, scope) {
|
||||
VoiceNoteRecorderController(
|
||||
scope = scope,
|
||||
outputDirectory = context.cacheDir,
|
||||
engine = AndroidVoiceNoteRecordingEngine(context),
|
||||
requestPermission = viewModel::requestVoiceNotePermission,
|
||||
acquireMic = viewModel::tryAcquireVoiceNoteMic,
|
||||
releaseMic = viewModel::releaseVoiceNoteMic,
|
||||
onFinished = { recording ->
|
||||
scope.launch(Dispatchers.IO) {
|
||||
val attachment = runCatching { stageVoiceNoteAttachment(recording) }
|
||||
withContext(Dispatchers.Main) {
|
||||
attachment.fold(
|
||||
onSuccess = {
|
||||
currentOnFinished(it)
|
||||
controller.completePreparation()
|
||||
},
|
||||
onFailure = { controller.reportFailure("Could not prepare voice note.") },
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
DisposableEffect(controller, lifecycleOwner) {
|
||||
val observer =
|
||||
LifecycleEventObserver { _, event ->
|
||||
if (event == Lifecycle.Event.ON_STOP) controller.cancel()
|
||||
}
|
||||
lifecycleOwner.lifecycle.addObserver(observer)
|
||||
onDispose {
|
||||
lifecycleOwner.lifecycle.removeObserver(observer)
|
||||
controller.cancel()
|
||||
}
|
||||
}
|
||||
return controller
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun VoiceNotePreparing(modifier: Modifier = Modifier) {
|
||||
Surface(
|
||||
modifier = modifier.fillMaxWidth().heightIn(min = ClawTheme.spacing.touchTarget),
|
||||
shape = RoundedCornerShape(ClawTheme.radii.control),
|
||||
color = ClawTheme.colors.surfaceRaised,
|
||||
contentColor = ClawTheme.colors.textSubtle,
|
||||
border = BorderStroke(1.dp, ClawTheme.colors.border),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(10.dp),
|
||||
) {
|
||||
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(18.dp))
|
||||
Text(text = "Preparing voice note…", style = ClawTheme.type.label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun VoiceNoteRecordButton(
|
||||
enabled: Boolean,
|
||||
onClick: () -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
Surface(
|
||||
onClick = onClick,
|
||||
enabled = enabled,
|
||||
modifier = modifier.size(ClawTheme.spacing.touchTarget),
|
||||
shape = CircleShape,
|
||||
color = ClawTheme.colors.surfaceRaised,
|
||||
contentColor = if (enabled) ClawTheme.colors.text else ClawTheme.colors.textSubtle,
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(imageVector = Icons.Default.Mic, contentDescription = "Record voice note", modifier = Modifier.size(18.dp))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun VoiceNoteRecordingControls(
|
||||
elapsedMs: Long,
|
||||
onCancel: () -> Unit,
|
||||
onDone: () -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
Surface(
|
||||
modifier = modifier.fillMaxWidth().heightIn(min = ClawTheme.spacing.touchTarget),
|
||||
shape = RoundedCornerShape(ClawTheme.radii.control),
|
||||
color = ClawTheme.colors.surfaceRaised,
|
||||
contentColor = ClawTheme.colors.text,
|
||||
border = BorderStroke(1.dp, ClawTheme.colors.border),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(10.dp),
|
||||
) {
|
||||
Box(modifier = Modifier.size(8.dp).background(ClawTheme.colors.danger, CircleShape))
|
||||
Text(
|
||||
text = formatVoiceNoteDuration(elapsedMs),
|
||||
style = ClawTheme.type.label.copy(fontWeight = FontWeight.SemiBold),
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
Surface(
|
||||
onClick = onCancel,
|
||||
modifier = Modifier.size(36.dp),
|
||||
shape = CircleShape,
|
||||
color = ClawTheme.colors.canvas,
|
||||
contentColor = ClawTheme.colors.text,
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(imageVector = Icons.Default.Close, contentDescription = "Cancel voice note", modifier = Modifier.size(17.dp))
|
||||
}
|
||||
}
|
||||
Surface(
|
||||
onClick = onDone,
|
||||
modifier = Modifier.size(36.dp),
|
||||
shape = CircleShape,
|
||||
color = ClawTheme.colors.primary,
|
||||
contentColor = ClawTheme.colors.primaryText,
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(imageVector = Icons.Default.Check, contentDescription = "Finish voice note", modifier = Modifier.size(17.dp))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun VoiceNoteRecorderError(state: VoiceNoteRecorderState) {
|
||||
val message = (state as? VoiceNoteRecorderState.Failure)?.message ?: return
|
||||
Text(text = message, style = ClawTheme.type.caption, color = ClawTheme.colors.danger)
|
||||
}
|
||||
|
||||
internal fun ChatMessageContent.isAudioAttachment(): Boolean = type == "audio" || mimeType?.startsWith("audio/") == true
|
||||
|
||||
@Composable
|
||||
internal fun VoiceNoteMessageRow(durationMs: Long?) {
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.Mic,
|
||||
contentDescription = null,
|
||||
modifier = Modifier.size(16.dp),
|
||||
tint = ClawTheme.colors.textMuted,
|
||||
)
|
||||
Text(text = "Voice note", style = ClawTheme.type.body, color = ClawTheme.colors.text)
|
||||
durationMs?.let { duration ->
|
||||
Text(
|
||||
text = formatVoiceNoteDuration(duration),
|
||||
style = ClawTheme.type.caption,
|
||||
color = ClawTheme.colors.textMuted,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -847,6 +847,32 @@ class GatewayBootstrapAuthTest {
|
||||
assertFalse(runtime.prefs.voiceMicEnabled.value)
|
||||
}
|
||||
|
||||
@Test
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
fun voiceNoteMicOwnershipBlocksLocalVoiceAndGatewayPtt() =
|
||||
runBlocking {
|
||||
val app = RuntimeEnvironment.getApplication()
|
||||
shadowOf(app).grantPermissions(Manifest.permission.RECORD_AUDIO)
|
||||
val runtime = createTestRuntime(app)
|
||||
val dispatcher = readField<InvokeDispatcher>(runtime, "invokeDispatcher")
|
||||
Dispatchers.setMain(Dispatchers.Unconfined)
|
||||
try {
|
||||
assertTrue(runtime.tryAcquireVoiceNoteMic())
|
||||
|
||||
runtime.setMicEnabled(true)
|
||||
runtime.setTalkModeEnabled(true)
|
||||
val ptt = dispatcher.handleInvoke(OpenClawTalkCommand.PttStart.rawValue, null)
|
||||
|
||||
assertEquals(VoiceCaptureMode.Off, runtime.voiceCaptureMode.value)
|
||||
assertEquals("MIC_BUSY", ptt.error?.code)
|
||||
assertEquals("MIC_BUSY: voice note recording is active", ptt.error?.message)
|
||||
assertFalse(readField<MutableStateFlow<Boolean>>(runtime, "externalAudioCaptureActive").value)
|
||||
} finally {
|
||||
runtime.releaseVoiceNoteMic()
|
||||
Dispatchers.resetMain()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
fun talkPttStart_cleansPreparedCaptureWhenBeginFails() =
|
||||
|
||||
@@ -153,6 +153,63 @@ class ChatControllerMessageIdentityTest {
|
||||
assertNotEquals(reconciled[0].id, reconciled[1].id)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun reconcileMessageIdsPreservesOptimisticVoiceNoteDuration() {
|
||||
val previous =
|
||||
ChatMessage(
|
||||
id = "local-user",
|
||||
role = "user",
|
||||
content =
|
||||
listOf(
|
||||
ChatMessageContent(type = "text", text = "See attached."),
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "voice-note.m4a", durationMs = 4_321L),
|
||||
),
|
||||
timestampMs = 1_000L,
|
||||
idempotencyKey = "run:user",
|
||||
)
|
||||
val incoming =
|
||||
previous.copy(
|
||||
id = "gateway-user",
|
||||
content = previous.content.map { it.copy(durationMs = null) },
|
||||
)
|
||||
|
||||
val reconciled = reconcileMessageIds(previous = listOf(previous), incoming = listOf(incoming)).single()
|
||||
|
||||
assertEquals("local-user", reconciled.id)
|
||||
assertEquals(4_321L, reconciled.content[1].durationMs)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun reconcileMessageIdsPreservesMultipleVoiceNoteDurationsInOrder() {
|
||||
val previous =
|
||||
ChatMessage(
|
||||
id = "local-user",
|
||||
role = "user",
|
||||
content =
|
||||
listOf(
|
||||
ChatMessageContent(type = "text", text = "See attached."),
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "first.m4a", durationMs = 1_000L),
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "second.m4a", durationMs = 2_000L),
|
||||
),
|
||||
timestampMs = 1_000L,
|
||||
idempotencyKey = "run:user",
|
||||
)
|
||||
val incoming =
|
||||
previous.copy(
|
||||
id = "gateway-user",
|
||||
content =
|
||||
listOf(
|
||||
ChatMessageContent(type = "text", text = "See attached."),
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "stored-a.m4a"),
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "stored-b.m4a"),
|
||||
),
|
||||
)
|
||||
|
||||
val reconciled = reconcileMessageIds(previous = listOf(previous), incoming = listOf(incoming)).single()
|
||||
|
||||
assertEquals(listOf(1_000L, 2_000L), reconciled.content.drop(1).map { it.durationMs })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun mergeOptimisticMessagesKeepsOutgoingUserTurnWhenHistoryOmitsIt() {
|
||||
val optimistic =
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package ai.openclaw.app.chat
|
||||
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Test
|
||||
@@ -46,4 +47,42 @@ class ChatMessageContentParsingTest {
|
||||
parseChatMessageContent(managedImage),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun parsesDirectAndAttachmentAudioBlocks() {
|
||||
val direct =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"audio","mimeType":"audio/mp4","fileName":"voice.m4a"}""",
|
||||
)
|
||||
val attachment =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mpeg","label":"reply.mp3"}}""",
|
||||
)
|
||||
|
||||
assertEquals(
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/mp4", fileName = "voice.m4a"),
|
||||
parseChatMessageContent(direct),
|
||||
)
|
||||
assertEquals(
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/mpeg", fileName = "reply.mp3"),
|
||||
parseChatMessageContent(attachment),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun parsesTranscriptAudioMediaFieldsAlongsideCaption() {
|
||||
val message =
|
||||
Json
|
||||
.parseToJsonElement(
|
||||
"""{"content":[{"type":"text","text":"See attached."}],"MediaPaths":["media/inbound/voice.m4a"],"MediaTypes":["audio/x-m4a"]}""",
|
||||
).jsonObject
|
||||
|
||||
assertEquals(
|
||||
listOf(
|
||||
ChatMessageContent(type = "text", text = "See attached."),
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/x-m4a", fileName = "voice.m4a"),
|
||||
),
|
||||
parseChatMessageContents(message),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package ai.openclaw.app.chat
|
||||
|
||||
import ai.openclaw.app.ui.chat.stageVoiceNoteAttachment
|
||||
import ai.openclaw.app.ui.chat.toOutgoingAttachment
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.test.runCurrent
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import java.nio.file.Files
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
class ChatVoiceNoteAttachmentTest {
|
||||
@Test
|
||||
fun stagedVoiceNoteUsesAudioPayloadAndDurationInLocalEcho() =
|
||||
runTest {
|
||||
val file = Files.createTempFile("voice-note-", ".m4a").toFile()
|
||||
file.writeBytes("voice-bytes".encodeToByteArray())
|
||||
val pending = stageVoiceNoteAttachment(VoiceNoteRecording(file = file, durationMs = 12_345L))
|
||||
val outgoing = pending.toOutgoingAttachment()
|
||||
var sentParams: JsonObject? = null
|
||||
val json = Json { ignoreUnknownKeys = true }
|
||||
val chat =
|
||||
ChatController(
|
||||
scope = backgroundScope,
|
||||
json = json,
|
||||
requestGateway = { method, paramsJson ->
|
||||
when (method) {
|
||||
"chat.send" -> {
|
||||
sentParams = json.parseToJsonElement(paramsJson.orEmpty()) as JsonObject
|
||||
"""{"runId":"voice-run","status":"started"}"""
|
||||
}
|
||||
else -> "{}"
|
||||
}
|
||||
},
|
||||
)
|
||||
chat.handleGatewayEvent("health", null)
|
||||
runCurrent()
|
||||
|
||||
val accepted =
|
||||
chat.sendMessageAwaitAcceptance(
|
||||
message = "listen",
|
||||
thinkingLevel = "off",
|
||||
attachments = listOf(outgoing),
|
||||
)
|
||||
|
||||
assertTrue(accepted)
|
||||
assertFalse(file.exists())
|
||||
assertEquals("audio", outgoing.type)
|
||||
assertEquals(VOICE_NOTE_MIME_TYPE, outgoing.mimeType)
|
||||
assertTrue(outgoing.fileName.endsWith(".m4a"))
|
||||
assertEquals("dm9pY2UtYnl0ZXM=", outgoing.base64)
|
||||
|
||||
val sentAttachment = ((sentParams?.get("attachments") as JsonArray).single() as JsonObject)
|
||||
assertEquals("audio", (sentAttachment["type"] as JsonPrimitive).content)
|
||||
assertEquals(VOICE_NOTE_MIME_TYPE, (sentAttachment["mimeType"] as JsonPrimitive).content)
|
||||
assertTrue((sentAttachment["fileName"] as JsonPrimitive).content.endsWith(".m4a"))
|
||||
assertEquals("dm9pY2UtYnl0ZXM=", (sentAttachment["content"] as JsonPrimitive).content)
|
||||
|
||||
val echoedAudio =
|
||||
chat.messages.value
|
||||
.single()
|
||||
.content
|
||||
.single { it.type == "audio" }
|
||||
assertEquals(VOICE_NOTE_MIME_TYPE, echoedAudio.mimeType)
|
||||
assertEquals(12_345L, echoedAudio.durationMs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package ai.openclaw.app.chat
|
||||
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.test.advanceTimeBy
|
||||
import kotlinx.coroutines.test.runCurrent
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import java.io.File
|
||||
import java.nio.file.Files
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
class VoiceNoteRecorderControllerTest {
|
||||
private class FakeEngine(
|
||||
var durationMs: Long = 1_200L,
|
||||
var outputBytes: ByteArray = byteArrayOf(1, 2, 3),
|
||||
) : VoiceNoteRecordingEngine {
|
||||
var startCount = 0
|
||||
var stopCount = 0
|
||||
var cancelCount = 0
|
||||
var outputFile: File? = null
|
||||
|
||||
override fun start(outputFile: File) {
|
||||
startCount += 1
|
||||
this.outputFile = outputFile
|
||||
outputFile.writeBytes(outputBytes)
|
||||
}
|
||||
|
||||
override fun stop(): Long {
|
||||
stopCount += 1
|
||||
return durationMs
|
||||
}
|
||||
|
||||
override fun cancel() {
|
||||
cancelCount += 1
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun startTransitionsToRecordingAndPublishesElapsedTime() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
var now = 1_000L
|
||||
val engine = FakeEngine()
|
||||
val controller = controller(directory, engine, elapsedRealtimeMillis = { now })
|
||||
|
||||
assertTrue(controller.start())
|
||||
assertEquals(VoiceNoteRecorderState.Recording(startedAtMillis = 1_000L), controller.state.value)
|
||||
|
||||
now = 3_500L
|
||||
advanceTimeBy(250L)
|
||||
runCurrent()
|
||||
assertEquals(2_500L, controller.elapsedMs.value)
|
||||
|
||||
controller.cancel()
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun stopReturnsRetainedFileAndDuration() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine(durationMs = 4_321L)
|
||||
val finished = mutableListOf<VoiceNoteRecording>()
|
||||
val controller = controller(directory, engine, onFinished = finished::add)
|
||||
|
||||
controller.start()
|
||||
assertTrue(controller.finish())
|
||||
|
||||
val recording = finished.single()
|
||||
assertEquals(4_321L, recording.durationMs)
|
||||
assertTrue(recording.file.exists())
|
||||
assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value)
|
||||
controller.completePreparation()
|
||||
assertEquals(VoiceNoteRecorderState.Idle, controller.state.value)
|
||||
recording.file.delete()
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun stopMarksMpeg4ContainerAsM4aForGatewaySniffing() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val mpeg4 = ByteArray(24)
|
||||
"ftypmp42".toByteArray(Charsets.US_ASCII).copyInto(mpeg4, destinationOffset = 4)
|
||||
val engine = FakeEngine(outputBytes = mpeg4)
|
||||
val finished = mutableListOf<VoiceNoteRecording>()
|
||||
val controller = controller(directory, engine, onFinished = finished::add)
|
||||
|
||||
controller.start()
|
||||
controller.finish()
|
||||
|
||||
val recording = finished.single()
|
||||
val majorBrand =
|
||||
recording.file
|
||||
.readBytes()
|
||||
.copyOfRange(8, 12)
|
||||
.toString(Charsets.US_ASCII)
|
||||
assertEquals("M4A ", majorBrand)
|
||||
recording.file.delete()
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun cancelDeletesTemporaryFileAndReturnsIdle() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine()
|
||||
val controller = controller(directory, engine)
|
||||
|
||||
controller.start()
|
||||
val file = requireNotNull(engine.outputFile)
|
||||
controller.cancel()
|
||||
|
||||
assertFalse(file.exists())
|
||||
assertEquals(1, engine.cancelCount)
|
||||
assertEquals(VoiceNoteRecorderState.Idle, controller.state.value)
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun durationCapUsesNormalFinishPath() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
var now = 1_000L
|
||||
val engine = FakeEngine(durationMs = VOICE_NOTE_MAX_DURATION_MS)
|
||||
val finished = mutableListOf<VoiceNoteRecording>()
|
||||
val controller = controller(directory, engine, onFinished = finished::add, elapsedRealtimeMillis = { now })
|
||||
|
||||
controller.start()
|
||||
now += VOICE_NOTE_MAX_DURATION_MS
|
||||
advanceTimeBy(250L)
|
||||
runCurrent()
|
||||
|
||||
assertEquals(1, engine.stopCount)
|
||||
assertEquals(VOICE_NOTE_MAX_DURATION_MS, finished.single().durationMs)
|
||||
finished.single().file.delete()
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun oversizeRecordingFailsAndDeletesFile() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine(outputBytes = ByteArray(VOICE_NOTE_MAX_BYTES.toInt() + 1))
|
||||
val finished = mutableListOf<VoiceNoteRecording>()
|
||||
val controller = controller(directory, engine, onFinished = finished::add)
|
||||
|
||||
controller.start()
|
||||
val file = requireNotNull(engine.outputFile)
|
||||
assertFalse(controller.finish())
|
||||
|
||||
assertFalse(file.exists())
|
||||
assertTrue(finished.isEmpty())
|
||||
assertEquals(
|
||||
VoiceNoteRecorderState.Failure("Voice note is too large. Record a shorter message."),
|
||||
controller.state.value,
|
||||
)
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun startIsRefusedWhileAlreadyRecording() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine()
|
||||
val controller = controller(directory, engine)
|
||||
|
||||
assertTrue(controller.start())
|
||||
assertFalse(controller.start())
|
||||
assertEquals(1, engine.startCount)
|
||||
|
||||
controller.cancel()
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun startIsRefusedWhilePreparingAttachment() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine()
|
||||
val finished = mutableListOf<VoiceNoteRecording>()
|
||||
val controller = controller(directory, engine, onFinished = finished::add)
|
||||
|
||||
controller.start()
|
||||
controller.finish()
|
||||
|
||||
assertFalse(controller.start())
|
||||
assertEquals(1, engine.startCount)
|
||||
assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value)
|
||||
|
||||
finished.single().file.delete()
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun cancelDuringPreparingDeletesHandedOffFile() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine()
|
||||
val finished = mutableListOf<VoiceNoteRecording>()
|
||||
val controller = controller(directory, engine, onFinished = finished::add)
|
||||
|
||||
controller.start()
|
||||
controller.finish()
|
||||
assertEquals(VoiceNoteRecorderState.Preparing, controller.state.value)
|
||||
assertTrue(finished.single().file.exists())
|
||||
|
||||
// Composition-scoped staging may be cancelled before it runs; cancel()
|
||||
// must still delete the handed-off recording.
|
||||
controller.cancel()
|
||||
|
||||
assertEquals(VoiceNoteRecorderState.Idle, controller.state.value)
|
||||
assertFalse(finished.single().file.exists())
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun reportFailureDuringPreparingDeletesHandedOffFile() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine()
|
||||
val finished = mutableListOf<VoiceNoteRecording>()
|
||||
val controller = controller(directory, engine, onFinished = finished::add)
|
||||
|
||||
controller.start()
|
||||
controller.finish()
|
||||
|
||||
controller.reportFailure("Could not prepare voice note.")
|
||||
|
||||
assertEquals(
|
||||
VoiceNoteRecorderState.Failure("Could not prepare voice note."),
|
||||
controller.state.value,
|
||||
)
|
||||
assertFalse(finished.single().file.exists())
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun startIsRefusedWhileVoiceCaptureOwnsMic() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine()
|
||||
val controller = controller(directory, engine, acquireMic = { false })
|
||||
|
||||
assertFalse(controller.start())
|
||||
|
||||
assertEquals(0, engine.startCount)
|
||||
assertEquals(
|
||||
VoiceNoteRecorderState.Failure("Voice capture is already using the microphone."),
|
||||
controller.state.value,
|
||||
)
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun permissionDeniedIsUserVisibleAndDoesNotStartEngine() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine()
|
||||
val controller = controller(directory, engine, requestPermission = { false })
|
||||
|
||||
assertFalse(controller.start())
|
||||
|
||||
assertEquals(0, engine.startCount)
|
||||
assertEquals(
|
||||
VoiceNoteRecorderState.Failure("Microphone permission is required to record a voice note."),
|
||||
controller.state.value,
|
||||
)
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
private fun kotlinx.coroutines.test.TestScope.controller(
|
||||
directory: File,
|
||||
engine: FakeEngine,
|
||||
requestPermission: suspend () -> Boolean = { true },
|
||||
acquireMic: () -> Boolean = { true },
|
||||
releaseMic: () -> Unit = {},
|
||||
onFinished: (VoiceNoteRecording) -> Unit = {},
|
||||
elapsedRealtimeMillis: () -> Long = { 1_000L },
|
||||
): VoiceNoteRecorderController =
|
||||
VoiceNoteRecorderController(
|
||||
scope = this,
|
||||
outputDirectory = directory,
|
||||
engine = engine,
|
||||
requestPermission = requestPermission,
|
||||
acquireMic = acquireMic,
|
||||
releaseMic = releaseMic,
|
||||
onFinished = onFinished,
|
||||
elapsedRealtimeMillis = elapsedRealtimeMillis,
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user