mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-12 07:20:45 +00:00
fix(android): remove mic and screen foreground services
This commit is contained in:
@@ -3,8 +3,6 @@
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
|
||||
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
|
||||
<uses-permission
|
||||
android:name="android.permission.NEARBY_WIFI_DEVICES"
|
||||
@@ -45,7 +43,7 @@
|
||||
<service
|
||||
android:name=".NodeForegroundService"
|
||||
android:exported="false"
|
||||
android:foregroundServiceType="dataSync|microphone|mediaProjection" />
|
||||
android:foregroundServiceType="dataSync" />
|
||||
<service
|
||||
android:name=".node.DeviceNotificationListenerService"
|
||||
android:label="@string/app_name"
|
||||
|
||||
@@ -18,18 +18,14 @@ import kotlinx.coroutines.launch
|
||||
class MainActivity : ComponentActivity() {
|
||||
private val viewModel: MainViewModel by viewModels()
|
||||
private lateinit var permissionRequester: PermissionRequester
|
||||
private lateinit var screenCaptureRequester: ScreenCaptureRequester
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
WindowCompat.setDecorFitsSystemWindows(window, false)
|
||||
permissionRequester = PermissionRequester(this)
|
||||
screenCaptureRequester = ScreenCaptureRequester(this)
|
||||
viewModel.camera.attachLifecycleOwner(this)
|
||||
viewModel.camera.attachPermissionRequester(permissionRequester)
|
||||
viewModel.sms.attachPermissionRequester(permissionRequester)
|
||||
viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester)
|
||||
viewModel.screenRecorder.attachPermissionRequester(permissionRequester)
|
||||
|
||||
lifecycleScope.launch {
|
||||
repeatOnLifecycle(Lifecycle.State.STARTED) {
|
||||
|
||||
@@ -6,7 +6,6 @@ import ai.openclaw.app.gateway.GatewayEndpoint
|
||||
import ai.openclaw.app.chat.OutgoingAttachment
|
||||
import ai.openclaw.app.node.CameraCaptureManager
|
||||
import ai.openclaw.app.node.CanvasController
|
||||
import ai.openclaw.app.node.ScreenRecordManager
|
||||
import ai.openclaw.app.node.SmsManager
|
||||
import ai.openclaw.app.voice.VoiceConversationEntry
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
@@ -20,7 +19,6 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
|
||||
val canvasRehydratePending: StateFlow<Boolean> = runtime.canvasRehydratePending
|
||||
val canvasRehydrateErrorText: StateFlow<String?> = runtime.canvasRehydrateErrorText
|
||||
val camera: CameraCaptureManager = runtime.camera
|
||||
val screenRecorder: ScreenRecordManager = runtime.screenRecorder
|
||||
val sms: SmsManager = runtime.sms
|
||||
|
||||
val gateways: StateFlow<List<GatewayEndpoint>> = runtime.gateways
|
||||
@@ -38,7 +36,6 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
|
||||
|
||||
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
|
||||
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
|
||||
val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
|
||||
|
||||
val instanceId: StateFlow<String> = runtime.instanceId
|
||||
val displayName: StateFlow<String> = runtime.displayName
|
||||
|
||||
@@ -5,13 +5,10 @@ import android.app.NotificationChannel
|
||||
import android.app.NotificationManager
|
||||
import android.app.Service
|
||||
import android.app.PendingIntent
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.pm.ServiceInfo
|
||||
import androidx.core.app.NotificationCompat
|
||||
import androidx.core.content.ContextCompat
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
@@ -23,14 +20,13 @@ import kotlinx.coroutines.launch
|
||||
class NodeForegroundService : Service() {
|
||||
private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
|
||||
private var notificationJob: Job? = null
|
||||
private var lastRequiresMic = false
|
||||
private var didStartForeground = false
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
ensureChannel()
|
||||
val initial = buildNotification(title = "OpenClaw Node", text = "Starting…")
|
||||
startForegroundWithTypes(notification = initial, requiresMic = false)
|
||||
startForegroundWithTypes(notification = initial)
|
||||
|
||||
val runtime = (application as NodeApp).runtime
|
||||
notificationJob =
|
||||
@@ -53,11 +49,8 @@ class NodeForegroundService : Service() {
|
||||
}
|
||||
val text = (server?.let { "$status · $it" } ?: status) + micSuffix
|
||||
|
||||
val requiresMic =
|
||||
micEnabled && hasRecordAudioPermission()
|
||||
startForegroundWithTypes(
|
||||
notification = buildNotification(title = title, text = text),
|
||||
requiresMic = requiresMic,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -135,30 +128,15 @@ class NodeForegroundService : Service() {
|
||||
mgr.notify(NOTIFICATION_ID, notification)
|
||||
}
|
||||
|
||||
private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
|
||||
if (didStartForeground && requiresMic == lastRequiresMic) {
|
||||
private fun startForegroundWithTypes(notification: Notification) {
|
||||
if (didStartForeground) {
|
||||
updateNotification(notification)
|
||||
return
|
||||
}
|
||||
|
||||
lastRequiresMic = requiresMic
|
||||
val types =
|
||||
if (requiresMic) {
|
||||
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
|
||||
} else {
|
||||
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
|
||||
}
|
||||
startForeground(NOTIFICATION_ID, notification, types)
|
||||
startForeground(NOTIFICATION_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
|
||||
didStartForeground = true
|
||||
}
|
||||
|
||||
private fun hasRecordAudioPermission(): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val CHANNEL_ID = "connection"
|
||||
private const val NOTIFICATION_ID = 1
|
||||
|
||||
@@ -50,7 +50,6 @@ class NodeRuntime(context: Context) {
|
||||
val canvas = CanvasController()
|
||||
val camera = CameraCaptureManager(appContext)
|
||||
val location = LocationCaptureManager(appContext)
|
||||
val screenRecorder = ScreenRecordManager(appContext)
|
||||
val sms = SmsManager(appContext)
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
@@ -113,12 +112,6 @@ class NodeRuntime(context: Context) {
|
||||
appContext = appContext,
|
||||
)
|
||||
|
||||
private val screenHandler: ScreenHandler = ScreenHandler(
|
||||
screenRecorder = screenRecorder,
|
||||
setScreenRecordActive = { _screenRecordActive.value = it },
|
||||
invokeErrorFromThrowable = { invokeErrorFromThrowable(it) },
|
||||
)
|
||||
|
||||
private val smsHandlerImpl: SmsHandler = SmsHandler(
|
||||
sms = sms,
|
||||
)
|
||||
@@ -153,7 +146,6 @@ class NodeRuntime(context: Context) {
|
||||
contactsHandler = contactsHandler,
|
||||
calendarHandler = calendarHandler,
|
||||
motionHandler = motionHandler,
|
||||
screenHandler = screenHandler,
|
||||
smsHandler = smsHandlerImpl,
|
||||
a2uiHandler = a2uiHandler,
|
||||
debugHandler = debugHandler,
|
||||
@@ -199,9 +191,6 @@ class NodeRuntime(context: Context) {
|
||||
private val _cameraFlashToken = MutableStateFlow(0L)
|
||||
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
|
||||
|
||||
private val _screenRecordActive = MutableStateFlow(false)
|
||||
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
|
||||
|
||||
private val _canvasA2uiHydrated = MutableStateFlow(false)
|
||||
val canvasA2uiHydrated: StateFlow<Boolean> = _canvasA2uiHydrated.asStateFlow()
|
||||
private val _canvasRehydratePending = MutableStateFlow(false)
|
||||
@@ -616,6 +605,9 @@ class NodeRuntime(context: Context) {
|
||||
|
||||
fun setForeground(value: Boolean) {
|
||||
_isForeground.value = value
|
||||
if (!value) {
|
||||
stopActiveVoiceSession()
|
||||
}
|
||||
}
|
||||
|
||||
fun setDisplayName(value: String) {
|
||||
@@ -660,11 +652,7 @@ class NodeRuntime(context: Context) {
|
||||
|
||||
fun setVoiceScreenActive(active: Boolean) {
|
||||
if (!active) {
|
||||
// User left voice screen — stop mic and TTS
|
||||
talkMode.ttsOnAllResponses = false
|
||||
talkMode.stopTts()
|
||||
micCapture.setMicEnabled(false)
|
||||
prefs.setTalkEnabled(false)
|
||||
stopActiveVoiceSession()
|
||||
}
|
||||
// Don't re-enable on active=true; mic toggle drives that
|
||||
}
|
||||
@@ -693,6 +681,14 @@ class NodeRuntime(context: Context) {
|
||||
talkMode.setPlaybackEnabled(value)
|
||||
}
|
||||
|
||||
private fun stopActiveVoiceSession() {
|
||||
talkMode.ttsOnAllResponses = false
|
||||
talkMode.stopTts()
|
||||
micCapture.setMicEnabled(false)
|
||||
prefs.setTalkEnabled(false)
|
||||
externalAudioCaptureActive.value = false
|
||||
}
|
||||
|
||||
fun refreshGatewayConnection() {
|
||||
val endpoint =
|
||||
connectedEndpoint ?: run {
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
package ai.openclaw.app
|
||||
|
||||
import android.app.Activity
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.media.projection.MediaProjectionManager
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.result.ActivityResultLauncher
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.appcompat.app.AlertDialog
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlin.coroutines.resume
|
||||
|
||||
class ScreenCaptureRequester(private val activity: ComponentActivity) {
|
||||
data class CaptureResult(val resultCode: Int, val data: Intent)
|
||||
|
||||
private val mutex = Mutex()
|
||||
private var pending: CompletableDeferred<CaptureResult?>? = null
|
||||
|
||||
private val launcher: ActivityResultLauncher<Intent> =
|
||||
activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
|
||||
val p = pending
|
||||
pending = null
|
||||
val data = result.data
|
||||
if (result.resultCode == Activity.RESULT_OK && data != null) {
|
||||
p?.complete(CaptureResult(result.resultCode, data))
|
||||
} else {
|
||||
p?.complete(null)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun requestCapture(timeoutMs: Long = 20_000): CaptureResult? =
|
||||
mutex.withLock {
|
||||
val proceed = showRationaleDialog()
|
||||
if (!proceed) return null
|
||||
|
||||
val mgr = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
val intent = mgr.createScreenCaptureIntent()
|
||||
|
||||
val deferred = CompletableDeferred<CaptureResult?>()
|
||||
pending = deferred
|
||||
withContext(Dispatchers.Main) { launcher.launch(intent) }
|
||||
|
||||
withContext(Dispatchers.Default) { withTimeout(timeoutMs) { deferred.await() } }
|
||||
}
|
||||
|
||||
private suspend fun showRationaleDialog(): Boolean =
|
||||
withContext(Dispatchers.Main) {
|
||||
suspendCancellableCoroutine { cont ->
|
||||
AlertDialog.Builder(activity)
|
||||
.setTitle("Screen recording required")
|
||||
.setMessage("OpenClaw needs to record the screen for this command.")
|
||||
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
|
||||
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
|
||||
.setOnCancelListener { cont.resume(false) }
|
||||
.show()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -219,14 +219,6 @@ class DeviceHandler(
|
||||
promptableWhenDenied = true,
|
||||
),
|
||||
)
|
||||
// Screen capture on Android is interactive per-capture consent, not a sticky app permission.
|
||||
put(
|
||||
"screenCapture",
|
||||
permissionStateJson(
|
||||
granted = false,
|
||||
promptableWhenDenied = true,
|
||||
),
|
||||
)
|
||||
},
|
||||
)
|
||||
}.toString()
|
||||
|
||||
@@ -11,7 +11,6 @@ import ai.openclaw.app.protocol.OpenClawLocationCommand
|
||||
import ai.openclaw.app.protocol.OpenClawMotionCommand
|
||||
import ai.openclaw.app.protocol.OpenClawNotificationsCommand
|
||||
import ai.openclaw.app.protocol.OpenClawPhotosCommand
|
||||
import ai.openclaw.app.protocol.OpenClawScreenCommand
|
||||
import ai.openclaw.app.protocol.OpenClawSmsCommand
|
||||
import ai.openclaw.app.protocol.OpenClawSystemCommand
|
||||
|
||||
@@ -59,7 +58,6 @@ object InvokeCommandRegistry {
|
||||
val capabilityManifest: List<NodeCapabilitySpec> =
|
||||
listOf(
|
||||
NodeCapabilitySpec(name = OpenClawCapability.Canvas.rawValue),
|
||||
NodeCapabilitySpec(name = OpenClawCapability.Screen.rawValue),
|
||||
NodeCapabilitySpec(name = OpenClawCapability.Device.rawValue),
|
||||
NodeCapabilitySpec(name = OpenClawCapability.Notifications.rawValue),
|
||||
NodeCapabilitySpec(name = OpenClawCapability.System.rawValue),
|
||||
@@ -122,10 +120,6 @@ object InvokeCommandRegistry {
|
||||
name = OpenClawCanvasA2UICommand.Reset.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawScreenCommand.Record.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawSystemCommand.Notify.rawValue,
|
||||
),
|
||||
|
||||
@@ -10,7 +10,6 @@ import ai.openclaw.app.protocol.OpenClawDeviceCommand
|
||||
import ai.openclaw.app.protocol.OpenClawLocationCommand
|
||||
import ai.openclaw.app.protocol.OpenClawMotionCommand
|
||||
import ai.openclaw.app.protocol.OpenClawNotificationsCommand
|
||||
import ai.openclaw.app.protocol.OpenClawScreenCommand
|
||||
import ai.openclaw.app.protocol.OpenClawSmsCommand
|
||||
import ai.openclaw.app.protocol.OpenClawSystemCommand
|
||||
|
||||
@@ -25,7 +24,6 @@ class InvokeDispatcher(
|
||||
private val contactsHandler: ContactsHandler,
|
||||
private val calendarHandler: CalendarHandler,
|
||||
private val motionHandler: MotionHandler,
|
||||
private val screenHandler: ScreenHandler,
|
||||
private val smsHandler: SmsHandler,
|
||||
private val a2uiHandler: A2UIHandler,
|
||||
private val debugHandler: DebugHandler,
|
||||
@@ -160,9 +158,6 @@ class InvokeDispatcher(
|
||||
OpenClawMotionCommand.Activity.rawValue -> motionHandler.handleMotionActivity(paramsJson)
|
||||
OpenClawMotionCommand.Pedometer.rawValue -> motionHandler.handleMotionPedometer(paramsJson)
|
||||
|
||||
// Screen command
|
||||
OpenClawScreenCommand.Record.rawValue -> screenHandler.handleScreenRecord(paramsJson)
|
||||
|
||||
// SMS command
|
||||
OpenClawSmsCommand.Send.rawValue -> smsHandler.handleSmsSend(paramsJson)
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package ai.openclaw.app.node
|
||||
|
||||
import ai.openclaw.app.gateway.GatewaySession
|
||||
|
||||
class ScreenHandler(
|
||||
private val screenRecorder: ScreenRecordManager,
|
||||
private val setScreenRecordActive: (Boolean) -> Unit,
|
||||
private val invokeErrorFromThrowable: (Throwable) -> Pair<String, String>,
|
||||
) {
|
||||
suspend fun handleScreenRecord(paramsJson: String?): GatewaySession.InvokeResult {
|
||||
setScreenRecordActive(true)
|
||||
try {
|
||||
val res =
|
||||
try {
|
||||
screenRecorder.record(paramsJson)
|
||||
} catch (err: Throwable) {
|
||||
val (code, message) = invokeErrorFromThrowable(err)
|
||||
return GatewaySession.InvokeResult.error(code = code, message = message)
|
||||
}
|
||||
return GatewaySession.InvokeResult.ok(res.payloadJson)
|
||||
} finally {
|
||||
setScreenRecordActive(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
package ai.openclaw.app.node
|
||||
|
||||
import android.content.Context
|
||||
import android.hardware.display.DisplayManager
|
||||
import android.media.MediaRecorder
|
||||
import android.media.projection.MediaProjectionManager
|
||||
import android.os.Build
|
||||
import android.util.Base64
|
||||
import ai.openclaw.app.ScreenCaptureRequester
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import java.io.File
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
class ScreenRecordManager(private val context: Context) {
|
||||
data class Payload(val payloadJson: String)
|
||||
|
||||
@Volatile private var screenCaptureRequester: ScreenCaptureRequester? = null
|
||||
@Volatile private var permissionRequester: ai.openclaw.app.PermissionRequester? = null
|
||||
|
||||
fun attachScreenCaptureRequester(requester: ScreenCaptureRequester) {
|
||||
screenCaptureRequester = requester
|
||||
}
|
||||
|
||||
fun attachPermissionRequester(requester: ai.openclaw.app.PermissionRequester) {
|
||||
permissionRequester = requester
|
||||
}
|
||||
|
||||
suspend fun record(paramsJson: String?): Payload =
|
||||
withContext(Dispatchers.Default) {
|
||||
val requester =
|
||||
screenCaptureRequester
|
||||
?: throw IllegalStateException(
|
||||
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
|
||||
)
|
||||
|
||||
val params = parseJsonParamsObject(paramsJson)
|
||||
val durationMs = (parseDurationMs(params) ?: 10_000).coerceIn(250, 60_000)
|
||||
val fps = (parseFps(params) ?: 10.0).coerceIn(1.0, 60.0)
|
||||
val fpsInt = fps.roundToInt().coerceIn(1, 60)
|
||||
val screenIndex = parseScreenIndex(params)
|
||||
val includeAudio = parseIncludeAudio(params) ?: true
|
||||
val format = parseString(params, key = "format")
|
||||
if (format != null && format.lowercase() != "mp4") {
|
||||
throw IllegalArgumentException("INVALID_REQUEST: screen format must be mp4")
|
||||
}
|
||||
if (screenIndex != null && screenIndex != 0) {
|
||||
throw IllegalArgumentException("INVALID_REQUEST: screenIndex must be 0 on Android")
|
||||
}
|
||||
|
||||
val capture = requester.requestCapture()
|
||||
?: throw IllegalStateException(
|
||||
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
|
||||
)
|
||||
|
||||
val mgr =
|
||||
context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
val projection = mgr.getMediaProjection(capture.resultCode, capture.data)
|
||||
?: throw IllegalStateException("UNAVAILABLE: screen capture unavailable")
|
||||
|
||||
val metrics = context.resources.displayMetrics
|
||||
val width = metrics.widthPixels
|
||||
val height = metrics.heightPixels
|
||||
val densityDpi = metrics.densityDpi
|
||||
|
||||
val file = File.createTempFile("openclaw-screen-", ".mp4")
|
||||
if (includeAudio) ensureMicPermission()
|
||||
|
||||
val recorder = createMediaRecorder()
|
||||
var virtualDisplay: android.hardware.display.VirtualDisplay? = null
|
||||
try {
|
||||
if (includeAudio) {
|
||||
recorder.setAudioSource(MediaRecorder.AudioSource.MIC)
|
||||
}
|
||||
recorder.setVideoSource(MediaRecorder.VideoSource.SURFACE)
|
||||
recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
|
||||
recorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264)
|
||||
if (includeAudio) {
|
||||
recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
|
||||
recorder.setAudioChannels(1)
|
||||
recorder.setAudioSamplingRate(44_100)
|
||||
recorder.setAudioEncodingBitRate(96_000)
|
||||
}
|
||||
recorder.setVideoSize(width, height)
|
||||
recorder.setVideoFrameRate(fpsInt)
|
||||
recorder.setVideoEncodingBitRate(estimateBitrate(width, height, fpsInt))
|
||||
recorder.setOutputFile(file.absolutePath)
|
||||
recorder.prepare()
|
||||
|
||||
val surface = recorder.surface
|
||||
virtualDisplay =
|
||||
projection.createVirtualDisplay(
|
||||
"openclaw-screen",
|
||||
width,
|
||||
height,
|
||||
densityDpi,
|
||||
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
|
||||
surface,
|
||||
null,
|
||||
null,
|
||||
)
|
||||
|
||||
recorder.start()
|
||||
delay(durationMs.toLong())
|
||||
} finally {
|
||||
try {
|
||||
recorder.stop()
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
recorder.reset()
|
||||
recorder.release()
|
||||
virtualDisplay?.release()
|
||||
projection.stop()
|
||||
}
|
||||
|
||||
val bytes = withContext(Dispatchers.IO) { file.readBytes() }
|
||||
file.delete()
|
||||
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
|
||||
Payload(
|
||||
"""{"format":"mp4","base64":"$base64","durationMs":$durationMs,"fps":$fpsInt,"screenIndex":0,"hasAudio":$includeAudio}""",
|
||||
)
|
||||
}
|
||||
|
||||
private fun createMediaRecorder(): MediaRecorder = MediaRecorder(context)
|
||||
|
||||
private suspend fun ensureMicPermission() {
|
||||
val granted =
|
||||
androidx.core.content.ContextCompat.checkSelfPermission(
|
||||
context,
|
||||
android.Manifest.permission.RECORD_AUDIO,
|
||||
) == android.content.pm.PackageManager.PERMISSION_GRANTED
|
||||
if (granted) return
|
||||
|
||||
val requester =
|
||||
permissionRequester
|
||||
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
|
||||
val results = requester.requestIfMissing(listOf(android.Manifest.permission.RECORD_AUDIO))
|
||||
if (results[android.Manifest.permission.RECORD_AUDIO] != true) {
|
||||
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseDurationMs(params: JsonObject?): Int? =
|
||||
parseJsonInt(params, "durationMs")
|
||||
|
||||
private fun parseFps(params: JsonObject?): Double? =
|
||||
parseJsonDouble(params, "fps")
|
||||
|
||||
private fun parseScreenIndex(params: JsonObject?): Int? =
|
||||
parseJsonInt(params, "screenIndex")
|
||||
|
||||
private fun parseIncludeAudio(params: JsonObject?): Boolean? = parseJsonBooleanFlag(params, "includeAudio")
|
||||
|
||||
private fun parseString(params: JsonObject?, key: String): String? =
|
||||
parseJsonString(params, key)
|
||||
|
||||
private fun estimateBitrate(width: Int, height: Int, fps: Int): Int {
|
||||
val pixels = width.toLong() * height.toLong()
|
||||
val raw = (pixels * fps.toLong() * 2L).toInt()
|
||||
return raw.coerceIn(1_000_000, 12_000_000)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package ai.openclaw.app.protocol
|
||||
enum class OpenClawCapability(val rawValue: String) {
|
||||
Canvas("canvas"),
|
||||
Camera("camera"),
|
||||
Screen("screen"),
|
||||
Sms("sms"),
|
||||
VoiceWake("voiceWake"),
|
||||
Location("location"),
|
||||
@@ -51,15 +50,6 @@ enum class OpenClawCameraCommand(val rawValue: String) {
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawScreenCommand(val rawValue: String) {
|
||||
Record("screen.record"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "screen."
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawSmsCommand(val rawValue: String) {
|
||||
Send("sms.send"),
|
||||
;
|
||||
|
||||
@@ -1402,7 +1402,7 @@ private fun PermissionsStep(
|
||||
InlineDivider()
|
||||
PermissionToggleRow(
|
||||
title = "Microphone",
|
||||
subtitle = "Voice tab transcription",
|
||||
subtitle = "Foreground Voice tab transcription",
|
||||
checked = enableMicrophone,
|
||||
granted = isPermissionGranted(context, Manifest.permission.RECORD_AUDIO),
|
||||
onCheckedChange = onMicrophoneChange,
|
||||
|
||||
@@ -402,9 +402,9 @@ fun SettingsSheet(viewModel: MainViewModel) {
|
||||
supportingContent = {
|
||||
Text(
|
||||
if (micPermissionGranted) {
|
||||
"Granted. Use the Voice tab mic button to capture transcript."
|
||||
"Granted. Use the Voice tab mic button to capture transcript while the app is open."
|
||||
} else {
|
||||
"Required for Voice tab transcription."
|
||||
"Required for foreground Voice tab transcription."
|
||||
},
|
||||
style = mobileCallout,
|
||||
)
|
||||
@@ -431,7 +431,7 @@ fun SettingsSheet(viewModel: MainViewModel) {
|
||||
}
|
||||
item {
|
||||
Text(
|
||||
"Voice wake and talk modes were removed. Voice now uses one mic on/off flow in the Voice tab.",
|
||||
"Voice wake and talk modes were removed. Voice now uses one mic on/off flow in the Voice tab while the app is open.",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
|
||||
@@ -94,7 +94,6 @@ class DeviceHandlerTest {
|
||||
"contacts",
|
||||
"calendar",
|
||||
"motion",
|
||||
"screenCapture",
|
||||
)
|
||||
for (key in expected) {
|
||||
val state = permissions.getValue(key).jsonObject
|
||||
|
||||
@@ -19,7 +19,6 @@ class InvokeCommandRegistryTest {
|
||||
private val coreCapabilities =
|
||||
setOf(
|
||||
OpenClawCapability.Canvas.rawValue,
|
||||
OpenClawCapability.Screen.rawValue,
|
||||
OpenClawCapability.Device.rawValue,
|
||||
OpenClawCapability.Notifications.rawValue,
|
||||
OpenClawCapability.System.rawValue,
|
||||
|
||||
@@ -24,7 +24,6 @@ class OpenClawProtocolConstantsTest {
|
||||
fun capabilitiesUseStableStrings() {
|
||||
assertEquals("canvas", OpenClawCapability.Canvas.rawValue)
|
||||
assertEquals("camera", OpenClawCapability.Camera.rawValue)
|
||||
assertEquals("screen", OpenClawCapability.Screen.rawValue)
|
||||
assertEquals("voiceWake", OpenClawCapability.VoiceWake.rawValue)
|
||||
assertEquals("location", OpenClawCapability.Location.rawValue)
|
||||
assertEquals("sms", OpenClawCapability.Sms.rawValue)
|
||||
@@ -44,11 +43,6 @@ class OpenClawProtocolConstantsTest {
|
||||
assertEquals("camera.clip", OpenClawCameraCommand.Clip.rawValue)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun screenCommandsUseStableStrings() {
|
||||
assertEquals("screen.record", OpenClawScreenCommand.Record.rawValue)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun notificationsCommandsUseStableStrings() {
|
||||
assertEquals("notifications.list", OpenClawNotificationsCommand.List.rawValue)
|
||||
|
||||
@@ -45,7 +45,7 @@ title: "Features"
|
||||
- Optional voice note transcription hook
|
||||
- WebChat and macOS menu bar app
|
||||
- iOS node with pairing, Canvas, camera, screen recording, location, and voice features
|
||||
- Android node with pairing, Connect tab, chat sessions, voice tab, Canvas/camera/screen, plus device, notifications, contacts/calendar, motion, photos, and SMS commands
|
||||
- Android node with pairing, Connect tab, chat sessions, voice tab, Canvas/camera, plus device, notifications, contacts/calendar, motion, photos, and SMS commands
|
||||
|
||||
<Note>
|
||||
Legacy Claude, Codex, Gemini, and Opencode paths have been removed. Pi is the only
|
||||
|
||||
@@ -89,7 +89,7 @@ The Gateway is the single source of truth for sessions, routing, and channel con
|
||||
Browser dashboard for chat, config, sessions, and nodes.
|
||||
</Card>
|
||||
<Card title="Mobile nodes" icon="smartphone">
|
||||
Pair iOS and Android nodes for Canvas, camera/screen, and voice-enabled workflows.
|
||||
Pair iOS and Android nodes for Canvas, camera, and voice-enabled workflows.
|
||||
</Card>
|
||||
</Columns>
|
||||
|
||||
@@ -164,7 +164,7 @@ Example:
|
||||
Channel-specific setup for WhatsApp, Telegram, Discord, and more.
|
||||
</Card>
|
||||
<Card title="Nodes" href="/nodes" icon="smartphone">
|
||||
iOS and Android nodes with pairing, Canvas, camera/screen, and device actions.
|
||||
iOS and Android nodes with pairing, Canvas, camera, and device actions.
|
||||
</Card>
|
||||
<Card title="Help" href="/help" icon="life-buoy">
|
||||
Common fixes and troubleshooting entry point.
|
||||
|
||||
@@ -216,7 +216,7 @@ Notes:
|
||||
|
||||
## Screen recordings (nodes)
|
||||
|
||||
Nodes expose `screen.record` (mp4). Example:
|
||||
Supported nodes expose `screen.record` (mp4). Example:
|
||||
|
||||
```bash
|
||||
openclaw nodes screen record --node <idOrNameOrIp> --duration 10s --fps 10
|
||||
@@ -225,10 +225,9 @@ openclaw nodes screen record --node <idOrNameOrIp> --duration 10s --fps 10 --no-
|
||||
|
||||
Notes:
|
||||
|
||||
- `screen.record` requires the node app to be foregrounded.
|
||||
- Android will show the system screen-capture prompt before recording.
|
||||
- `screen.record` availability depends on node platform.
|
||||
- Screen recordings are clamped to `<= 60s`.
|
||||
- `--no-audio` disables microphone capture (supported on iOS/Android; macOS uses system capture audio).
|
||||
- `--no-audio` disables microphone capture on supported platforms.
|
||||
- Use `--screen <index>` to select a display when multiple screens are available.
|
||||
|
||||
## Location (nodes)
|
||||
|
||||
@@ -118,7 +118,7 @@ The Android Chat tab supports session selection (default `main`, plus other exis
|
||||
- Send: `chat.send`
|
||||
- Push updates (best-effort): `chat.subscribe` → `event:"chat"`
|
||||
|
||||
### 7) Canvas + screen + camera
|
||||
### 7) Canvas + camera
|
||||
|
||||
#### Gateway Canvas Host (recommended for web content)
|
||||
|
||||
@@ -151,13 +151,9 @@ Camera commands (foreground only; permission-gated):
|
||||
|
||||
See [Camera node](/nodes/camera) for parameters and CLI helpers.
|
||||
|
||||
Screen commands:
|
||||
|
||||
- `screen.record` (mp4; foreground only)
|
||||
|
||||
### 8) Voice + expanded Android command surface
|
||||
|
||||
- Voice: Android uses a single mic on/off flow in the Voice tab with transcript capture and TTS playback (ElevenLabs when configured, system TTS fallback).
|
||||
- Voice: Android uses a single mic on/off flow in the Voice tab with transcript capture and TTS playback (ElevenLabs when configured, system TTS fallback). Voice stops when the app leaves the foreground.
|
||||
- Voice wake/talk-mode toggles are currently removed from Android UX/runtime.
|
||||
- Additional Android command families (availability depends on device + permissions):
|
||||
- `device.status`, `device.info`, `device.permissions`, `device.health`
|
||||
|
||||
@@ -12,7 +12,7 @@ import { resolveGatewayCredentialsFromConfig } from "./credentials.js";
|
||||
const LIVE = isTruthyEnvValue(process.env.LIVE) || isTruthyEnvValue(process.env.OPENCLAW_LIVE_TEST);
|
||||
const LIVE_ANDROID_NODE = isTruthyEnvValue(process.env.OPENCLAW_LIVE_ANDROID_NODE);
|
||||
const describeLive = LIVE && LIVE_ANDROID_NODE ? describe : describe.skip;
|
||||
const SKIPPED_INTERACTIVE_COMMANDS = new Set<string>(["screen.record"]);
|
||||
const SKIPPED_INTERACTIVE_COMMANDS = new Set<string>();
|
||||
|
||||
type CommandOutcome = "success" | "error";
|
||||
|
||||
@@ -120,15 +120,6 @@ const COMMAND_PROFILES: Record<string, CommandProfile> = {
|
||||
timeoutMs: 30_000,
|
||||
outcome: "success",
|
||||
},
|
||||
"screen.record": {
|
||||
buildParams: () => ({ durationMs: 1500, fps: 8, includeAudio: false }),
|
||||
timeoutMs: 60_000,
|
||||
outcome: "success",
|
||||
onSuccess: (payload) => {
|
||||
const obj = assertObjectPayload("screen.record", payload);
|
||||
expect(readString(obj.base64)).not.toBeNull();
|
||||
},
|
||||
},
|
||||
"camera.list": {
|
||||
buildParams: () => ({}),
|
||||
timeoutMs: 20_000,
|
||||
|
||||
Reference in New Issue
Block a user