feat(android): media renditions, MediaSession, video upload (#116037)

* feat(android): add media rendition and video support

* chore(android): refresh native i18n inventory

* fix(android): opt in to Media3 command API

* chore(android): sync native i18n source locations
This commit is contained in:
Peter Steinberger
2026-07-29 15:44:04 -04:00
committed by GitHub
parent d6f9affe79
commit e1cc617f86
26 changed files with 852 additions and 237 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -338,6 +338,7 @@ dependencies {
implementation(libs.okhttp)
implementation(libs.media3.datasource.okhttp)
implementation(libs.media3.exoplayer)
implementation(libs.media3.session)
implementation(libs.media3.ui)
implementation(libs.bcprov)
implementation(libs.coil.compose)

View File

@@ -131,6 +131,12 @@
<category android:name="android.intent.category.DEFAULT" />
<data android:mimeType="audio/*" />
</intent-filter>
<intent-filter>
<action android:name="android.intent.action.SEND" />
<action android:name="android.intent.action.SEND_MULTIPLE" />
<category android:name="android.intent.category.DEFAULT" />
<data android:mimeType="video/*" />
</intent-filter>
<intent-filter>
<action android:name="android.intent.action.SEND" />
<action android:name="android.intent.action.SEND_MULTIPLE" />

View File

@@ -45,6 +45,7 @@ data class ShareLaunchRequest(
enum class SharedAttachmentKind {
Image,
Audio,
Video,
Document,
}
@@ -63,6 +64,7 @@ internal val SHARED_ATTACHMENT_MIME_ALLOWLIST =
setOf(
"image/*",
"audio/*",
"video/*",
"application/pdf",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
@@ -71,7 +73,9 @@ internal val SHARED_ATTACHMENT_MIME_ALLOWLIST =
"text/markdown",
)
internal val SHARED_AUDIO_DOCUMENT_MIME_TYPES = SHARED_ATTACHMENT_MIME_ALLOWLIST.filterNot { it == "image/*" }.toTypedArray()
internal val SHARED_AUDIO_DOCUMENT_MIME_TYPES =
SHARED_ATTACHMENT_MIME_ALLOWLIST.filterNot { it == "image/*" || it == "video/*" }.toTypedArray()
internal val SHARED_VIDEO_MIME_TYPES = arrayOf("video/*")
/**
* Parses app-owned navigation actions that should open a specific home tab.
@@ -197,6 +201,7 @@ internal fun sharedAttachmentKindForMimeType(mimeType: String?): SharedAttachmen
return when {
normalized.startsWith("image/") -> SharedAttachmentKind.Image
normalized.startsWith("audio/") -> SharedAttachmentKind.Audio
normalized.startsWith("video/") -> SharedAttachmentKind.Video
normalized in SHARED_ATTACHMENT_MIME_ALLOWLIST -> SharedAttachmentKind.Document
else -> null
}

View File

@@ -1314,7 +1314,8 @@ class MainViewModel private constructor(
internal suspend fun loadChatMediaArtifact(
artifactId: String,
kind: GatewayMediaKind,
) = ensureRuntime().loadChatMediaArtifact(artifactId, kind)
playbackRendition: Boolean,
) = ensureRuntime().loadChatMediaArtifact(artifactId, kind, playbackRendition)
fun requestCanvasRehydrate(source: String = "screen_tab") {
ensureRuntime().requestCanvasRehydrate(source = source, force = true)

View File

@@ -4919,7 +4919,8 @@ class NodeRuntime private constructor(
internal suspend fun loadChatMediaArtifact(
artifactId: String,
kind: GatewayMediaKind,
) = chat.loadMediaArtifact(artifactId, kind)
playbackRendition: Boolean,
) = chat.loadMediaArtifact(artifactId, kind, playbackRendition)
fun loadChat(
sessionKey: String,

View File

@@ -50,6 +50,10 @@ internal const val OUTBOX_ATTACHMENT_CHUNK_BYTES = 512 * 1024
/** Upper bound of attachment bytes on one queued command (8 images plus a voice note fit). */
internal const val OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES = 8L * 1024L * 1024L
// Mirrors DEFAULT_CHAT_ATTACHMENT_MAX_MB in src/gateway/chat-attachments.ts. Only video raises
// the durable command ceiling; existing image, audio, and document admission stays at 8 MiB.
internal const val OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES = 20L * 1024L * 1024L
/** Upper bound of queued attachment bytes per gateway so the outbox database stays bounded. */
internal const val OUTBOX_MAX_GATEWAY_ATTACHMENT_BYTES = 48L * 1024L * 1024L
@@ -155,6 +159,20 @@ class LoadedOutboxAttachment(
val bytes: ByteArray,
)
private fun OutboxAttachmentPayload.isVideo(): Boolean = type == "video" || mimeType.startsWith("video/", ignoreCase = true)
internal fun outboxCommandAttachmentsWithinByteLimits(attachments: List<OutboxAttachmentPayload>): Boolean {
val totalBytes = attachments.sumOf { it.bytes.size.toLong() }
val nonVideoBytes = attachments.filterNot(OutboxAttachmentPayload::isVideo).sumOf { it.bytes.size.toLong() }
val totalLimit =
if (attachments.any(OutboxAttachmentPayload::isVideo)) {
OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES
} else {
OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
}
return totalBytes <= totalLimit && nonVideoBytes <= OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
}
sealed interface ChatOutboxEnqueueResult {
data class Queued(
val item: ChatOutboxItem,
@@ -162,7 +180,7 @@ sealed interface ChatOutboxEnqueueResult {
data object QueueFull : ChatOutboxEnqueueResult
/** One command's attachments exceed [OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES]; deleting rows cannot help. */
/** One command's attachments exceed its media-kind byte ceiling; deleting rows cannot help. */
data object AttachmentsTooLarge : ChatOutboxEnqueueResult
/** The per-gateway attachment byte budget is exhausted; deleting queued rows frees space. */
@@ -696,7 +714,7 @@ class RoomChatCommandOutbox internal constructor(
val key = sessionKey.trim().takeIf { it.isNotEmpty() } ?: return ChatOutboxEnqueueResult.Unavailable
val owner = normalizedOutboxOwnerAgentId(ownerAgentId) ?: return ChatOutboxEnqueueResult.Unavailable
val attachmentBytes = attachments.sumOf { it.bytes.size.toLong() }
if (attachmentBytes > OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES) {
if (!outboxCommandAttachmentsWithinByteLimits(attachments)) {
return ChatOutboxEnqueueResult.AttachmentsTooLarge
}
val dao = database.outboxDao()

View File

@@ -128,7 +128,8 @@ class ChatController internal constructor(
agentId: String?,
artifactId: String,
kind: GatewayMediaKind,
) -> GatewayLoadedMedia? = { _, _, _, _, _ -> null },
playbackRendition: Boolean,
) -> GatewayLoadedMedia? = { _, _, _, _, _, _ -> null },
private val commandOutbox: ChatCommandOutbox? = null,
private val recordModelRecent: (String) -> Unit = {},
private val onSessionDeleted: (ChatSessionDeletion) -> Unit = {},
@@ -163,8 +164,8 @@ class ChatController internal constructor(
loadGatewayImageArtifact = { gatewayId, sessionKey, agentId, artifactId ->
session.loadImageArtifact(gatewayId, sessionKey, agentId, artifactId)
},
loadGatewayMediaArtifact = { gatewayId, sessionKey, agentId, artifactId, kind ->
session.loadMediaArtifact(gatewayId, sessionKey, agentId, artifactId, kind)
loadGatewayMediaArtifact = { gatewayId, sessionKey, agentId, artifactId, kind, playbackRendition ->
session.loadMediaArtifact(gatewayId, sessionKey, agentId, artifactId, kind, playbackRendition)
},
commandOutbox = commandOutbox,
recordModelRecent = recordModelRecent,
@@ -186,6 +187,7 @@ class ChatController internal constructor(
suspend fun loadMediaArtifact(
artifactId: String,
kind: GatewayMediaKind,
playbackRendition: Boolean,
): GatewayLoadedMedia? {
val normalizedArtifactId = artifactId.trim().takeIf(String::isNotEmpty) ?: return null
if (kind == GatewayMediaKind.Image) return null
@@ -196,6 +198,7 @@ class ChatController internal constructor(
resolveAgentIdForSessionKey(sessionKey),
normalizedArtifactId,
kind,
playbackRendition,
)
}
@@ -6718,6 +6721,7 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
sizeBytes = obj["sizeBytes"].asLongOrNull(),
base64 = inlineContent?.takeIf { type == "image" && it.length <= CHAT_IMAGE_MAX_BASE64_CHARS },
durationMs = obj["durationMs"].asLongOrNull(),
playback = obj["playback"].asStringOrNull()?.takeIf { it == "native" || it == "transcode" },
)
}
@@ -6743,6 +6747,7 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
height = attachment["height"].asLongOrNull()?.toInt(),
sizeBytes = attachment["sizeBytes"].asLongOrNull(),
durationMs = attachment["durationMs"].asLongOrNull(),
playback = attachment["playback"].asStringOrNull()?.takeIf { it == "native" || it == "transcode" },
)
}

View File

@@ -79,6 +79,7 @@ data class ChatMessageContent(
val sizeBytes: Long? = null,
val base64: String? = null,
val durationMs: Long? = null,
val playback: String? = null,
val widget: ChatWidgetPreview? = null,
)

View File

@@ -32,6 +32,7 @@ private data class CachedMessageContent(
val height: Int? = null,
val sizeBytes: Long? = null,
val durationMs: Long? = null,
val playback: String? = null,
)
/**
@@ -320,6 +321,7 @@ class RoomChatTranscriptCache internal constructor(
height = part.height,
sizeBytes = part.sizeBytes,
durationMs = part.durationMs,
playback = part.playback,
)
},
timestampMs = row.timestampMs,
@@ -438,6 +440,7 @@ class RoomChatTranscriptCache internal constructor(
height = part.height,
sizeBytes = part.sizeBytes,
durationMs = part.durationMs,
playback = part.playback,
)
else -> null
}

View File

@@ -1,5 +1,6 @@
package ai.openclaw.app.gateway
import android.os.SystemClock
import android.util.Log
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CompletableDeferred
@@ -14,6 +15,7 @@ import kotlinx.coroutines.TimeoutCancellationException
import kotlinx.coroutines.async
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
@@ -28,12 +30,14 @@ import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import okhttp3.Interceptor
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.WebSocket
import okhttp3.WebSocketListener
import okio.Buffer
import java.io.IOException
import java.net.URI
import java.util.Base64
import java.util.Locale
@@ -84,9 +88,63 @@ sealed interface GatewayLoadedMedia {
val headers: Map<String, String>,
val client: OkHttpClient,
val mimeType: String?,
val retryPreparingPlayback: Boolean,
) : GatewayLoadedMedia
}
internal data class GatewayPlaybackRetryPolicy(
val maxElapsedMs: Long = 120_000L,
val initialDelayMs: Long = 500L,
val maxDelayMs: Long = 5_000L,
) {
init {
require(maxElapsedMs > 0L && initialDelayMs >= 0L && maxDelayMs >= initialDelayMs)
}
}
/** Shared bounded backoff for both buffered fetches and Media3's streaming HTTP path. */
internal class GatewayPlaybackRetryState(
private val policy: GatewayPlaybackRetryPolicy = GatewayPlaybackRetryPolicy(),
private val startedAtMs: Long = SystemClock.elapsedRealtime(),
) {
private var attempt = 0
fun canAttempt(nowMs: Long = SystemClock.elapsedRealtime()): Boolean = nowMs - startedAtMs < policy.maxElapsedMs
fun nextDelayMs(nowMs: Long = SystemClock.elapsedRealtime()): Long? {
val remainingMs = policy.maxElapsedMs - (nowMs - startedAtMs).coerceAtLeast(0L)
if (remainingMs <= 0L) return null
val multiplier = 1L shl attempt.coerceAtMost(30)
val delayMs = (policy.initialDelayMs * multiplier).coerceAtMost(policy.maxDelayMs)
attempt += 1
return delayMs.coerceAtMost(remainingMs)
}
}
/** Keeps Media3's OkHttp data source in its loading state while a rendition is being prepared. */
internal class GatewayPreparingPlaybackInterceptor(
private val policy: GatewayPlaybackRetryPolicy = GatewayPlaybackRetryPolicy(),
private val nowMs: () -> Long = SystemClock::elapsedRealtime,
private val sleepMs: (Long) -> Unit = Thread::sleep,
) : Interceptor {
override fun intercept(chain: Interceptor.Chain): okhttp3.Response {
val retry = GatewayPlaybackRetryState(policy = policy, startedAtMs = nowMs())
while (true) {
if (!retry.canAttempt(nowMs())) throw IOException("playback preparation timed out")
val response = chain.proceed(chain.request())
if (response.code != 202) return response
val retryDelayMs = retry.nextDelayMs(nowMs()) ?: return response
response.close()
try {
sleepMs(retryDelayMs)
} catch (error: InterruptedException) {
Thread.currentThread().interrupt()
throw IOException("playback preparation interrupted", error)
}
}
}
}
/**
* Role, scopes, commands, and permission snapshot sent with the connect frame.
*/
@@ -673,6 +731,7 @@ class GatewaySession(
agentId: String?,
artifactId: String,
kind: GatewayMediaKind,
playbackRendition: Boolean = false,
): GatewayLoadedMedia? {
val conn = readyConnection(expectedEndpointStableId) ?: return null
val params =
@@ -709,9 +768,9 @@ class GatewaySession(
conn.bufferedMedia(bytes = bytes, mimeType = resolvedMimeType)
} ?: payload["url"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty)?.let { ticketedPath ->
if (kind == GatewayMediaKind.Video) {
conn.resolveTicketedMediaStream(ticketedPath, mimeType)
conn.resolveTicketedMediaStream(ticketedPath, mimeType, playbackRendition)
} else {
conn.loadTicketedMedia(ticketedPath, kind)
conn.loadTicketedMedia(ticketedPath, kind, playbackRendition)
}
} ?: return null
return synchronized(lifecycleLock) {
@@ -941,13 +1000,15 @@ class GatewaySession(
fun resolveTicketedMediaStream(
ticketedPath: String,
mimeType: String?,
playbackRendition: Boolean,
): GatewayLoadedMedia.Streaming? {
val request = resolveTicketedMediaRequest(ticketedPath) ?: return null
val request = resolveTicketedMediaRequest(ticketedPath, playbackRendition) ?: return null
return GatewayLoadedMedia.Streaming(
url = request.url,
headers = request.headers + ("Accept" to "video/*"),
client = client,
mimeType = mimeType,
retryPreparingPlayback = playbackRendition,
)
}
@@ -965,38 +1026,53 @@ class GatewaySession(
suspend fun loadTicketedMedia(
ticketedPath: String,
kind: GatewayMediaKind,
playbackRendition: Boolean,
): GatewayLoadedMedia.Buffered? =
withContext(Dispatchers.IO) {
if (kind == GatewayMediaKind.Video) return@withContext null
val resolved = resolveTicketedMediaRequest(ticketedPath) ?: return@withContext null
val resolved = resolveTicketedMediaRequest(ticketedPath, playbackRendition) ?: return@withContext null
val request = Request.Builder().url(resolved.url).header("Accept", "${kind.wireValue}/*")
for ((name, value) in resolved.headers) {
request.header(name, value)
}
val call = client.newCall(request.build())
call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS)
call.execute().use { response ->
if (!response.isSuccessful) return@withContext null
val body = response.body
val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null
if (!mimeType.startsWith("${kind.wireValue}/")) return@withContext null
val maximumBytes = kind.maximumBufferedBytes
val declaredLength = body.contentLength()
if (declaredLength > maximumBytes) return@withContext null
val buffer = Buffer()
val source = body.source()
var total = 0L
while (true) {
val read = source.read(buffer, minOf(8192L, maximumBytes + 1L - total))
if (read == -1L) break
total += read
if (total > maximumBytes) return@withContext null
val retry = GatewayPlaybackRetryState()
repeat(Int.MAX_VALUE) {
if (!retry.canAttempt()) return@withContext null
val call = client.newCall(request.build())
call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS)
var retryDelayMs: Long? = null
call.execute().use { response ->
if (response.code == 202 && playbackRendition) {
retryDelayMs = retry.nextDelayMs()
} else {
if (!response.isSuccessful) return@withContext null
val body = response.body
val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null
if (!mimeType.startsWith("${kind.wireValue}/")) return@withContext null
val maximumBytes = kind.maximumBufferedBytes
val declaredLength = body.contentLength()
if (declaredLength > maximumBytes) return@withContext null
val buffer = Buffer()
val source = body.source()
var total = 0L
while (true) {
val read = source.read(buffer, minOf(8192L, maximumBytes + 1L - total))
if (read == -1L) break
total += read
if (total > maximumBytes) return@withContext null
}
return@withContext bufferedMedia(bytes = buffer.readByteArray(), mimeType = mimeType)
}
}
bufferedMedia(bytes = buffer.readByteArray(), mimeType = mimeType)
delay(retryDelayMs ?: return@withContext null)
}
null
}
private fun resolveTicketedMediaRequest(ticketedPath: String): TicketedMediaRequest? {
private fun resolveTicketedMediaRequest(
ticketedPath: String,
playbackRendition: Boolean = false,
): TicketedMediaRequest? {
val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return null
val rawPath = uri.rawPath ?: return null
val rawQuery = uri.rawQuery ?: return null
@@ -1007,7 +1083,13 @@ class GatewaySession(
.any { field -> field.substringBefore('=') == "mediaTicket" && field.substringAfter('=', "").isNotEmpty() }
if (!rawPath.startsWith("/api/chat/media/outgoing/") || !hasMediaTicket) return null
val scheme = if (tlsConfig != null) "https" else "http"
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath"
val playbackPath =
if (playbackRendition && rawQuery.split('&').none { it.substringBefore('=') == "playback" }) {
"$ticketedPath&playback=1"
} else {
ticketedPath
}
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$playbackPath"
val headers = mediaTransportHeaders()
return TicketedMediaRequest(url = url, headers = headers)
}

View File

@@ -6,6 +6,7 @@ import ai.openclaw.app.ChatShareDraft
import ai.openclaw.app.SharedAttachment
import ai.openclaw.app.chat.ChatComposerOwner
import ai.openclaw.app.chat.OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
import ai.openclaw.app.chat.OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES
import ai.openclaw.app.chat.VoiceNoteRecorderState
import androidx.compose.runtime.mutableStateMapOf
import androidx.compose.runtime.saveable.listSaver
@@ -391,8 +392,9 @@ internal const val CHAT_COMPOSER_MAX_ATTACHMENTS = 8
// Gateway chat attachments default to 20 MiB (images: 6 MiB); Android's outbox further caps audio/documents at 8 MiB.
internal const val CHAT_COMPOSER_MAX_IMAGE_DECODED_BYTES = 6L * 1024L * 1024L
internal const val CHAT_COMPOSER_MAX_AUDIO_DECODED_BYTES = OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
internal const val CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES = OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES
internal const val CHAT_COMPOSER_MAX_DOCUMENT_DECODED_BYTES = OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
internal const val CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES = OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
internal const val CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES = CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES
internal const val CHAT_COMPOSER_MAX_BASE64_CHARS = ((CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES + 2) / 3) * 4
internal const val CHAT_COMPOSER_MAX_TOTAL_ATTACHMENTS = 24
internal const val CHAT_COMPOSER_MAX_TOTAL_DECODED_ATTACHMENT_BYTES = CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES * 3
@@ -409,23 +411,44 @@ internal fun admitChatAttachments(
maxAttachmentCount: Int = CHAT_COMPOSER_MAX_ATTACHMENTS,
maxBase64Chars: Long = CHAT_COMPOSER_MAX_BASE64_CHARS,
maxDecodedBytes: Long = CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES,
maxNonVideoBase64Chars: Long = ((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 2) / 3) * 4,
maxNonVideoDecodedBytes: Long = OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES,
): ChatAttachmentAdmission {
require(maxAttachmentCount >= 0 && maxBase64Chars >= 0 && maxDecodedBytes >= 0)
require(
maxAttachmentCount >= 0 &&
maxBase64Chars >= 0 &&
maxDecodedBytes >= 0 &&
maxNonVideoBase64Chars >= 0 &&
maxNonVideoDecodedBytes >= 0,
)
val accepted = mutableListOf<PendingAttachment>()
var base64Chars = currentAttachments.sumOf { it.base64.length.toLong() }
var decodedBytes = currentAttachments.sumOf { decodedBase64ByteCount(it.base64) }
var nonVideoBase64Chars =
currentAttachments.filterNot { it.mimeType.startsWith("video/", ignoreCase = true) }.sumOf { it.base64.length.toLong() }
var nonVideoDecodedBytes =
currentAttachments
.filterNot { it.mimeType.startsWith("video/", ignoreCase = true) }
.sumOf { decodedBase64ByteCount(it.base64) }
var omittedCount = 0
for (candidate in candidates) {
val candidateBase64Chars = candidate.base64.length.toLong()
val candidateDecodedBytes = decodedBase64ByteCount(candidate.base64)
val candidateIsVideo = candidate.mimeType.startsWith("video/", ignoreCase = true)
val withinKind = candidateDecodedBytes <= chatComposerAttachmentDecodedByteLimit(candidate.mimeType)
val withinCount = currentAttachments.size + accepted.size < maxAttachmentCount
val withinBase64 = candidateBase64Chars <= maxBase64Chars - base64Chars
val withinDecoded = candidateDecodedBytes <= maxDecodedBytes - decodedBytes
if (withinKind && withinCount && withinBase64 && withinDecoded) {
val withinNonVideoBase64 = candidateIsVideo || candidateBase64Chars <= maxNonVideoBase64Chars - nonVideoBase64Chars
val withinNonVideoDecoded = candidateIsVideo || candidateDecodedBytes <= maxNonVideoDecodedBytes - nonVideoDecodedBytes
if (withinKind && withinCount && withinBase64 && withinDecoded && withinNonVideoBase64 && withinNonVideoDecoded) {
accepted += candidate
base64Chars += candidateBase64Chars
decodedBytes += candidateDecodedBytes
if (!candidateIsVideo) {
nonVideoBase64Chars += candidateBase64Chars
nonVideoDecodedBytes += candidateDecodedBytes
}
} else {
omittedCount += 1
}
@@ -437,6 +460,7 @@ internal fun chatComposerAttachmentDecodedByteLimit(mimeType: String): Long =
when {
mimeType.startsWith("image/", ignoreCase = true) -> CHAT_COMPOSER_MAX_IMAGE_DECODED_BYTES
mimeType.startsWith("audio/", ignoreCase = true) -> CHAT_COMPOSER_MAX_AUDIO_DECODED_BYTES
mimeType.startsWith("video/", ignoreCase = true) -> CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES
else -> CHAT_COMPOSER_MAX_DOCUMENT_DECODED_BYTES
}

View File

@@ -11,6 +11,7 @@ import android.content.ContentResolver
import android.database.Cursor
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.media.MediaMetadataRetriever
import android.net.Uri
import android.provider.OpenableColumns
import android.util.Base64
@@ -24,6 +25,8 @@ private const val CHAT_ATTACHMENT_MAX_WIDTH = 1600
private const val CHAT_ATTACHMENT_START_QUALITY = 85
private const val CHAT_DECODE_MAX_DIMENSION = 1600
private const val CHAT_IMAGE_CACHE_BYTES = 16 * 1024 * 1024
private const val VIDEO_THUMBNAIL_MAX_DIMENSION = 192
private const val VIDEO_THUMBNAIL_QUALITY = 72
private val decodedBitmapCache =
object : LruCache<String, Bitmap>(CHAT_IMAGE_CACHE_BYTES) {
@@ -33,7 +36,7 @@ private val decodedBitmapCache =
): Int = value.byteCount.coerceAtLeast(1)
}
internal fun loadPickedAudioOrDocumentAttachment(
internal fun loadPickedMediaOrDocumentAttachment(
resolver: ContentResolver,
uri: Uri,
): PendingAttachment {
@@ -65,9 +68,55 @@ internal fun loadSharedAttachment(
fileName = sharedAttachmentFileName(resolver, attachment.uri),
mimeType = mimeType,
base64 = Base64.encodeToString(bytes, Base64.NO_WRAP),
videoThumbnailBase64 =
if (kind == SharedAttachmentKind.Video) loadVideoThumbnailBase64(resolver, attachment.uri) else null,
)
}
/** Thumbnail extraction is presentation-only; an unsupported container still stages as a video. */
private fun loadVideoThumbnailBase64(
resolver: ContentResolver,
uri: Uri,
): String? =
runCatching {
val retriever = MediaMetadataRetriever()
try {
resolver.openAssetFileDescriptor(uri, "r")?.use { descriptor ->
if (descriptor.declaredLength >= 0L) {
retriever.setDataSource(descriptor.fileDescriptor, descriptor.startOffset, descriptor.declaredLength)
} else {
retriever.setDataSource(descriptor.fileDescriptor)
}
} ?: return@runCatching null
val frame = retriever.getFrameAtTime(-1L, MediaMetadataRetriever.OPTION_CLOSEST_SYNC) ?: return@runCatching null
try {
val longestEdge = max(frame.width, frame.height)
val preview =
if (longestEdge <= VIDEO_THUMBNAIL_MAX_DIMENSION) {
frame
} else {
val scale = VIDEO_THUMBNAIL_MAX_DIMENSION.toDouble() / longestEdge.toDouble()
frame.scale(
max(1, (frame.width * scale).roundToInt()),
max(1, (frame.height * scale).roundToInt()),
true,
)
}
try {
val output = ByteArrayOutputStream()
if (!preview.compress(Bitmap.CompressFormat.JPEG, VIDEO_THUMBNAIL_QUALITY, output)) return@runCatching null
Base64.encodeToString(output.toByteArray(), Base64.NO_WRAP)
} finally {
if (preview !== frame) preview.recycle()
}
} finally {
frame.recycle()
}
} finally {
retriever.release()
}
}.getOrNull()
private fun readBoundedAttachmentBytes(
resolver: ContentResolver,
uri: Uri,

View File

@@ -3,6 +3,7 @@ package ai.openclaw.app.ui.chat
import ai.openclaw.app.chat.ChatMessageContent
import ai.openclaw.app.gateway.GatewayLoadedMedia
import ai.openclaw.app.gateway.GatewayMediaKind
import ai.openclaw.app.gateway.GatewayPreparingPlaybackInterceptor
import ai.openclaw.app.i18n.nativeString
import ai.openclaw.app.ui.design.ClawTheme
import android.content.Context
@@ -63,6 +64,7 @@ import androidx.media3.datasource.DefaultDataSource
import androidx.media3.datasource.okhttp.OkHttpDataSource
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
import androidx.media3.session.MediaSession
import androidx.media3.ui.AspectRatioFrameLayout
import androidx.media3.ui.PlayerView
import kotlinx.coroutines.CancellationException
@@ -110,6 +112,60 @@ internal class ChatMediaPlaybackClaims<T>(
}
}
internal class ChatMediaSessionLifecycle<T : Any, S : Any>(
private val release: (S) -> Unit,
) {
private var activeOwner: T? = null
private var activeSession: S? = null
fun activate(
owner: T,
create: (T) -> S,
): S {
if (activeOwner === owner) return checkNotNull(activeSession)
releaseActive()
return create(owner).also { session ->
activeOwner = owner
activeSession = session
}
}
fun release(owner: T): Boolean {
if (activeOwner !== owner) return false
releaseActive()
return true
}
private fun releaseActive() {
activeSession?.let(release)
activeSession = null
activeOwner = null
}
}
@OptIn(UnstableApi::class)
internal fun inlineMediaSessionPlayerCommands(availableCommands: Player.Commands): Player.Commands =
availableCommands
.buildUpon()
.remove(Player.COMMAND_SET_MEDIA_ITEM)
.remove(Player.COMMAND_CHANGE_MEDIA_ITEMS)
.remove(Player.COMMAND_STOP)
.build()
@OptIn(UnstableApi::class)
private object ChatInlineMediaSessionCallback : MediaSession.Callback {
override fun onConnect(
session: MediaSession,
controller: MediaSession.ControllerInfo,
): MediaSession.ConnectionResult {
if (!controller.isTrusted) return MediaSession.ConnectionResult.reject()
return MediaSession.ConnectionResult
.AcceptedResultBuilder(session)
.setAvailablePlayerCommands(inlineMediaSessionPlayerCommands(session.player.availableCommands))
.build()
}
}
private object ChatMediaPlaybackArbiter {
private data class AudioFocusHandle(
val manager: AudioManager,
@@ -125,6 +181,7 @@ private object ChatMediaPlaybackArbiter {
private val mainHandler = Handler(Looper.getMainLooper())
private val claims = ChatMediaPlaybackClaims<ActivePlayback>(::pauseAndAbandon, ::stopAndRelease)
private val sessions = ChatMediaSessionLifecycle<ExoPlayer, MediaSession>(MediaSession::release)
private var playbackIntentGeneration = 0L
@Synchronized
@@ -192,6 +249,17 @@ private object ChatMediaPlaybackArbiter {
}
val playback = existing ?: ActivePlayback(player = player, onReleased = onReleased).also(claims::claim)
playback.audioFocus = AudioFocusHandle(manager = audioManager, request = focusRequest)
try {
sessions.activate(player) { activePlayer ->
MediaSession
.Builder(context, activePlayer)
.setCallback(ChatInlineMediaSessionCallback)
.build()
}
} catch (_: Throwable) {
pauseAndAbandon(playback)
return false
}
return true
}
@@ -203,6 +271,7 @@ private object ChatMediaPlaybackArbiter {
private fun pauseAndAbandon(playback: ActivePlayback) {
playback.player.pause()
sessions.release(playback.player)
playback.audioFocus?.let { focus -> focus.manager.abandonAudioFocusRequest(focus.request) }
playback.audioFocus = null
}
@@ -218,7 +287,7 @@ private object ChatMediaPlaybackArbiter {
internal fun ChatAudioPlayerCard(
content: ChatMessageContent,
playbackBlocked: Boolean,
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
loadMedia: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
) {
ChatMediaPlayerCard(
content = content,
@@ -232,7 +301,7 @@ internal fun ChatAudioPlayerCard(
internal fun ChatVideoPlayerCard(
content: ChatMessageContent,
playbackBlocked: Boolean,
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
loadMedia: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
) {
ChatMediaPlayerCard(
content = content,
@@ -248,7 +317,7 @@ private fun ChatMediaPlayerCard(
content: ChatMessageContent,
kind: GatewayMediaKind,
playbackBlocked: Boolean,
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
loadMedia: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
) {
val context = LocalContext.current
val lifecycleOwner = LocalLifecycleOwner.current
@@ -337,7 +406,7 @@ private fun ChatMediaPlayerCard(
scope.launch {
val loaded =
try {
loadMedia(artifactId, kind)
loadMedia(artifactId, kind, content.playback == "transcode")
} catch (error: CancellationException) {
throw error
} catch (_: Throwable) {
@@ -380,12 +449,14 @@ private fun ChatMediaPlayerCard(
}
override fun onPlaybackStateChanged(playbackState: Int) {
if (playbackState == Player.STATE_READY) loading = false
if (playbackState == Player.STATE_ENDED) {
ChatMediaPlaybackArbiter.pause(created)
}
}
override fun onPlayerError(playbackException: PlaybackException) {
loading = false
if (!ChatMediaPlaybackArbiter.release(created)) {
disposeUnclaimedPlayer(created, prepared.tempFile)
}
@@ -394,7 +465,7 @@ private fun ChatMediaPlayerCard(
},
)
player = created
loading = false
if (content.playback != "transcode") loading = false
if (!registerPrepared(created, prepared.tempFile, intentGeneration)) {
disposeUnclaimedPlayer(created, prepared.tempFile)
return@launch
@@ -448,6 +519,7 @@ private fun ChatMediaPlayerCard(
content = content,
player = player,
loading = loading,
preparingPlayback = loading && content.playback == "transcode",
isPlaying = isPlaying,
playbackBlocked = playbackBlocked,
error = error,
@@ -457,6 +529,7 @@ private fun ChatMediaPlayerCard(
AudioPlayerSurface(
content = content,
loading = loading,
preparingPlayback = loading && content.playback == "transcode",
isPlaying = isPlaying,
playbackBlocked = playbackBlocked,
error = error,
@@ -477,6 +550,7 @@ private fun ChatMediaPlayerCard(
private fun AudioPlayerSurface(
content: ChatMessageContent,
loading: Boolean,
preparingPlayback: Boolean,
isPlaying: Boolean,
playbackBlocked: Boolean,
error: String?,
@@ -525,7 +599,13 @@ private fun AudioPlayerSurface(
style = ClawTheme.type.body,
color = ClawTheme.colors.text,
)
val status = error ?: if (playbackBlocked) nativeString("Paused for voice playback") else null
val status =
when {
error != null -> error
preparingPlayback -> nativeString("Preparing playback…")
playbackBlocked -> nativeString("Paused for voice playback")
else -> null
}
status?.let { Text(it, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted) }
}
}
@@ -549,6 +629,7 @@ private fun VideoPlayerSurface(
content: ChatMessageContent,
player: ExoPlayer?,
loading: Boolean,
preparingPlayback: Boolean,
isPlaying: Boolean,
playbackBlocked: Boolean,
error: String?,
@@ -602,7 +683,14 @@ private fun VideoPlayerSurface(
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
)
(error ?: if (playbackBlocked) nativeString("Paused for voice playback") else null)?.let {
val status =
when {
error != null -> error
preparingPlayback -> nativeString("Preparing playback…")
playbackBlocked -> nativeString("Paused for voice playback")
else -> null
}
status?.let {
Text(it, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted)
}
}
@@ -627,6 +715,7 @@ private suspend fun prepareMediaSource(
headers = loaded.headers,
client = loaded.client,
tempFile = file,
retryPreparingPlayback = false,
)
}
is GatewayLoadedMedia.Streaming ->
@@ -636,6 +725,7 @@ private suspend fun prepareMediaSource(
headers = loaded.headers,
client = loaded.client,
tempFile = null,
retryPreparingPlayback = loaded.retryPreparingPlayback,
)
}
}
@@ -666,7 +756,16 @@ private fun buildMediaPlayer(
context: Context,
source: PreparedMediaSource,
): ExoPlayer {
val httpFactory = OkHttpDataSource.Factory(source.client).setDefaultRequestProperties(source.headers)
val client =
if (source.retryPreparingPlayback) {
source.client
.newBuilder()
.addInterceptor(GatewayPreparingPlaybackInterceptor())
.build()
} else {
source.client
}
val httpFactory = OkHttpDataSource.Factory(client).setDefaultRequestProperties(source.headers)
val dataSourceFactory = DefaultDataSource.Factory(context, httpFactory)
val player =
ExoPlayer
@@ -698,6 +797,7 @@ private data class PreparedMediaSource(
val headers: Map<String, String>,
val client: okhttp3.OkHttpClient,
val tempFile: File?,
val retryPreparingPlayback: Boolean,
)
internal fun ChatMessageContent.isVideoAttachment(): Boolean = type == "video" || mimeType?.startsWith("video/") == true

View File

@@ -110,7 +110,7 @@ internal fun ChatMessageBubble(
imageResolverReady: Boolean = false,
loadImageArtifact: suspend (String) -> GatewayLoadedImage? = { null },
inlineMediaPlaybackBlocked: Boolean = false,
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia? = { _, _ -> null },
loadMediaArtifact: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia? = { _, _, _ -> null },
) {
val role = normalizeVisibleChatMessageRole(message.role) ?: return
val style = bubbleStyle(role)
@@ -242,7 +242,7 @@ private fun ChatMessageBody(
imageResolverReady: Boolean,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
inlineMediaPlaybackBlocked: Boolean,
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
loadMediaArtifact: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
) {
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
for (part in content) {

View File

@@ -8,6 +8,7 @@ import ai.openclaw.app.MainViewModel
import ai.openclaw.app.PendingAssistantAutoSend
import ai.openclaw.app.R
import ai.openclaw.app.SHARED_AUDIO_DOCUMENT_MIME_TYPES
import ai.openclaw.app.SHARED_VIDEO_MIME_TYPES
import ai.openclaw.app.chat.ChatCommandEntry
import ai.openclaw.app.chat.ChatComposerOwner
import ai.openclaw.app.chat.ChatMessage
@@ -73,6 +74,7 @@ import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.Image
import androidx.compose.foundation.background
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement
@@ -116,6 +118,7 @@ import androidx.compose.material.icons.filled.MoreVert
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Star
import androidx.compose.material.icons.filled.StarBorder
import androidx.compose.material.icons.filled.Videocam
import androidx.compose.material3.DropdownMenu
import androidx.compose.material3.DropdownMenuItem
import androidx.compose.material3.ExperimentalMaterial3Api
@@ -138,12 +141,15 @@ import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.geometry.CornerRadius
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.SolidColor
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.input.key.onPreInterceptKeyBeforeSoftKeyboard
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.semantics.contentDescription
@@ -448,7 +454,7 @@ fun ChatScreen(
}
}
}
val pickAudioOrDocument =
val pickMediaOrDocument =
rememberLauncherForActivityResult(ActivityResultContracts.OpenDocument()) { uri ->
val lease = filePickerOwnerCheckpoint.consume() ?: return@rememberLauncherForActivityResult
if (uri == null) {
@@ -469,7 +475,7 @@ fun ChatScreen(
) {
listOfNotNull(
try {
loadPickedAudioOrDocumentAttachment(resolver, uri)
loadPickedMediaOrDocumentAttachment(resolver, uri)
} catch (err: CancellationException) {
throw err
} catch (_: Throwable) {
@@ -798,7 +804,13 @@ fun ChatScreen(
if (!viewModel.isCurrentChatComposerOwner(composerOwner)) return@ChatComposer
val authorizationId = composerState.beginMediaAcquisition(composerOwner) ?: return@ChatComposer
filePickerOwnerCheckpoint.begin(composerOwner, authorizationId)
pickAudioOrDocument.launch(SHARED_AUDIO_DOCUMENT_MIME_TYPES)
pickMediaOrDocument.launch(SHARED_AUDIO_DOCUMENT_MIME_TYPES)
},
onPickVideo = {
if (!viewModel.isCurrentChatComposerOwner(composerOwner)) return@ChatComposer
val authorizationId = composerState.beginMediaAcquisition(composerOwner) ?: return@ChatComposer
filePickerOwnerCheckpoint.begin(composerOwner, authorizationId)
pickMediaOrDocument.launch(SHARED_VIDEO_MIME_TYPES)
},
onRemoveAttachment = { id -> composerState.removeAttachments(composerOwner, setOf(id)) },
voiceNoteState = voiceNoteState,
@@ -1271,7 +1283,7 @@ private fun ChatMessageList(
inlineMediaPlaybackBlocked: Boolean,
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
loadMediaArtifact: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
modifier: Modifier = Modifier,
) {
val baseTimeline =
@@ -1647,7 +1659,7 @@ private fun ChatBubble(
inlineWidgetResolverReady: Boolean,
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
loadMediaArtifact: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
) {
val normalizedRole = role.trim().lowercase(Locale.US)
val isUser = normalizedRole == "user"
@@ -2038,6 +2050,7 @@ private fun ChatComposer(
onOpenModelPicker: () -> Unit,
onPickImages: () -> Unit,
onPickAudioOrDocument: () -> Unit,
onPickVideo: () -> Unit,
onRemoveAttachment: (String) -> Unit,
voiceNoteState: VoiceNoteRecorderState,
voiceNoteElapsedMs: Long,
@@ -2156,6 +2169,7 @@ private fun ChatComposer(
onValueChange = onValueChange,
onPickImages = onPickImages,
onPickAudioOrDocument = onPickAudioOrDocument,
onPickVideo = onPickVideo,
onStartVoiceNote = onStartVoiceNote,
recordVoiceNoteEnabled = recordVoiceNoteEnabled,
dictationActive = dictationActive,
@@ -2600,6 +2614,7 @@ private fun ChatInputPill(
onValueChange: (String) -> Unit,
onPickImages: () -> Unit,
onPickAudioOrDocument: () -> Unit,
onPickVideo: () -> Unit,
onStartVoiceNote: () -> Unit,
recordVoiceNoteEnabled: Boolean,
dictationActive: Boolean,
@@ -2635,6 +2650,11 @@ private fun ChatInputPill(
Icon(imageVector = Icons.Default.AttachFile, contentDescription = nativeString("Attachment"), modifier = Modifier.size(20.dp))
}
}
Surface(onClick = onPickVideo, modifier = Modifier.size(ClawTheme.spacing.touchTarget), shape = CircleShape, color = ClawTheme.colors.surfaceRaised, contentColor = ClawTheme.colors.text) {
Box(contentAlignment = Alignment.Center) {
Icon(imageVector = Icons.Default.Videocam, contentDescription = nativeString("Attach video"), modifier = Modifier.size(20.dp))
}
}
Box(modifier = Modifier.weight(1f)) {
ChatTextFieldValueAdapter(
value = value,
@@ -2761,6 +2781,10 @@ private fun AttachmentChip(
attachment: PendingAttachment,
onRemove: () -> Unit,
) {
val videoThumbnail =
remember(attachment.videoThumbnailBase64) {
attachment.videoThumbnailBase64?.let(::decodeBase64Bitmap)
}
Surface(
shape = RoundedCornerShape(ClawTheme.radii.pill),
color = ClawTheme.colors.surfaceRaised,
@@ -2774,6 +2798,17 @@ private fun AttachmentChip(
) {
if (attachment.mimeType.startsWith("audio/")) {
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(14.dp), tint = ClawTheme.colors.textMuted)
} else if (attachment.mimeType.startsWith("video/")) {
if (videoThumbnail != null) {
Image(
bitmap = videoThumbnail.asImageBitmap(),
contentDescription = null,
contentScale = ContentScale.Crop,
modifier = Modifier.size(28.dp).clip(RoundedCornerShape(5.dp)),
)
} else {
Icon(imageVector = Icons.Default.Videocam, contentDescription = null, modifier = Modifier.size(14.dp), tint = ClawTheme.colors.textMuted)
}
}
Text(
text =

View File

@@ -19,6 +19,7 @@ data class PendingAttachment(
val mimeType: String,
val base64: String,
val durationMs: Long? = null,
val videoThumbnailBase64: String? = null,
)
internal data class ChatComposerAttachmentMigration(
@@ -205,6 +206,7 @@ internal fun List<SessionEditorAttachment>.toPendingAttachments(): List<PendingA
internal fun attachmentTypeForMimeType(mimeType: String): String =
when {
mimeType.startsWith("audio/") -> "audio"
mimeType.startsWith("video/") -> "video"
mimeType.startsWith("image/") -> "image"
else -> "file"
}

View File

@@ -129,6 +129,24 @@ class ShareLaunchTest {
assertTrue(parsed.attachments.all { it.kind == SharedAttachmentKind.Audio })
}
@Test
fun acceptsVideoShareFromProviderMimeType() {
val video = Uri.parse("content://media/shared/video")
val parsed =
parseShare(
intent =
Intent(Intent.ACTION_SEND)
.setType("video/*")
.putExtra(Intent.EXTRA_STREAM, video),
mimeTypes = mapOf(video to "video/mp4"),
)
requireNotNull(parsed)
assertEquals(SharedAttachmentKind.Video, parsed.attachments.single().kind)
assertEquals("video/mp4", parsed.attachments.single().mimeType)
assertTrue("video/*" in SHARED_ATTACHMENT_MIME_ALLOWLIST)
}
@Test
fun acceptsCuratedDocumentShare() {
val document = Uri.parse("content://docs/shared/report")

View File

@@ -107,7 +107,7 @@ class ChatControllerOutboxTest {
enqueueGate?.await()
if (gatewayIds.values.count { it == gatewayId } >= capacity) return ChatOutboxEnqueueResult.QueueFull
val commandBytes = attachments.sumOf { it.bytes.size.toLong() }
if (commandBytes > OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES) return ChatOutboxEnqueueResult.AttachmentsTooLarge
if (!outboxCommandAttachmentsWithinByteLimits(attachments)) return ChatOutboxEnqueueResult.AttachmentsTooLarge
val queuedBytes = attachmentBytes.values.sumOf { list -> list.sumOf { it.size.toLong() } }
if (commandBytes > 0 && queuedBytes + commandBytes > OUTBOX_MAX_GATEWAY_ATTACHMENT_BYTES) {
return ChatOutboxEnqueueResult.StorageFull

View File

@@ -332,6 +332,26 @@ class ChatMessageContentParsingTest {
)
}
@Test
fun parsesSupportedPlaybackRenditions() {
val direct =
Json.parseToJsonElement(
"""{"type":"video","mimeType":"video/mp4","playback":"transcode"}""",
)
val attachment =
Json.parseToJsonElement(
"""{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mp4","playback":"native"}}""",
)
val unsupported =
Json.parseToJsonElement(
"""{"type":"video","mimeType":"video/mp4","playback":"future"}""",
)
assertEquals("transcode", parseChatMessageContent(direct)?.playback)
assertEquals("native", parseChatMessageContent(attachment)?.playback)
assertEquals(null, parseChatMessageContent(unsupported)?.playback)
}
@Test
fun dropsOversizedInlineImageContentBeforeRendering() {
val oversized = "A".repeat(CHAT_IMAGE_MAX_BASE64_CHARS + 1)

View File

@@ -801,6 +801,59 @@ class RoomChatCommandOutboxTest {
assertTrue(store.load("gateway-a").isEmpty())
}
@Test
fun videoCommandsUseServerAttachmentCapWithoutRaisingOtherMediaCaps() =
runTest {
val aboveDefaultCap = ByteArray((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 1L).toInt())
val video =
store.enqueue(
gatewayId = "gateway-a",
sessionKey = "main",
text = "video",
thinkingLevel = "off",
nowMs = 10,
ownerAgentId = "main",
attachments = listOf(payload(aboveDefaultCap, type = "video", mimeType = "video/mp4")),
)
val document =
store.enqueue(
gatewayId = "gateway-a",
sessionKey = "main",
text = "document",
thinkingLevel = "off",
nowMs = 11,
ownerAgentId = "main",
attachments = listOf(payload(aboveDefaultCap, type = "file", mimeType = "application/pdf")),
)
assertTrue(video is ChatOutboxEnqueueResult.Queued)
assertEquals(ChatOutboxEnqueueResult.AttachmentsTooLarge, document)
assertEquals(20L * 1024L * 1024L, OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES)
}
@Test
fun mixedVideoCommandKeepsNonVideoAggregateCap() =
runTest {
val document = ByteArray(5 * 1024 * 1024)
val refused =
store.enqueue(
gatewayId = "gateway-a",
sessionKey = "main",
text = "mixed",
thinkingLevel = "off",
nowMs = 10,
ownerAgentId = "main",
attachments =
listOf(
payload(document, fileName = "one.pdf", type = "file", mimeType = "application/pdf"),
payload(document, fileName = "two.pdf", type = "file", mimeType = "application/pdf"),
payload(byteArrayOf(1), fileName = "clip.mp4", type = "video", mimeType = "video/mp4"),
),
)
assertEquals(ChatOutboxEnqueueResult.AttachmentsTooLarge, refused)
}
@Test
fun gatewayAttachmentByteBudgetRefusesWhenExhaustedAndRecoversAfterDelete() =
runTest {

View File

@@ -95,6 +95,7 @@ class RoomChatTranscriptCacheTest {
fileName = "demo.mp4",
artifactId = "artifact_managed_media_44444444-4444-4444-8444-444444444444",
durationMs = 5_300,
playback = "transcode",
width = 1920,
height = 1080,
)

View File

@@ -12,6 +12,8 @@ import kotlinx.coroutines.withTimeout
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.WebSocket
import okhttp3.WebSocketListener
@@ -31,6 +33,7 @@ import org.robolectric.RuntimeEnvironment
import org.robolectric.annotation.Config
import java.util.UUID
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicReference
private const val TEST_TIMEOUT_MS = 8_000L
@@ -76,6 +79,12 @@ class GatewaySessionCustomHeadersTest {
val videoAttachmentId = "22222222-2222-4222-8222-222222222222"
val videoArtifactId = "artifact_managed_media_$videoAttachmentId"
val videoPath = "/api/chat/media/outgoing/main/$videoAttachmentId/full?mediaTicket=video-ticket"
val audioAttachmentId = "33333333-3333-4333-8333-333333333333"
val audioArtifactId = "artifact_managed_media_$audioAttachmentId"
val audioPath = "/api/chat/media/outgoing/main/$audioAttachmentId/full?mediaTicket=audio-ticket"
val audioPlaybackPath = "$audioPath&playback=1"
val audioBytes = byteArrayOf(5, 6, 7, 8)
val audioRequestCount = AtomicInteger()
val server =
MockWebServer().apply {
dispatcher =
@@ -87,6 +96,14 @@ class GatewaySessionCustomHeadersTest {
.setHeader("Content-Type", "image/png")
.setBody(Buffer().write(imageBytes))
}
if (request.path == audioPlaybackPath) {
if (audioRequestCount.incrementAndGet() == 1) {
return MockResponse().setResponseCode(202).setBody("""{"status":"preparing"}""")
}
return MockResponse()
.setHeader("Content-Type", "audio/mp4")
.setBody(Buffer().write(audioBytes))
}
return MockResponse().withWebSocketUpgrade(
object : WebSocketListener() {
override fun onOpen(
@@ -118,6 +135,15 @@ class GatewaySessionCustomHeadersTest {
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"artifact":{"id":"$videoArtifactId","type":"video","mimeType":"video/mp4","download":{"mode":"url"}},"url":"$videoPath"}}""",
)
} else if (frame["params"]
?.jsonObject
?.get("artifactId")
?.jsonPrimitive
?.content == audioArtifactId
) {
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"artifact":{"id":"$audioArtifactId","type":"audio","mimeType":"audio/mp4","download":{"mode":"url"}},"url":"$audioPath"}}""",
)
} else {
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""",
@@ -184,6 +210,17 @@ class GatewaySessionCustomHeadersTest {
assertEquals("http://127.0.0.1:${server.port}$videoPath", streamed.url)
assertEquals("video/*", streamed.headers["Accept"])
assertEquals("video/mp4", streamed.mimeType)
assertEquals(false, streamed.retryPreparingPlayback)
val transcodedVideo =
session.loadMediaArtifact(stableId, "main", "main", videoArtifactId, GatewayMediaKind.Video, true) as GatewayLoadedMedia.Streaming
assertEquals("http://127.0.0.1:${server.port}$videoPath&playback=1", transcodedVideo.url)
assertTrue(transcodedVideo.retryPreparingPlayback)
val audio =
session.loadMediaArtifact(stableId, "main", "main", audioArtifactId, GatewayMediaKind.Audio, true) as GatewayLoadedMedia.Buffered
assertArrayEquals(audioBytes, audio.bytes)
assertEquals(2, audioRequestCount.get())
} finally {
session.disconnectAndJoin()
scope.cancel()
@@ -191,6 +228,74 @@ class GatewaySessionCustomHeadersTest {
}
}
@Test
fun preparingPlaybackInterceptorRetries202WithoutSurfacingLoadError() {
val server = MockWebServer()
server.enqueue(MockResponse().setResponseCode(202).setBody("""{"status":"preparing"}"""))
server.enqueue(MockResponse().setResponseCode(200).setBody("ready"))
server.start()
var nowMs = 0L
val client =
OkHttpClient
.Builder()
.addInterceptor(
GatewayPreparingPlaybackInterceptor(
policy = GatewayPlaybackRetryPolicy(maxElapsedMs = 100L, initialDelayMs = 0L, maxDelayMs = 0L),
nowMs = { nowMs++ },
sleepMs = {},
),
).build()
try {
client.newCall(Request.Builder().url(server.url("/video?playback=1")).build()).execute().use { response ->
assertEquals(200, response.code)
assertEquals("ready", response.body.string())
}
assertEquals(2, server.requestCount)
} finally {
server.shutdown()
}
}
@Test
fun preparingPlaybackRetryStopsAtTwoMinuteCap() {
val retry = GatewayPlaybackRetryState(startedAtMs = 1_000L)
assertTrue(retry.canAttempt(nowMs = 1_000L))
assertEquals(500L, retry.nextDelayMs(nowMs = 1_000L))
assertEquals(false, retry.canAttempt(nowMs = 121_000L))
assertNull(retry.nextDelayMs(nowMs = 121_001L))
}
@Test
fun preparingPlaybackInterceptorDoesNotStartRequestAfterOvershootingDeadline() {
val server = MockWebServer()
server.enqueue(MockResponse().setResponseCode(202).setBody("""{"status":"preparing"}"""))
server.start()
var nowMs = 0L
val client =
OkHttpClient
.Builder()
.addInterceptor(
GatewayPreparingPlaybackInterceptor(
policy = GatewayPlaybackRetryPolicy(maxElapsedMs = 2L, initialDelayMs = 1L, maxDelayMs = 1L),
nowMs = { nowMs },
sleepMs = { delayMs -> nowMs += delayMs + 1L },
),
).build()
try {
val failure =
runCatching {
client.newCall(Request.Builder().url(server.url("/video?playback=1")).build()).execute().use { }
}.exceptionOrNull()
assertTrue(failure is java.io.IOException)
assertEquals(1, server.requestCount)
} finally {
server.shutdown()
}
}
@Test
fun tlsUpgradeRequest_carriesLatestSanitizedHeadersForOnlyThisGateway() {
val app = RuntimeEnvironment.getApplication()

View File

@@ -716,7 +716,29 @@ class ChatComposerDraftTest {
fun attachmentAdmissionUsesPerKindDecodedBudgets() {
assertEquals(CHAT_COMPOSER_MAX_IMAGE_DECODED_BYTES, chatComposerAttachmentDecodedByteLimit("image/png"))
assertEquals(CHAT_COMPOSER_MAX_AUDIO_DECODED_BYTES, chatComposerAttachmentDecodedByteLimit("audio/mpeg"))
assertEquals(CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES, chatComposerAttachmentDecodedByteLimit("video/mp4"))
assertEquals(CHAT_COMPOSER_MAX_DOCUMENT_DECODED_BYTES, chatComposerAttachmentDecodedByteLimit("application/pdf"))
assertEquals(20L * 1024L * 1024L, CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES)
}
@Test
fun videoPositionDoesNotRelaxNonVideoAdmissionBudget() {
val video = PendingAttachment("video", "clip.mp4", "video/mp4", "AAAA")
val document = PendingAttachment("document", "report.pdf", "application/pdf", "AAAAAAAA")
fun admit(candidates: List<PendingAttachment>) =
admitChatAttachments(
currentAttachments = emptyList(),
candidates = candidates,
maxAttachmentCount = 8,
maxBase64Chars = 100,
maxDecodedBytes = 9,
maxNonVideoBase64Chars = 100,
maxNonVideoDecodedBytes = 3,
)
assertEquals(listOf(video), admit(listOf(video, document)).accepted)
assertEquals(listOf(video), admit(listOf(document, video)).accepted)
}
@Test

View File

@@ -7,6 +7,7 @@ import androidx.compose.ui.test.assertIsDisplayed
import androidx.compose.ui.test.junit4.v2.createComposeRule
import androidx.compose.ui.test.onAllNodesWithContentDescription
import androidx.compose.ui.test.onNodeWithText
import androidx.media3.common.Player
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertSame
@@ -39,6 +40,10 @@ class ChatMediaPlayerTest {
}
}
private class FakeSession {
var released = false
}
@get:Rule
val composeRule = createComposeRule()
@@ -75,6 +80,47 @@ class ChatMediaPlayerTest {
assertSame(player, claims.active)
}
@Test
fun playbackClaimsCreateAndReleaseOnlyOneMediaSession() {
val first = FakePlayer()
val second = FakePlayer()
val sessions = ChatMediaSessionLifecycle<FakePlayer, FakeSession> { it.released = true }
val claims =
ChatMediaPlaybackClaims<FakePlayer>(
pause = { player -> sessions.release(player) },
release = { player -> sessions.release(player) },
)
claims.claim(first)
val firstSession = sessions.activate(first) { FakeSession() }
assertSame(firstSession, sessions.activate(first) { error("duplicate session") })
claims.claim(second)
val secondSession = sessions.activate(second) { FakeSession() }
assertTrue(firstSession.released)
assertFalse(secondSession.released)
claims.pauseIf { it === second }
assertTrue(secondSession.released)
}
@Test
fun mediaSessionControllersCannotReplaceInlineMediaItem() {
val commands =
inlineMediaSessionPlayerCommands(
Player.Commands
.Builder()
.addAllCommands()
.build(),
)
assertTrue(commands.contains(Player.COMMAND_PLAY_PAUSE))
assertTrue(commands.contains(Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM))
assertFalse(commands.contains(Player.COMMAND_SET_MEDIA_ITEM))
assertFalse(commands.contains(Player.COMMAND_CHANGE_MEDIA_ITEMS))
assertFalse(commands.contains(Player.COMMAND_STOP))
}
@Test
fun legacyMediaPartsRenderLabelsWithoutPlayControlsOrClaims() {
val audio =
@@ -102,7 +148,7 @@ class ChatMediaPlayerTest {
content = listOf(audio, video),
timestampMs = 1,
),
loadMediaArtifact = { _, _ ->
loadMediaArtifact = { _, _, _ ->
loadCount += 1
null
},

View File

@@ -90,6 +90,7 @@ kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serializa
material = { module = "com.google.android.material:material", version.ref = "material" }
media3-datasource-okhttp = { module = "androidx.media3:media3-datasource-okhttp", version.ref = "media3" }
media3-exoplayer = { module = "androidx.media3:media3-exoplayer", version.ref = "media3" }
media3-session = { module = "androidx.media3:media3-session", version.ref = "media3" }
media3-ui = { module = "androidx.media3:media3-ui", version.ref = "media3" }
mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp" }
okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }