mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-02 09:01:33 +00:00
feat(android): media renditions, MediaSession, video upload (#116037)
* feat(android): add media rendition and video support * chore(android): refresh native i18n inventory * fix(android): opt in to Media3 command API * chore(android): sync native i18n source locations
This commit is contained in:
committed by
GitHub
parent
d6f9affe79
commit
e1cc617f86
File diff suppressed because it is too large
Load Diff
@@ -338,6 +338,7 @@ dependencies {
|
||||
implementation(libs.okhttp)
|
||||
implementation(libs.media3.datasource.okhttp)
|
||||
implementation(libs.media3.exoplayer)
|
||||
implementation(libs.media3.session)
|
||||
implementation(libs.media3.ui)
|
||||
implementation(libs.bcprov)
|
||||
implementation(libs.coil.compose)
|
||||
|
||||
@@ -131,6 +131,12 @@
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
<data android:mimeType="audio/*" />
|
||||
</intent-filter>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.SEND" />
|
||||
<action android:name="android.intent.action.SEND_MULTIPLE" />
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
<data android:mimeType="video/*" />
|
||||
</intent-filter>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.SEND" />
|
||||
<action android:name="android.intent.action.SEND_MULTIPLE" />
|
||||
|
||||
@@ -45,6 +45,7 @@ data class ShareLaunchRequest(
|
||||
enum class SharedAttachmentKind {
|
||||
Image,
|
||||
Audio,
|
||||
Video,
|
||||
Document,
|
||||
}
|
||||
|
||||
@@ -63,6 +64,7 @@ internal val SHARED_ATTACHMENT_MIME_ALLOWLIST =
|
||||
setOf(
|
||||
"image/*",
|
||||
"audio/*",
|
||||
"video/*",
|
||||
"application/pdf",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
@@ -71,7 +73,9 @@ internal val SHARED_ATTACHMENT_MIME_ALLOWLIST =
|
||||
"text/markdown",
|
||||
)
|
||||
|
||||
internal val SHARED_AUDIO_DOCUMENT_MIME_TYPES = SHARED_ATTACHMENT_MIME_ALLOWLIST.filterNot { it == "image/*" }.toTypedArray()
|
||||
internal val SHARED_AUDIO_DOCUMENT_MIME_TYPES =
|
||||
SHARED_ATTACHMENT_MIME_ALLOWLIST.filterNot { it == "image/*" || it == "video/*" }.toTypedArray()
|
||||
internal val SHARED_VIDEO_MIME_TYPES = arrayOf("video/*")
|
||||
|
||||
/**
|
||||
* Parses app-owned navigation actions that should open a specific home tab.
|
||||
@@ -197,6 +201,7 @@ internal fun sharedAttachmentKindForMimeType(mimeType: String?): SharedAttachmen
|
||||
return when {
|
||||
normalized.startsWith("image/") -> SharedAttachmentKind.Image
|
||||
normalized.startsWith("audio/") -> SharedAttachmentKind.Audio
|
||||
normalized.startsWith("video/") -> SharedAttachmentKind.Video
|
||||
normalized in SHARED_ATTACHMENT_MIME_ALLOWLIST -> SharedAttachmentKind.Document
|
||||
else -> null
|
||||
}
|
||||
|
||||
@@ -1314,7 +1314,8 @@ class MainViewModel private constructor(
|
||||
internal suspend fun loadChatMediaArtifact(
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
) = ensureRuntime().loadChatMediaArtifact(artifactId, kind)
|
||||
playbackRendition: Boolean,
|
||||
) = ensureRuntime().loadChatMediaArtifact(artifactId, kind, playbackRendition)
|
||||
|
||||
fun requestCanvasRehydrate(source: String = "screen_tab") {
|
||||
ensureRuntime().requestCanvasRehydrate(source = source, force = true)
|
||||
|
||||
@@ -4919,7 +4919,8 @@ class NodeRuntime private constructor(
|
||||
internal suspend fun loadChatMediaArtifact(
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
) = chat.loadMediaArtifact(artifactId, kind)
|
||||
playbackRendition: Boolean,
|
||||
) = chat.loadMediaArtifact(artifactId, kind, playbackRendition)
|
||||
|
||||
fun loadChat(
|
||||
sessionKey: String,
|
||||
|
||||
@@ -50,6 +50,10 @@ internal const val OUTBOX_ATTACHMENT_CHUNK_BYTES = 512 * 1024
|
||||
/** Upper bound of attachment bytes on one queued command (8 images plus a voice note fit). */
|
||||
internal const val OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES = 8L * 1024L * 1024L
|
||||
|
||||
// Mirrors DEFAULT_CHAT_ATTACHMENT_MAX_MB in src/gateway/chat-attachments.ts. Only video raises
|
||||
// the durable command ceiling; existing image, audio, and document admission stays at 8 MiB.
|
||||
internal const val OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES = 20L * 1024L * 1024L
|
||||
|
||||
/** Upper bound of queued attachment bytes per gateway so the outbox database stays bounded. */
|
||||
internal const val OUTBOX_MAX_GATEWAY_ATTACHMENT_BYTES = 48L * 1024L * 1024L
|
||||
|
||||
@@ -155,6 +159,20 @@ class LoadedOutboxAttachment(
|
||||
val bytes: ByteArray,
|
||||
)
|
||||
|
||||
private fun OutboxAttachmentPayload.isVideo(): Boolean = type == "video" || mimeType.startsWith("video/", ignoreCase = true)
|
||||
|
||||
internal fun outboxCommandAttachmentsWithinByteLimits(attachments: List<OutboxAttachmentPayload>): Boolean {
|
||||
val totalBytes = attachments.sumOf { it.bytes.size.toLong() }
|
||||
val nonVideoBytes = attachments.filterNot(OutboxAttachmentPayload::isVideo).sumOf { it.bytes.size.toLong() }
|
||||
val totalLimit =
|
||||
if (attachments.any(OutboxAttachmentPayload::isVideo)) {
|
||||
OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES
|
||||
} else {
|
||||
OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
|
||||
}
|
||||
return totalBytes <= totalLimit && nonVideoBytes <= OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
|
||||
}
|
||||
|
||||
sealed interface ChatOutboxEnqueueResult {
|
||||
data class Queued(
|
||||
val item: ChatOutboxItem,
|
||||
@@ -162,7 +180,7 @@ sealed interface ChatOutboxEnqueueResult {
|
||||
|
||||
data object QueueFull : ChatOutboxEnqueueResult
|
||||
|
||||
/** One command's attachments exceed [OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES]; deleting rows cannot help. */
|
||||
/** One command's attachments exceed its media-kind byte ceiling; deleting rows cannot help. */
|
||||
data object AttachmentsTooLarge : ChatOutboxEnqueueResult
|
||||
|
||||
/** The per-gateway attachment byte budget is exhausted; deleting queued rows frees space. */
|
||||
@@ -696,7 +714,7 @@ class RoomChatCommandOutbox internal constructor(
|
||||
val key = sessionKey.trim().takeIf { it.isNotEmpty() } ?: return ChatOutboxEnqueueResult.Unavailable
|
||||
val owner = normalizedOutboxOwnerAgentId(ownerAgentId) ?: return ChatOutboxEnqueueResult.Unavailable
|
||||
val attachmentBytes = attachments.sumOf { it.bytes.size.toLong() }
|
||||
if (attachmentBytes > OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES) {
|
||||
if (!outboxCommandAttachmentsWithinByteLimits(attachments)) {
|
||||
return ChatOutboxEnqueueResult.AttachmentsTooLarge
|
||||
}
|
||||
val dao = database.outboxDao()
|
||||
|
||||
@@ -128,7 +128,8 @@ class ChatController internal constructor(
|
||||
agentId: String?,
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
) -> GatewayLoadedMedia? = { _, _, _, _, _ -> null },
|
||||
playbackRendition: Boolean,
|
||||
) -> GatewayLoadedMedia? = { _, _, _, _, _, _ -> null },
|
||||
private val commandOutbox: ChatCommandOutbox? = null,
|
||||
private val recordModelRecent: (String) -> Unit = {},
|
||||
private val onSessionDeleted: (ChatSessionDeletion) -> Unit = {},
|
||||
@@ -163,8 +164,8 @@ class ChatController internal constructor(
|
||||
loadGatewayImageArtifact = { gatewayId, sessionKey, agentId, artifactId ->
|
||||
session.loadImageArtifact(gatewayId, sessionKey, agentId, artifactId)
|
||||
},
|
||||
loadGatewayMediaArtifact = { gatewayId, sessionKey, agentId, artifactId, kind ->
|
||||
session.loadMediaArtifact(gatewayId, sessionKey, agentId, artifactId, kind)
|
||||
loadGatewayMediaArtifact = { gatewayId, sessionKey, agentId, artifactId, kind, playbackRendition ->
|
||||
session.loadMediaArtifact(gatewayId, sessionKey, agentId, artifactId, kind, playbackRendition)
|
||||
},
|
||||
commandOutbox = commandOutbox,
|
||||
recordModelRecent = recordModelRecent,
|
||||
@@ -186,6 +187,7 @@ class ChatController internal constructor(
|
||||
suspend fun loadMediaArtifact(
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
playbackRendition: Boolean,
|
||||
): GatewayLoadedMedia? {
|
||||
val normalizedArtifactId = artifactId.trim().takeIf(String::isNotEmpty) ?: return null
|
||||
if (kind == GatewayMediaKind.Image) return null
|
||||
@@ -196,6 +198,7 @@ class ChatController internal constructor(
|
||||
resolveAgentIdForSessionKey(sessionKey),
|
||||
normalizedArtifactId,
|
||||
kind,
|
||||
playbackRendition,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -6718,6 +6721,7 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
sizeBytes = obj["sizeBytes"].asLongOrNull(),
|
||||
base64 = inlineContent?.takeIf { type == "image" && it.length <= CHAT_IMAGE_MAX_BASE64_CHARS },
|
||||
durationMs = obj["durationMs"].asLongOrNull(),
|
||||
playback = obj["playback"].asStringOrNull()?.takeIf { it == "native" || it == "transcode" },
|
||||
)
|
||||
}
|
||||
|
||||
@@ -6743,6 +6747,7 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
height = attachment["height"].asLongOrNull()?.toInt(),
|
||||
sizeBytes = attachment["sizeBytes"].asLongOrNull(),
|
||||
durationMs = attachment["durationMs"].asLongOrNull(),
|
||||
playback = attachment["playback"].asStringOrNull()?.takeIf { it == "native" || it == "transcode" },
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -79,6 +79,7 @@ data class ChatMessageContent(
|
||||
val sizeBytes: Long? = null,
|
||||
val base64: String? = null,
|
||||
val durationMs: Long? = null,
|
||||
val playback: String? = null,
|
||||
val widget: ChatWidgetPreview? = null,
|
||||
)
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ private data class CachedMessageContent(
|
||||
val height: Int? = null,
|
||||
val sizeBytes: Long? = null,
|
||||
val durationMs: Long? = null,
|
||||
val playback: String? = null,
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -320,6 +321,7 @@ class RoomChatTranscriptCache internal constructor(
|
||||
height = part.height,
|
||||
sizeBytes = part.sizeBytes,
|
||||
durationMs = part.durationMs,
|
||||
playback = part.playback,
|
||||
)
|
||||
},
|
||||
timestampMs = row.timestampMs,
|
||||
@@ -438,6 +440,7 @@ class RoomChatTranscriptCache internal constructor(
|
||||
height = part.height,
|
||||
sizeBytes = part.sizeBytes,
|
||||
durationMs = part.durationMs,
|
||||
playback = part.playback,
|
||||
)
|
||||
else -> null
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package ai.openclaw.app.gateway
|
||||
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
@@ -14,6 +15,7 @@ import kotlinx.coroutines.TimeoutCancellationException
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.cancelAndJoin
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
@@ -28,12 +30,14 @@ import kotlinx.serialization.json.JsonNull
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import okhttp3.Interceptor
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import okhttp3.WebSocket
|
||||
import okhttp3.WebSocketListener
|
||||
import okio.Buffer
|
||||
import java.io.IOException
|
||||
import java.net.URI
|
||||
import java.util.Base64
|
||||
import java.util.Locale
|
||||
@@ -84,9 +88,63 @@ sealed interface GatewayLoadedMedia {
|
||||
val headers: Map<String, String>,
|
||||
val client: OkHttpClient,
|
||||
val mimeType: String?,
|
||||
val retryPreparingPlayback: Boolean,
|
||||
) : GatewayLoadedMedia
|
||||
}
|
||||
|
||||
internal data class GatewayPlaybackRetryPolicy(
|
||||
val maxElapsedMs: Long = 120_000L,
|
||||
val initialDelayMs: Long = 500L,
|
||||
val maxDelayMs: Long = 5_000L,
|
||||
) {
|
||||
init {
|
||||
require(maxElapsedMs > 0L && initialDelayMs >= 0L && maxDelayMs >= initialDelayMs)
|
||||
}
|
||||
}
|
||||
|
||||
/** Shared bounded backoff for both buffered fetches and Media3's streaming HTTP path. */
|
||||
internal class GatewayPlaybackRetryState(
|
||||
private val policy: GatewayPlaybackRetryPolicy = GatewayPlaybackRetryPolicy(),
|
||||
private val startedAtMs: Long = SystemClock.elapsedRealtime(),
|
||||
) {
|
||||
private var attempt = 0
|
||||
|
||||
fun canAttempt(nowMs: Long = SystemClock.elapsedRealtime()): Boolean = nowMs - startedAtMs < policy.maxElapsedMs
|
||||
|
||||
fun nextDelayMs(nowMs: Long = SystemClock.elapsedRealtime()): Long? {
|
||||
val remainingMs = policy.maxElapsedMs - (nowMs - startedAtMs).coerceAtLeast(0L)
|
||||
if (remainingMs <= 0L) return null
|
||||
val multiplier = 1L shl attempt.coerceAtMost(30)
|
||||
val delayMs = (policy.initialDelayMs * multiplier).coerceAtMost(policy.maxDelayMs)
|
||||
attempt += 1
|
||||
return delayMs.coerceAtMost(remainingMs)
|
||||
}
|
||||
}
|
||||
|
||||
/** Keeps Media3's OkHttp data source in its loading state while a rendition is being prepared. */
|
||||
internal class GatewayPreparingPlaybackInterceptor(
|
||||
private val policy: GatewayPlaybackRetryPolicy = GatewayPlaybackRetryPolicy(),
|
||||
private val nowMs: () -> Long = SystemClock::elapsedRealtime,
|
||||
private val sleepMs: (Long) -> Unit = Thread::sleep,
|
||||
) : Interceptor {
|
||||
override fun intercept(chain: Interceptor.Chain): okhttp3.Response {
|
||||
val retry = GatewayPlaybackRetryState(policy = policy, startedAtMs = nowMs())
|
||||
while (true) {
|
||||
if (!retry.canAttempt(nowMs())) throw IOException("playback preparation timed out")
|
||||
val response = chain.proceed(chain.request())
|
||||
if (response.code != 202) return response
|
||||
val retryDelayMs = retry.nextDelayMs(nowMs()) ?: return response
|
||||
response.close()
|
||||
try {
|
||||
sleepMs(retryDelayMs)
|
||||
} catch (error: InterruptedException) {
|
||||
Thread.currentThread().interrupt()
|
||||
throw IOException("playback preparation interrupted", error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Role, scopes, commands, and permission snapshot sent with the connect frame.
|
||||
*/
|
||||
@@ -673,6 +731,7 @@ class GatewaySession(
|
||||
agentId: String?,
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
playbackRendition: Boolean = false,
|
||||
): GatewayLoadedMedia? {
|
||||
val conn = readyConnection(expectedEndpointStableId) ?: return null
|
||||
val params =
|
||||
@@ -709,9 +768,9 @@ class GatewaySession(
|
||||
conn.bufferedMedia(bytes = bytes, mimeType = resolvedMimeType)
|
||||
} ?: payload["url"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty)?.let { ticketedPath ->
|
||||
if (kind == GatewayMediaKind.Video) {
|
||||
conn.resolveTicketedMediaStream(ticketedPath, mimeType)
|
||||
conn.resolveTicketedMediaStream(ticketedPath, mimeType, playbackRendition)
|
||||
} else {
|
||||
conn.loadTicketedMedia(ticketedPath, kind)
|
||||
conn.loadTicketedMedia(ticketedPath, kind, playbackRendition)
|
||||
}
|
||||
} ?: return null
|
||||
return synchronized(lifecycleLock) {
|
||||
@@ -941,13 +1000,15 @@ class GatewaySession(
|
||||
fun resolveTicketedMediaStream(
|
||||
ticketedPath: String,
|
||||
mimeType: String?,
|
||||
playbackRendition: Boolean,
|
||||
): GatewayLoadedMedia.Streaming? {
|
||||
val request = resolveTicketedMediaRequest(ticketedPath) ?: return null
|
||||
val request = resolveTicketedMediaRequest(ticketedPath, playbackRendition) ?: return null
|
||||
return GatewayLoadedMedia.Streaming(
|
||||
url = request.url,
|
||||
headers = request.headers + ("Accept" to "video/*"),
|
||||
client = client,
|
||||
mimeType = mimeType,
|
||||
retryPreparingPlayback = playbackRendition,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -965,38 +1026,53 @@ class GatewaySession(
|
||||
suspend fun loadTicketedMedia(
|
||||
ticketedPath: String,
|
||||
kind: GatewayMediaKind,
|
||||
playbackRendition: Boolean,
|
||||
): GatewayLoadedMedia.Buffered? =
|
||||
withContext(Dispatchers.IO) {
|
||||
if (kind == GatewayMediaKind.Video) return@withContext null
|
||||
val resolved = resolveTicketedMediaRequest(ticketedPath) ?: return@withContext null
|
||||
val resolved = resolveTicketedMediaRequest(ticketedPath, playbackRendition) ?: return@withContext null
|
||||
val request = Request.Builder().url(resolved.url).header("Accept", "${kind.wireValue}/*")
|
||||
for ((name, value) in resolved.headers) {
|
||||
request.header(name, value)
|
||||
}
|
||||
val call = client.newCall(request.build())
|
||||
call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS)
|
||||
call.execute().use { response ->
|
||||
if (!response.isSuccessful) return@withContext null
|
||||
val body = response.body
|
||||
val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null
|
||||
if (!mimeType.startsWith("${kind.wireValue}/")) return@withContext null
|
||||
val maximumBytes = kind.maximumBufferedBytes
|
||||
val declaredLength = body.contentLength()
|
||||
if (declaredLength > maximumBytes) return@withContext null
|
||||
val buffer = Buffer()
|
||||
val source = body.source()
|
||||
var total = 0L
|
||||
while (true) {
|
||||
val read = source.read(buffer, minOf(8192L, maximumBytes + 1L - total))
|
||||
if (read == -1L) break
|
||||
total += read
|
||||
if (total > maximumBytes) return@withContext null
|
||||
val retry = GatewayPlaybackRetryState()
|
||||
repeat(Int.MAX_VALUE) {
|
||||
if (!retry.canAttempt()) return@withContext null
|
||||
val call = client.newCall(request.build())
|
||||
call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS)
|
||||
var retryDelayMs: Long? = null
|
||||
call.execute().use { response ->
|
||||
if (response.code == 202 && playbackRendition) {
|
||||
retryDelayMs = retry.nextDelayMs()
|
||||
} else {
|
||||
if (!response.isSuccessful) return@withContext null
|
||||
val body = response.body
|
||||
val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null
|
||||
if (!mimeType.startsWith("${kind.wireValue}/")) return@withContext null
|
||||
val maximumBytes = kind.maximumBufferedBytes
|
||||
val declaredLength = body.contentLength()
|
||||
if (declaredLength > maximumBytes) return@withContext null
|
||||
val buffer = Buffer()
|
||||
val source = body.source()
|
||||
var total = 0L
|
||||
while (true) {
|
||||
val read = source.read(buffer, minOf(8192L, maximumBytes + 1L - total))
|
||||
if (read == -1L) break
|
||||
total += read
|
||||
if (total > maximumBytes) return@withContext null
|
||||
}
|
||||
return@withContext bufferedMedia(bytes = buffer.readByteArray(), mimeType = mimeType)
|
||||
}
|
||||
}
|
||||
bufferedMedia(bytes = buffer.readByteArray(), mimeType = mimeType)
|
||||
delay(retryDelayMs ?: return@withContext null)
|
||||
}
|
||||
null
|
||||
}
|
||||
|
||||
private fun resolveTicketedMediaRequest(ticketedPath: String): TicketedMediaRequest? {
|
||||
private fun resolveTicketedMediaRequest(
|
||||
ticketedPath: String,
|
||||
playbackRendition: Boolean = false,
|
||||
): TicketedMediaRequest? {
|
||||
val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return null
|
||||
val rawPath = uri.rawPath ?: return null
|
||||
val rawQuery = uri.rawQuery ?: return null
|
||||
@@ -1007,7 +1083,13 @@ class GatewaySession(
|
||||
.any { field -> field.substringBefore('=') == "mediaTicket" && field.substringAfter('=', "").isNotEmpty() }
|
||||
if (!rawPath.startsWith("/api/chat/media/outgoing/") || !hasMediaTicket) return null
|
||||
val scheme = if (tlsConfig != null) "https" else "http"
|
||||
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath"
|
||||
val playbackPath =
|
||||
if (playbackRendition && rawQuery.split('&').none { it.substringBefore('=') == "playback" }) {
|
||||
"$ticketedPath&playback=1"
|
||||
} else {
|
||||
ticketedPath
|
||||
}
|
||||
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$playbackPath"
|
||||
val headers = mediaTransportHeaders()
|
||||
return TicketedMediaRequest(url = url, headers = headers)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import ai.openclaw.app.ChatShareDraft
|
||||
import ai.openclaw.app.SharedAttachment
|
||||
import ai.openclaw.app.chat.ChatComposerOwner
|
||||
import ai.openclaw.app.chat.OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
|
||||
import ai.openclaw.app.chat.OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES
|
||||
import ai.openclaw.app.chat.VoiceNoteRecorderState
|
||||
import androidx.compose.runtime.mutableStateMapOf
|
||||
import androidx.compose.runtime.saveable.listSaver
|
||||
@@ -391,8 +392,9 @@ internal const val CHAT_COMPOSER_MAX_ATTACHMENTS = 8
|
||||
// Gateway chat attachments default to 20 MiB (images: 6 MiB); Android's outbox further caps audio/documents at 8 MiB.
|
||||
internal const val CHAT_COMPOSER_MAX_IMAGE_DECODED_BYTES = 6L * 1024L * 1024L
|
||||
internal const val CHAT_COMPOSER_MAX_AUDIO_DECODED_BYTES = OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
|
||||
internal const val CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES = OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES
|
||||
internal const val CHAT_COMPOSER_MAX_DOCUMENT_DECODED_BYTES = OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
|
||||
internal const val CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES = OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES
|
||||
internal const val CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES = CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES
|
||||
internal const val CHAT_COMPOSER_MAX_BASE64_CHARS = ((CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES + 2) / 3) * 4
|
||||
internal const val CHAT_COMPOSER_MAX_TOTAL_ATTACHMENTS = 24
|
||||
internal const val CHAT_COMPOSER_MAX_TOTAL_DECODED_ATTACHMENT_BYTES = CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES * 3
|
||||
@@ -409,23 +411,44 @@ internal fun admitChatAttachments(
|
||||
maxAttachmentCount: Int = CHAT_COMPOSER_MAX_ATTACHMENTS,
|
||||
maxBase64Chars: Long = CHAT_COMPOSER_MAX_BASE64_CHARS,
|
||||
maxDecodedBytes: Long = CHAT_COMPOSER_MAX_DECODED_ATTACHMENT_BYTES,
|
||||
maxNonVideoBase64Chars: Long = ((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 2) / 3) * 4,
|
||||
maxNonVideoDecodedBytes: Long = OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES,
|
||||
): ChatAttachmentAdmission {
|
||||
require(maxAttachmentCount >= 0 && maxBase64Chars >= 0 && maxDecodedBytes >= 0)
|
||||
require(
|
||||
maxAttachmentCount >= 0 &&
|
||||
maxBase64Chars >= 0 &&
|
||||
maxDecodedBytes >= 0 &&
|
||||
maxNonVideoBase64Chars >= 0 &&
|
||||
maxNonVideoDecodedBytes >= 0,
|
||||
)
|
||||
val accepted = mutableListOf<PendingAttachment>()
|
||||
var base64Chars = currentAttachments.sumOf { it.base64.length.toLong() }
|
||||
var decodedBytes = currentAttachments.sumOf { decodedBase64ByteCount(it.base64) }
|
||||
var nonVideoBase64Chars =
|
||||
currentAttachments.filterNot { it.mimeType.startsWith("video/", ignoreCase = true) }.sumOf { it.base64.length.toLong() }
|
||||
var nonVideoDecodedBytes =
|
||||
currentAttachments
|
||||
.filterNot { it.mimeType.startsWith("video/", ignoreCase = true) }
|
||||
.sumOf { decodedBase64ByteCount(it.base64) }
|
||||
var omittedCount = 0
|
||||
for (candidate in candidates) {
|
||||
val candidateBase64Chars = candidate.base64.length.toLong()
|
||||
val candidateDecodedBytes = decodedBase64ByteCount(candidate.base64)
|
||||
val candidateIsVideo = candidate.mimeType.startsWith("video/", ignoreCase = true)
|
||||
val withinKind = candidateDecodedBytes <= chatComposerAttachmentDecodedByteLimit(candidate.mimeType)
|
||||
val withinCount = currentAttachments.size + accepted.size < maxAttachmentCount
|
||||
val withinBase64 = candidateBase64Chars <= maxBase64Chars - base64Chars
|
||||
val withinDecoded = candidateDecodedBytes <= maxDecodedBytes - decodedBytes
|
||||
if (withinKind && withinCount && withinBase64 && withinDecoded) {
|
||||
val withinNonVideoBase64 = candidateIsVideo || candidateBase64Chars <= maxNonVideoBase64Chars - nonVideoBase64Chars
|
||||
val withinNonVideoDecoded = candidateIsVideo || candidateDecodedBytes <= maxNonVideoDecodedBytes - nonVideoDecodedBytes
|
||||
if (withinKind && withinCount && withinBase64 && withinDecoded && withinNonVideoBase64 && withinNonVideoDecoded) {
|
||||
accepted += candidate
|
||||
base64Chars += candidateBase64Chars
|
||||
decodedBytes += candidateDecodedBytes
|
||||
if (!candidateIsVideo) {
|
||||
nonVideoBase64Chars += candidateBase64Chars
|
||||
nonVideoDecodedBytes += candidateDecodedBytes
|
||||
}
|
||||
} else {
|
||||
omittedCount += 1
|
||||
}
|
||||
@@ -437,6 +460,7 @@ internal fun chatComposerAttachmentDecodedByteLimit(mimeType: String): Long =
|
||||
when {
|
||||
mimeType.startsWith("image/", ignoreCase = true) -> CHAT_COMPOSER_MAX_IMAGE_DECODED_BYTES
|
||||
mimeType.startsWith("audio/", ignoreCase = true) -> CHAT_COMPOSER_MAX_AUDIO_DECODED_BYTES
|
||||
mimeType.startsWith("video/", ignoreCase = true) -> CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES
|
||||
else -> CHAT_COMPOSER_MAX_DOCUMENT_DECODED_BYTES
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import android.content.ContentResolver
|
||||
import android.database.Cursor
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.BitmapFactory
|
||||
import android.media.MediaMetadataRetriever
|
||||
import android.net.Uri
|
||||
import android.provider.OpenableColumns
|
||||
import android.util.Base64
|
||||
@@ -24,6 +25,8 @@ private const val CHAT_ATTACHMENT_MAX_WIDTH = 1600
|
||||
private const val CHAT_ATTACHMENT_START_QUALITY = 85
|
||||
private const val CHAT_DECODE_MAX_DIMENSION = 1600
|
||||
private const val CHAT_IMAGE_CACHE_BYTES = 16 * 1024 * 1024
|
||||
private const val VIDEO_THUMBNAIL_MAX_DIMENSION = 192
|
||||
private const val VIDEO_THUMBNAIL_QUALITY = 72
|
||||
|
||||
private val decodedBitmapCache =
|
||||
object : LruCache<String, Bitmap>(CHAT_IMAGE_CACHE_BYTES) {
|
||||
@@ -33,7 +36,7 @@ private val decodedBitmapCache =
|
||||
): Int = value.byteCount.coerceAtLeast(1)
|
||||
}
|
||||
|
||||
internal fun loadPickedAudioOrDocumentAttachment(
|
||||
internal fun loadPickedMediaOrDocumentAttachment(
|
||||
resolver: ContentResolver,
|
||||
uri: Uri,
|
||||
): PendingAttachment {
|
||||
@@ -65,9 +68,55 @@ internal fun loadSharedAttachment(
|
||||
fileName = sharedAttachmentFileName(resolver, attachment.uri),
|
||||
mimeType = mimeType,
|
||||
base64 = Base64.encodeToString(bytes, Base64.NO_WRAP),
|
||||
videoThumbnailBase64 =
|
||||
if (kind == SharedAttachmentKind.Video) loadVideoThumbnailBase64(resolver, attachment.uri) else null,
|
||||
)
|
||||
}
|
||||
|
||||
/** Thumbnail extraction is presentation-only; an unsupported container still stages as a video. */
|
||||
private fun loadVideoThumbnailBase64(
|
||||
resolver: ContentResolver,
|
||||
uri: Uri,
|
||||
): String? =
|
||||
runCatching {
|
||||
val retriever = MediaMetadataRetriever()
|
||||
try {
|
||||
resolver.openAssetFileDescriptor(uri, "r")?.use { descriptor ->
|
||||
if (descriptor.declaredLength >= 0L) {
|
||||
retriever.setDataSource(descriptor.fileDescriptor, descriptor.startOffset, descriptor.declaredLength)
|
||||
} else {
|
||||
retriever.setDataSource(descriptor.fileDescriptor)
|
||||
}
|
||||
} ?: return@runCatching null
|
||||
val frame = retriever.getFrameAtTime(-1L, MediaMetadataRetriever.OPTION_CLOSEST_SYNC) ?: return@runCatching null
|
||||
try {
|
||||
val longestEdge = max(frame.width, frame.height)
|
||||
val preview =
|
||||
if (longestEdge <= VIDEO_THUMBNAIL_MAX_DIMENSION) {
|
||||
frame
|
||||
} else {
|
||||
val scale = VIDEO_THUMBNAIL_MAX_DIMENSION.toDouble() / longestEdge.toDouble()
|
||||
frame.scale(
|
||||
max(1, (frame.width * scale).roundToInt()),
|
||||
max(1, (frame.height * scale).roundToInt()),
|
||||
true,
|
||||
)
|
||||
}
|
||||
try {
|
||||
val output = ByteArrayOutputStream()
|
||||
if (!preview.compress(Bitmap.CompressFormat.JPEG, VIDEO_THUMBNAIL_QUALITY, output)) return@runCatching null
|
||||
Base64.encodeToString(output.toByteArray(), Base64.NO_WRAP)
|
||||
} finally {
|
||||
if (preview !== frame) preview.recycle()
|
||||
}
|
||||
} finally {
|
||||
frame.recycle()
|
||||
}
|
||||
} finally {
|
||||
retriever.release()
|
||||
}
|
||||
}.getOrNull()
|
||||
|
||||
private fun readBoundedAttachmentBytes(
|
||||
resolver: ContentResolver,
|
||||
uri: Uri,
|
||||
|
||||
@@ -3,6 +3,7 @@ package ai.openclaw.app.ui.chat
|
||||
import ai.openclaw.app.chat.ChatMessageContent
|
||||
import ai.openclaw.app.gateway.GatewayLoadedMedia
|
||||
import ai.openclaw.app.gateway.GatewayMediaKind
|
||||
import ai.openclaw.app.gateway.GatewayPreparingPlaybackInterceptor
|
||||
import ai.openclaw.app.i18n.nativeString
|
||||
import ai.openclaw.app.ui.design.ClawTheme
|
||||
import android.content.Context
|
||||
@@ -63,6 +64,7 @@ import androidx.media3.datasource.DefaultDataSource
|
||||
import androidx.media3.datasource.okhttp.OkHttpDataSource
|
||||
import androidx.media3.exoplayer.ExoPlayer
|
||||
import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
|
||||
import androidx.media3.session.MediaSession
|
||||
import androidx.media3.ui.AspectRatioFrameLayout
|
||||
import androidx.media3.ui.PlayerView
|
||||
import kotlinx.coroutines.CancellationException
|
||||
@@ -110,6 +112,60 @@ internal class ChatMediaPlaybackClaims<T>(
|
||||
}
|
||||
}
|
||||
|
||||
internal class ChatMediaSessionLifecycle<T : Any, S : Any>(
|
||||
private val release: (S) -> Unit,
|
||||
) {
|
||||
private var activeOwner: T? = null
|
||||
private var activeSession: S? = null
|
||||
|
||||
fun activate(
|
||||
owner: T,
|
||||
create: (T) -> S,
|
||||
): S {
|
||||
if (activeOwner === owner) return checkNotNull(activeSession)
|
||||
releaseActive()
|
||||
return create(owner).also { session ->
|
||||
activeOwner = owner
|
||||
activeSession = session
|
||||
}
|
||||
}
|
||||
|
||||
fun release(owner: T): Boolean {
|
||||
if (activeOwner !== owner) return false
|
||||
releaseActive()
|
||||
return true
|
||||
}
|
||||
|
||||
private fun releaseActive() {
|
||||
activeSession?.let(release)
|
||||
activeSession = null
|
||||
activeOwner = null
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
internal fun inlineMediaSessionPlayerCommands(availableCommands: Player.Commands): Player.Commands =
|
||||
availableCommands
|
||||
.buildUpon()
|
||||
.remove(Player.COMMAND_SET_MEDIA_ITEM)
|
||||
.remove(Player.COMMAND_CHANGE_MEDIA_ITEMS)
|
||||
.remove(Player.COMMAND_STOP)
|
||||
.build()
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
private object ChatInlineMediaSessionCallback : MediaSession.Callback {
|
||||
override fun onConnect(
|
||||
session: MediaSession,
|
||||
controller: MediaSession.ControllerInfo,
|
||||
): MediaSession.ConnectionResult {
|
||||
if (!controller.isTrusted) return MediaSession.ConnectionResult.reject()
|
||||
return MediaSession.ConnectionResult
|
||||
.AcceptedResultBuilder(session)
|
||||
.setAvailablePlayerCommands(inlineMediaSessionPlayerCommands(session.player.availableCommands))
|
||||
.build()
|
||||
}
|
||||
}
|
||||
|
||||
private object ChatMediaPlaybackArbiter {
|
||||
private data class AudioFocusHandle(
|
||||
val manager: AudioManager,
|
||||
@@ -125,6 +181,7 @@ private object ChatMediaPlaybackArbiter {
|
||||
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val claims = ChatMediaPlaybackClaims<ActivePlayback>(::pauseAndAbandon, ::stopAndRelease)
|
||||
private val sessions = ChatMediaSessionLifecycle<ExoPlayer, MediaSession>(MediaSession::release)
|
||||
private var playbackIntentGeneration = 0L
|
||||
|
||||
@Synchronized
|
||||
@@ -192,6 +249,17 @@ private object ChatMediaPlaybackArbiter {
|
||||
}
|
||||
val playback = existing ?: ActivePlayback(player = player, onReleased = onReleased).also(claims::claim)
|
||||
playback.audioFocus = AudioFocusHandle(manager = audioManager, request = focusRequest)
|
||||
try {
|
||||
sessions.activate(player) { activePlayer ->
|
||||
MediaSession
|
||||
.Builder(context, activePlayer)
|
||||
.setCallback(ChatInlineMediaSessionCallback)
|
||||
.build()
|
||||
}
|
||||
} catch (_: Throwable) {
|
||||
pauseAndAbandon(playback)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -203,6 +271,7 @@ private object ChatMediaPlaybackArbiter {
|
||||
|
||||
private fun pauseAndAbandon(playback: ActivePlayback) {
|
||||
playback.player.pause()
|
||||
sessions.release(playback.player)
|
||||
playback.audioFocus?.let { focus -> focus.manager.abandonAudioFocusRequest(focus.request) }
|
||||
playback.audioFocus = null
|
||||
}
|
||||
@@ -218,7 +287,7 @@ private object ChatMediaPlaybackArbiter {
|
||||
internal fun ChatAudioPlayerCard(
|
||||
content: ChatMessageContent,
|
||||
playbackBlocked: Boolean,
|
||||
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
loadMedia: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
ChatMediaPlayerCard(
|
||||
content = content,
|
||||
@@ -232,7 +301,7 @@ internal fun ChatAudioPlayerCard(
|
||||
internal fun ChatVideoPlayerCard(
|
||||
content: ChatMessageContent,
|
||||
playbackBlocked: Boolean,
|
||||
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
loadMedia: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
ChatMediaPlayerCard(
|
||||
content = content,
|
||||
@@ -248,7 +317,7 @@ private fun ChatMediaPlayerCard(
|
||||
content: ChatMessageContent,
|
||||
kind: GatewayMediaKind,
|
||||
playbackBlocked: Boolean,
|
||||
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
loadMedia: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
@@ -337,7 +406,7 @@ private fun ChatMediaPlayerCard(
|
||||
scope.launch {
|
||||
val loaded =
|
||||
try {
|
||||
loadMedia(artifactId, kind)
|
||||
loadMedia(artifactId, kind, content.playback == "transcode")
|
||||
} catch (error: CancellationException) {
|
||||
throw error
|
||||
} catch (_: Throwable) {
|
||||
@@ -380,12 +449,14 @@ private fun ChatMediaPlayerCard(
|
||||
}
|
||||
|
||||
override fun onPlaybackStateChanged(playbackState: Int) {
|
||||
if (playbackState == Player.STATE_READY) loading = false
|
||||
if (playbackState == Player.STATE_ENDED) {
|
||||
ChatMediaPlaybackArbiter.pause(created)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onPlayerError(playbackException: PlaybackException) {
|
||||
loading = false
|
||||
if (!ChatMediaPlaybackArbiter.release(created)) {
|
||||
disposeUnclaimedPlayer(created, prepared.tempFile)
|
||||
}
|
||||
@@ -394,7 +465,7 @@ private fun ChatMediaPlayerCard(
|
||||
},
|
||||
)
|
||||
player = created
|
||||
loading = false
|
||||
if (content.playback != "transcode") loading = false
|
||||
if (!registerPrepared(created, prepared.tempFile, intentGeneration)) {
|
||||
disposeUnclaimedPlayer(created, prepared.tempFile)
|
||||
return@launch
|
||||
@@ -448,6 +519,7 @@ private fun ChatMediaPlayerCard(
|
||||
content = content,
|
||||
player = player,
|
||||
loading = loading,
|
||||
preparingPlayback = loading && content.playback == "transcode",
|
||||
isPlaying = isPlaying,
|
||||
playbackBlocked = playbackBlocked,
|
||||
error = error,
|
||||
@@ -457,6 +529,7 @@ private fun ChatMediaPlayerCard(
|
||||
AudioPlayerSurface(
|
||||
content = content,
|
||||
loading = loading,
|
||||
preparingPlayback = loading && content.playback == "transcode",
|
||||
isPlaying = isPlaying,
|
||||
playbackBlocked = playbackBlocked,
|
||||
error = error,
|
||||
@@ -477,6 +550,7 @@ private fun ChatMediaPlayerCard(
|
||||
private fun AudioPlayerSurface(
|
||||
content: ChatMessageContent,
|
||||
loading: Boolean,
|
||||
preparingPlayback: Boolean,
|
||||
isPlaying: Boolean,
|
||||
playbackBlocked: Boolean,
|
||||
error: String?,
|
||||
@@ -525,7 +599,13 @@ private fun AudioPlayerSurface(
|
||||
style = ClawTheme.type.body,
|
||||
color = ClawTheme.colors.text,
|
||||
)
|
||||
val status = error ?: if (playbackBlocked) nativeString("Paused for voice playback") else null
|
||||
val status =
|
||||
when {
|
||||
error != null -> error
|
||||
preparingPlayback -> nativeString("Preparing playback…")
|
||||
playbackBlocked -> nativeString("Paused for voice playback")
|
||||
else -> null
|
||||
}
|
||||
status?.let { Text(it, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted) }
|
||||
}
|
||||
}
|
||||
@@ -549,6 +629,7 @@ private fun VideoPlayerSurface(
|
||||
content: ChatMessageContent,
|
||||
player: ExoPlayer?,
|
||||
loading: Boolean,
|
||||
preparingPlayback: Boolean,
|
||||
isPlaying: Boolean,
|
||||
playbackBlocked: Boolean,
|
||||
error: String?,
|
||||
@@ -602,7 +683,14 @@ private fun VideoPlayerSurface(
|
||||
style = ClawTheme.type.caption,
|
||||
color = ClawTheme.colors.textMuted,
|
||||
)
|
||||
(error ?: if (playbackBlocked) nativeString("Paused for voice playback") else null)?.let {
|
||||
val status =
|
||||
when {
|
||||
error != null -> error
|
||||
preparingPlayback -> nativeString("Preparing playback…")
|
||||
playbackBlocked -> nativeString("Paused for voice playback")
|
||||
else -> null
|
||||
}
|
||||
status?.let {
|
||||
Text(it, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted)
|
||||
}
|
||||
}
|
||||
@@ -627,6 +715,7 @@ private suspend fun prepareMediaSource(
|
||||
headers = loaded.headers,
|
||||
client = loaded.client,
|
||||
tempFile = file,
|
||||
retryPreparingPlayback = false,
|
||||
)
|
||||
}
|
||||
is GatewayLoadedMedia.Streaming ->
|
||||
@@ -636,6 +725,7 @@ private suspend fun prepareMediaSource(
|
||||
headers = loaded.headers,
|
||||
client = loaded.client,
|
||||
tempFile = null,
|
||||
retryPreparingPlayback = loaded.retryPreparingPlayback,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -666,7 +756,16 @@ private fun buildMediaPlayer(
|
||||
context: Context,
|
||||
source: PreparedMediaSource,
|
||||
): ExoPlayer {
|
||||
val httpFactory = OkHttpDataSource.Factory(source.client).setDefaultRequestProperties(source.headers)
|
||||
val client =
|
||||
if (source.retryPreparingPlayback) {
|
||||
source.client
|
||||
.newBuilder()
|
||||
.addInterceptor(GatewayPreparingPlaybackInterceptor())
|
||||
.build()
|
||||
} else {
|
||||
source.client
|
||||
}
|
||||
val httpFactory = OkHttpDataSource.Factory(client).setDefaultRequestProperties(source.headers)
|
||||
val dataSourceFactory = DefaultDataSource.Factory(context, httpFactory)
|
||||
val player =
|
||||
ExoPlayer
|
||||
@@ -698,6 +797,7 @@ private data class PreparedMediaSource(
|
||||
val headers: Map<String, String>,
|
||||
val client: okhttp3.OkHttpClient,
|
||||
val tempFile: File?,
|
||||
val retryPreparingPlayback: Boolean,
|
||||
)
|
||||
|
||||
internal fun ChatMessageContent.isVideoAttachment(): Boolean = type == "video" || mimeType?.startsWith("video/") == true
|
||||
|
||||
@@ -110,7 +110,7 @@ internal fun ChatMessageBubble(
|
||||
imageResolverReady: Boolean = false,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage? = { null },
|
||||
inlineMediaPlaybackBlocked: Boolean = false,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia? = { _, _ -> null },
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia? = { _, _, _ -> null },
|
||||
) {
|
||||
val role = normalizeVisibleChatMessageRole(message.role) ?: return
|
||||
val style = bubbleStyle(role)
|
||||
@@ -242,7 +242,7 @@ private fun ChatMessageBody(
|
||||
imageResolverReady: Boolean,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
inlineMediaPlaybackBlocked: Boolean,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
for (part in content) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import ai.openclaw.app.MainViewModel
|
||||
import ai.openclaw.app.PendingAssistantAutoSend
|
||||
import ai.openclaw.app.R
|
||||
import ai.openclaw.app.SHARED_AUDIO_DOCUMENT_MIME_TYPES
|
||||
import ai.openclaw.app.SHARED_VIDEO_MIME_TYPES
|
||||
import ai.openclaw.app.chat.ChatCommandEntry
|
||||
import ai.openclaw.app.chat.ChatComposerOwner
|
||||
import ai.openclaw.app.chat.ChatMessage
|
||||
@@ -73,6 +74,7 @@ import androidx.compose.animation.core.rememberInfiniteTransition
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.Canvas
|
||||
import androidx.compose.foundation.Image
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.horizontalScroll
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
@@ -116,6 +118,7 @@ import androidx.compose.material.icons.filled.MoreVert
|
||||
import androidx.compose.material.icons.filled.Refresh
|
||||
import androidx.compose.material.icons.filled.Star
|
||||
import androidx.compose.material.icons.filled.StarBorder
|
||||
import androidx.compose.material.icons.filled.Videocam
|
||||
import androidx.compose.material3.DropdownMenu
|
||||
import androidx.compose.material3.DropdownMenuItem
|
||||
import androidx.compose.material3.ExperimentalMaterial3Api
|
||||
@@ -138,12 +141,15 @@ import androidx.compose.runtime.saveable.rememberSaveable
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.ui.geometry.CornerRadius
|
||||
import androidx.compose.ui.geometry.Offset
|
||||
import androidx.compose.ui.geometry.Size
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.SolidColor
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.input.key.onPreInterceptKeyBeforeSoftKeyboard
|
||||
import androidx.compose.ui.layout.ContentScale
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.res.stringResource
|
||||
import androidx.compose.ui.semantics.contentDescription
|
||||
@@ -448,7 +454,7 @@ fun ChatScreen(
|
||||
}
|
||||
}
|
||||
}
|
||||
val pickAudioOrDocument =
|
||||
val pickMediaOrDocument =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.OpenDocument()) { uri ->
|
||||
val lease = filePickerOwnerCheckpoint.consume() ?: return@rememberLauncherForActivityResult
|
||||
if (uri == null) {
|
||||
@@ -469,7 +475,7 @@ fun ChatScreen(
|
||||
) {
|
||||
listOfNotNull(
|
||||
try {
|
||||
loadPickedAudioOrDocumentAttachment(resolver, uri)
|
||||
loadPickedMediaOrDocumentAttachment(resolver, uri)
|
||||
} catch (err: CancellationException) {
|
||||
throw err
|
||||
} catch (_: Throwable) {
|
||||
@@ -798,7 +804,13 @@ fun ChatScreen(
|
||||
if (!viewModel.isCurrentChatComposerOwner(composerOwner)) return@ChatComposer
|
||||
val authorizationId = composerState.beginMediaAcquisition(composerOwner) ?: return@ChatComposer
|
||||
filePickerOwnerCheckpoint.begin(composerOwner, authorizationId)
|
||||
pickAudioOrDocument.launch(SHARED_AUDIO_DOCUMENT_MIME_TYPES)
|
||||
pickMediaOrDocument.launch(SHARED_AUDIO_DOCUMENT_MIME_TYPES)
|
||||
},
|
||||
onPickVideo = {
|
||||
if (!viewModel.isCurrentChatComposerOwner(composerOwner)) return@ChatComposer
|
||||
val authorizationId = composerState.beginMediaAcquisition(composerOwner) ?: return@ChatComposer
|
||||
filePickerOwnerCheckpoint.begin(composerOwner, authorizationId)
|
||||
pickMediaOrDocument.launch(SHARED_VIDEO_MIME_TYPES)
|
||||
},
|
||||
onRemoveAttachment = { id -> composerState.removeAttachments(composerOwner, setOf(id)) },
|
||||
voiceNoteState = voiceNoteState,
|
||||
@@ -1271,7 +1283,7 @@ private fun ChatMessageList(
|
||||
inlineMediaPlaybackBlocked: Boolean,
|
||||
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
val baseTimeline =
|
||||
@@ -1647,7 +1659,7 @@ private fun ChatBubble(
|
||||
inlineWidgetResolverReady: Boolean,
|
||||
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind, Boolean) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
val normalizedRole = role.trim().lowercase(Locale.US)
|
||||
val isUser = normalizedRole == "user"
|
||||
@@ -2038,6 +2050,7 @@ private fun ChatComposer(
|
||||
onOpenModelPicker: () -> Unit,
|
||||
onPickImages: () -> Unit,
|
||||
onPickAudioOrDocument: () -> Unit,
|
||||
onPickVideo: () -> Unit,
|
||||
onRemoveAttachment: (String) -> Unit,
|
||||
voiceNoteState: VoiceNoteRecorderState,
|
||||
voiceNoteElapsedMs: Long,
|
||||
@@ -2156,6 +2169,7 @@ private fun ChatComposer(
|
||||
onValueChange = onValueChange,
|
||||
onPickImages = onPickImages,
|
||||
onPickAudioOrDocument = onPickAudioOrDocument,
|
||||
onPickVideo = onPickVideo,
|
||||
onStartVoiceNote = onStartVoiceNote,
|
||||
recordVoiceNoteEnabled = recordVoiceNoteEnabled,
|
||||
dictationActive = dictationActive,
|
||||
@@ -2600,6 +2614,7 @@ private fun ChatInputPill(
|
||||
onValueChange: (String) -> Unit,
|
||||
onPickImages: () -> Unit,
|
||||
onPickAudioOrDocument: () -> Unit,
|
||||
onPickVideo: () -> Unit,
|
||||
onStartVoiceNote: () -> Unit,
|
||||
recordVoiceNoteEnabled: Boolean,
|
||||
dictationActive: Boolean,
|
||||
@@ -2635,6 +2650,11 @@ private fun ChatInputPill(
|
||||
Icon(imageVector = Icons.Default.AttachFile, contentDescription = nativeString("Attachment"), modifier = Modifier.size(20.dp))
|
||||
}
|
||||
}
|
||||
Surface(onClick = onPickVideo, modifier = Modifier.size(ClawTheme.spacing.touchTarget), shape = CircleShape, color = ClawTheme.colors.surfaceRaised, contentColor = ClawTheme.colors.text) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(imageVector = Icons.Default.Videocam, contentDescription = nativeString("Attach video"), modifier = Modifier.size(20.dp))
|
||||
}
|
||||
}
|
||||
Box(modifier = Modifier.weight(1f)) {
|
||||
ChatTextFieldValueAdapter(
|
||||
value = value,
|
||||
@@ -2761,6 +2781,10 @@ private fun AttachmentChip(
|
||||
attachment: PendingAttachment,
|
||||
onRemove: () -> Unit,
|
||||
) {
|
||||
val videoThumbnail =
|
||||
remember(attachment.videoThumbnailBase64) {
|
||||
attachment.videoThumbnailBase64?.let(::decodeBase64Bitmap)
|
||||
}
|
||||
Surface(
|
||||
shape = RoundedCornerShape(ClawTheme.radii.pill),
|
||||
color = ClawTheme.colors.surfaceRaised,
|
||||
@@ -2774,6 +2798,17 @@ private fun AttachmentChip(
|
||||
) {
|
||||
if (attachment.mimeType.startsWith("audio/")) {
|
||||
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(14.dp), tint = ClawTheme.colors.textMuted)
|
||||
} else if (attachment.mimeType.startsWith("video/")) {
|
||||
if (videoThumbnail != null) {
|
||||
Image(
|
||||
bitmap = videoThumbnail.asImageBitmap(),
|
||||
contentDescription = null,
|
||||
contentScale = ContentScale.Crop,
|
||||
modifier = Modifier.size(28.dp).clip(RoundedCornerShape(5.dp)),
|
||||
)
|
||||
} else {
|
||||
Icon(imageVector = Icons.Default.Videocam, contentDescription = null, modifier = Modifier.size(14.dp), tint = ClawTheme.colors.textMuted)
|
||||
}
|
||||
}
|
||||
Text(
|
||||
text =
|
||||
|
||||
@@ -19,6 +19,7 @@ data class PendingAttachment(
|
||||
val mimeType: String,
|
||||
val base64: String,
|
||||
val durationMs: Long? = null,
|
||||
val videoThumbnailBase64: String? = null,
|
||||
)
|
||||
|
||||
internal data class ChatComposerAttachmentMigration(
|
||||
@@ -205,6 +206,7 @@ internal fun List<SessionEditorAttachment>.toPendingAttachments(): List<PendingA
|
||||
internal fun attachmentTypeForMimeType(mimeType: String): String =
|
||||
when {
|
||||
mimeType.startsWith("audio/") -> "audio"
|
||||
mimeType.startsWith("video/") -> "video"
|
||||
mimeType.startsWith("image/") -> "image"
|
||||
else -> "file"
|
||||
}
|
||||
|
||||
@@ -129,6 +129,24 @@ class ShareLaunchTest {
|
||||
assertTrue(parsed.attachments.all { it.kind == SharedAttachmentKind.Audio })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun acceptsVideoShareFromProviderMimeType() {
|
||||
val video = Uri.parse("content://media/shared/video")
|
||||
val parsed =
|
||||
parseShare(
|
||||
intent =
|
||||
Intent(Intent.ACTION_SEND)
|
||||
.setType("video/*")
|
||||
.putExtra(Intent.EXTRA_STREAM, video),
|
||||
mimeTypes = mapOf(video to "video/mp4"),
|
||||
)
|
||||
|
||||
requireNotNull(parsed)
|
||||
assertEquals(SharedAttachmentKind.Video, parsed.attachments.single().kind)
|
||||
assertEquals("video/mp4", parsed.attachments.single().mimeType)
|
||||
assertTrue("video/*" in SHARED_ATTACHMENT_MIME_ALLOWLIST)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun acceptsCuratedDocumentShare() {
|
||||
val document = Uri.parse("content://docs/shared/report")
|
||||
|
||||
@@ -107,7 +107,7 @@ class ChatControllerOutboxTest {
|
||||
enqueueGate?.await()
|
||||
if (gatewayIds.values.count { it == gatewayId } >= capacity) return ChatOutboxEnqueueResult.QueueFull
|
||||
val commandBytes = attachments.sumOf { it.bytes.size.toLong() }
|
||||
if (commandBytes > OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES) return ChatOutboxEnqueueResult.AttachmentsTooLarge
|
||||
if (!outboxCommandAttachmentsWithinByteLimits(attachments)) return ChatOutboxEnqueueResult.AttachmentsTooLarge
|
||||
val queuedBytes = attachmentBytes.values.sumOf { list -> list.sumOf { it.size.toLong() } }
|
||||
if (commandBytes > 0 && queuedBytes + commandBytes > OUTBOX_MAX_GATEWAY_ATTACHMENT_BYTES) {
|
||||
return ChatOutboxEnqueueResult.StorageFull
|
||||
|
||||
@@ -332,6 +332,26 @@ class ChatMessageContentParsingTest {
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun parsesSupportedPlaybackRenditions() {
|
||||
val direct =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"video","mimeType":"video/mp4","playback":"transcode"}""",
|
||||
)
|
||||
val attachment =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mp4","playback":"native"}}""",
|
||||
)
|
||||
val unsupported =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"video","mimeType":"video/mp4","playback":"future"}""",
|
||||
)
|
||||
|
||||
assertEquals("transcode", parseChatMessageContent(direct)?.playback)
|
||||
assertEquals("native", parseChatMessageContent(attachment)?.playback)
|
||||
assertEquals(null, parseChatMessageContent(unsupported)?.playback)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dropsOversizedInlineImageContentBeforeRendering() {
|
||||
val oversized = "A".repeat(CHAT_IMAGE_MAX_BASE64_CHARS + 1)
|
||||
|
||||
@@ -801,6 +801,59 @@ class RoomChatCommandOutboxTest {
|
||||
assertTrue(store.load("gateway-a").isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun videoCommandsUseServerAttachmentCapWithoutRaisingOtherMediaCaps() =
|
||||
runTest {
|
||||
val aboveDefaultCap = ByteArray((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 1L).toInt())
|
||||
val video =
|
||||
store.enqueue(
|
||||
gatewayId = "gateway-a",
|
||||
sessionKey = "main",
|
||||
text = "video",
|
||||
thinkingLevel = "off",
|
||||
nowMs = 10,
|
||||
ownerAgentId = "main",
|
||||
attachments = listOf(payload(aboveDefaultCap, type = "video", mimeType = "video/mp4")),
|
||||
)
|
||||
val document =
|
||||
store.enqueue(
|
||||
gatewayId = "gateway-a",
|
||||
sessionKey = "main",
|
||||
text = "document",
|
||||
thinkingLevel = "off",
|
||||
nowMs = 11,
|
||||
ownerAgentId = "main",
|
||||
attachments = listOf(payload(aboveDefaultCap, type = "file", mimeType = "application/pdf")),
|
||||
)
|
||||
|
||||
assertTrue(video is ChatOutboxEnqueueResult.Queued)
|
||||
assertEquals(ChatOutboxEnqueueResult.AttachmentsTooLarge, document)
|
||||
assertEquals(20L * 1024L * 1024L, OUTBOX_MAX_VIDEO_COMMAND_ATTACHMENT_BYTES)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun mixedVideoCommandKeepsNonVideoAggregateCap() =
|
||||
runTest {
|
||||
val document = ByteArray(5 * 1024 * 1024)
|
||||
val refused =
|
||||
store.enqueue(
|
||||
gatewayId = "gateway-a",
|
||||
sessionKey = "main",
|
||||
text = "mixed",
|
||||
thinkingLevel = "off",
|
||||
nowMs = 10,
|
||||
ownerAgentId = "main",
|
||||
attachments =
|
||||
listOf(
|
||||
payload(document, fileName = "one.pdf", type = "file", mimeType = "application/pdf"),
|
||||
payload(document, fileName = "two.pdf", type = "file", mimeType = "application/pdf"),
|
||||
payload(byteArrayOf(1), fileName = "clip.mp4", type = "video", mimeType = "video/mp4"),
|
||||
),
|
||||
)
|
||||
|
||||
assertEquals(ChatOutboxEnqueueResult.AttachmentsTooLarge, refused)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun gatewayAttachmentByteBudgetRefusesWhenExhaustedAndRecoversAfterDelete() =
|
||||
runTest {
|
||||
|
||||
@@ -95,6 +95,7 @@ class RoomChatTranscriptCacheTest {
|
||||
fileName = "demo.mp4",
|
||||
artifactId = "artifact_managed_media_44444444-4444-4444-8444-444444444444",
|
||||
durationMs = 5_300,
|
||||
playback = "transcode",
|
||||
width = 1920,
|
||||
height = 1080,
|
||||
)
|
||||
|
||||
@@ -12,6 +12,8 @@ import kotlinx.coroutines.withTimeout
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import okhttp3.WebSocket
|
||||
import okhttp3.WebSocketListener
|
||||
@@ -31,6 +33,7 @@ import org.robolectric.RuntimeEnvironment
|
||||
import org.robolectric.annotation.Config
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
private const val TEST_TIMEOUT_MS = 8_000L
|
||||
@@ -76,6 +79,12 @@ class GatewaySessionCustomHeadersTest {
|
||||
val videoAttachmentId = "22222222-2222-4222-8222-222222222222"
|
||||
val videoArtifactId = "artifact_managed_media_$videoAttachmentId"
|
||||
val videoPath = "/api/chat/media/outgoing/main/$videoAttachmentId/full?mediaTicket=video-ticket"
|
||||
val audioAttachmentId = "33333333-3333-4333-8333-333333333333"
|
||||
val audioArtifactId = "artifact_managed_media_$audioAttachmentId"
|
||||
val audioPath = "/api/chat/media/outgoing/main/$audioAttachmentId/full?mediaTicket=audio-ticket"
|
||||
val audioPlaybackPath = "$audioPath&playback=1"
|
||||
val audioBytes = byteArrayOf(5, 6, 7, 8)
|
||||
val audioRequestCount = AtomicInteger()
|
||||
val server =
|
||||
MockWebServer().apply {
|
||||
dispatcher =
|
||||
@@ -87,6 +96,14 @@ class GatewaySessionCustomHeadersTest {
|
||||
.setHeader("Content-Type", "image/png")
|
||||
.setBody(Buffer().write(imageBytes))
|
||||
}
|
||||
if (request.path == audioPlaybackPath) {
|
||||
if (audioRequestCount.incrementAndGet() == 1) {
|
||||
return MockResponse().setResponseCode(202).setBody("""{"status":"preparing"}""")
|
||||
}
|
||||
return MockResponse()
|
||||
.setHeader("Content-Type", "audio/mp4")
|
||||
.setBody(Buffer().write(audioBytes))
|
||||
}
|
||||
return MockResponse().withWebSocketUpgrade(
|
||||
object : WebSocketListener() {
|
||||
override fun onOpen(
|
||||
@@ -118,6 +135,15 @@ class GatewaySessionCustomHeadersTest {
|
||||
webSocket.send(
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"artifact":{"id":"$videoArtifactId","type":"video","mimeType":"video/mp4","download":{"mode":"url"}},"url":"$videoPath"}}""",
|
||||
)
|
||||
} else if (frame["params"]
|
||||
?.jsonObject
|
||||
?.get("artifactId")
|
||||
?.jsonPrimitive
|
||||
?.content == audioArtifactId
|
||||
) {
|
||||
webSocket.send(
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"artifact":{"id":"$audioArtifactId","type":"audio","mimeType":"audio/mp4","download":{"mode":"url"}},"url":"$audioPath"}}""",
|
||||
)
|
||||
} else {
|
||||
webSocket.send(
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""",
|
||||
@@ -184,6 +210,17 @@ class GatewaySessionCustomHeadersTest {
|
||||
assertEquals("http://127.0.0.1:${server.port}$videoPath", streamed.url)
|
||||
assertEquals("video/*", streamed.headers["Accept"])
|
||||
assertEquals("video/mp4", streamed.mimeType)
|
||||
assertEquals(false, streamed.retryPreparingPlayback)
|
||||
|
||||
val transcodedVideo =
|
||||
session.loadMediaArtifact(stableId, "main", "main", videoArtifactId, GatewayMediaKind.Video, true) as GatewayLoadedMedia.Streaming
|
||||
assertEquals("http://127.0.0.1:${server.port}$videoPath&playback=1", transcodedVideo.url)
|
||||
assertTrue(transcodedVideo.retryPreparingPlayback)
|
||||
|
||||
val audio =
|
||||
session.loadMediaArtifact(stableId, "main", "main", audioArtifactId, GatewayMediaKind.Audio, true) as GatewayLoadedMedia.Buffered
|
||||
assertArrayEquals(audioBytes, audio.bytes)
|
||||
assertEquals(2, audioRequestCount.get())
|
||||
} finally {
|
||||
session.disconnectAndJoin()
|
||||
scope.cancel()
|
||||
@@ -191,6 +228,74 @@ class GatewaySessionCustomHeadersTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun preparingPlaybackInterceptorRetries202WithoutSurfacingLoadError() {
|
||||
val server = MockWebServer()
|
||||
server.enqueue(MockResponse().setResponseCode(202).setBody("""{"status":"preparing"}"""))
|
||||
server.enqueue(MockResponse().setResponseCode(200).setBody("ready"))
|
||||
server.start()
|
||||
var nowMs = 0L
|
||||
val client =
|
||||
OkHttpClient
|
||||
.Builder()
|
||||
.addInterceptor(
|
||||
GatewayPreparingPlaybackInterceptor(
|
||||
policy = GatewayPlaybackRetryPolicy(maxElapsedMs = 100L, initialDelayMs = 0L, maxDelayMs = 0L),
|
||||
nowMs = { nowMs++ },
|
||||
sleepMs = {},
|
||||
),
|
||||
).build()
|
||||
|
||||
try {
|
||||
client.newCall(Request.Builder().url(server.url("/video?playback=1")).build()).execute().use { response ->
|
||||
assertEquals(200, response.code)
|
||||
assertEquals("ready", response.body.string())
|
||||
}
|
||||
assertEquals(2, server.requestCount)
|
||||
} finally {
|
||||
server.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun preparingPlaybackRetryStopsAtTwoMinuteCap() {
|
||||
val retry = GatewayPlaybackRetryState(startedAtMs = 1_000L)
|
||||
|
||||
assertTrue(retry.canAttempt(nowMs = 1_000L))
|
||||
assertEquals(500L, retry.nextDelayMs(nowMs = 1_000L))
|
||||
assertEquals(false, retry.canAttempt(nowMs = 121_000L))
|
||||
assertNull(retry.nextDelayMs(nowMs = 121_001L))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun preparingPlaybackInterceptorDoesNotStartRequestAfterOvershootingDeadline() {
|
||||
val server = MockWebServer()
|
||||
server.enqueue(MockResponse().setResponseCode(202).setBody("""{"status":"preparing"}"""))
|
||||
server.start()
|
||||
var nowMs = 0L
|
||||
val client =
|
||||
OkHttpClient
|
||||
.Builder()
|
||||
.addInterceptor(
|
||||
GatewayPreparingPlaybackInterceptor(
|
||||
policy = GatewayPlaybackRetryPolicy(maxElapsedMs = 2L, initialDelayMs = 1L, maxDelayMs = 1L),
|
||||
nowMs = { nowMs },
|
||||
sleepMs = { delayMs -> nowMs += delayMs + 1L },
|
||||
),
|
||||
).build()
|
||||
|
||||
try {
|
||||
val failure =
|
||||
runCatching {
|
||||
client.newCall(Request.Builder().url(server.url("/video?playback=1")).build()).execute().use { }
|
||||
}.exceptionOrNull()
|
||||
assertTrue(failure is java.io.IOException)
|
||||
assertEquals(1, server.requestCount)
|
||||
} finally {
|
||||
server.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun tlsUpgradeRequest_carriesLatestSanitizedHeadersForOnlyThisGateway() {
|
||||
val app = RuntimeEnvironment.getApplication()
|
||||
|
||||
@@ -716,7 +716,29 @@ class ChatComposerDraftTest {
|
||||
fun attachmentAdmissionUsesPerKindDecodedBudgets() {
|
||||
assertEquals(CHAT_COMPOSER_MAX_IMAGE_DECODED_BYTES, chatComposerAttachmentDecodedByteLimit("image/png"))
|
||||
assertEquals(CHAT_COMPOSER_MAX_AUDIO_DECODED_BYTES, chatComposerAttachmentDecodedByteLimit("audio/mpeg"))
|
||||
assertEquals(CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES, chatComposerAttachmentDecodedByteLimit("video/mp4"))
|
||||
assertEquals(CHAT_COMPOSER_MAX_DOCUMENT_DECODED_BYTES, chatComposerAttachmentDecodedByteLimit("application/pdf"))
|
||||
assertEquals(20L * 1024L * 1024L, CHAT_COMPOSER_MAX_VIDEO_DECODED_BYTES)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun videoPositionDoesNotRelaxNonVideoAdmissionBudget() {
|
||||
val video = PendingAttachment("video", "clip.mp4", "video/mp4", "AAAA")
|
||||
val document = PendingAttachment("document", "report.pdf", "application/pdf", "AAAAAAAA")
|
||||
|
||||
fun admit(candidates: List<PendingAttachment>) =
|
||||
admitChatAttachments(
|
||||
currentAttachments = emptyList(),
|
||||
candidates = candidates,
|
||||
maxAttachmentCount = 8,
|
||||
maxBase64Chars = 100,
|
||||
maxDecodedBytes = 9,
|
||||
maxNonVideoBase64Chars = 100,
|
||||
maxNonVideoDecodedBytes = 3,
|
||||
)
|
||||
|
||||
assertEquals(listOf(video), admit(listOf(video, document)).accepted)
|
||||
assertEquals(listOf(video), admit(listOf(document, video)).accepted)
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -7,6 +7,7 @@ import androidx.compose.ui.test.assertIsDisplayed
|
||||
import androidx.compose.ui.test.junit4.v2.createComposeRule
|
||||
import androidx.compose.ui.test.onAllNodesWithContentDescription
|
||||
import androidx.compose.ui.test.onNodeWithText
|
||||
import androidx.media3.common.Player
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertSame
|
||||
@@ -39,6 +40,10 @@ class ChatMediaPlayerTest {
|
||||
}
|
||||
}
|
||||
|
||||
private class FakeSession {
|
||||
var released = false
|
||||
}
|
||||
|
||||
@get:Rule
|
||||
val composeRule = createComposeRule()
|
||||
|
||||
@@ -75,6 +80,47 @@ class ChatMediaPlayerTest {
|
||||
assertSame(player, claims.active)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun playbackClaimsCreateAndReleaseOnlyOneMediaSession() {
|
||||
val first = FakePlayer()
|
||||
val second = FakePlayer()
|
||||
val sessions = ChatMediaSessionLifecycle<FakePlayer, FakeSession> { it.released = true }
|
||||
val claims =
|
||||
ChatMediaPlaybackClaims<FakePlayer>(
|
||||
pause = { player -> sessions.release(player) },
|
||||
release = { player -> sessions.release(player) },
|
||||
)
|
||||
|
||||
claims.claim(first)
|
||||
val firstSession = sessions.activate(first) { FakeSession() }
|
||||
assertSame(firstSession, sessions.activate(first) { error("duplicate session") })
|
||||
|
||||
claims.claim(second)
|
||||
val secondSession = sessions.activate(second) { FakeSession() }
|
||||
assertTrue(firstSession.released)
|
||||
assertFalse(secondSession.released)
|
||||
|
||||
claims.pauseIf { it === second }
|
||||
assertTrue(secondSession.released)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun mediaSessionControllersCannotReplaceInlineMediaItem() {
|
||||
val commands =
|
||||
inlineMediaSessionPlayerCommands(
|
||||
Player.Commands
|
||||
.Builder()
|
||||
.addAllCommands()
|
||||
.build(),
|
||||
)
|
||||
|
||||
assertTrue(commands.contains(Player.COMMAND_PLAY_PAUSE))
|
||||
assertTrue(commands.contains(Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM))
|
||||
assertFalse(commands.contains(Player.COMMAND_SET_MEDIA_ITEM))
|
||||
assertFalse(commands.contains(Player.COMMAND_CHANGE_MEDIA_ITEMS))
|
||||
assertFalse(commands.contains(Player.COMMAND_STOP))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun legacyMediaPartsRenderLabelsWithoutPlayControlsOrClaims() {
|
||||
val audio =
|
||||
@@ -102,7 +148,7 @@ class ChatMediaPlayerTest {
|
||||
content = listOf(audio, video),
|
||||
timestampMs = 1,
|
||||
),
|
||||
loadMediaArtifact = { _, _ ->
|
||||
loadMediaArtifact = { _, _, _ ->
|
||||
loadCount += 1
|
||||
null
|
||||
},
|
||||
|
||||
@@ -90,6 +90,7 @@ kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serializa
|
||||
material = { module = "com.google.android.material:material", version.ref = "material" }
|
||||
media3-datasource-okhttp = { module = "androidx.media3:media3-datasource-okhttp", version.ref = "media3" }
|
||||
media3-exoplayer = { module = "androidx.media3:media3-exoplayer", version.ref = "media3" }
|
||||
media3-session = { module = "androidx.media3:media3-session", version.ref = "media3" }
|
||||
media3-ui = { module = "androidx.media3:media3-ui", version.ref = "media3" }
|
||||
mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp" }
|
||||
okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
|
||||
|
||||
Reference in New Issue
Block a user