feat(android): add gfm chat markdown renderer

This commit is contained in:
Ayaan Zaidi
2026-02-25 09:01:13 +05:30
committed by Ayaan Zaidi
parent 6969027025
commit ff4dc050cc
2 changed files with 490 additions and 119 deletions

View File

@@ -126,6 +126,11 @@ dependencies {
implementation("androidx.exifinterface:exifinterface:1.4.2") implementation("androidx.exifinterface:exifinterface:1.4.2")
implementation("com.squareup.okhttp3:okhttp:5.3.2") implementation("com.squareup.okhttp3:okhttp:5.3.2")
implementation("org.bouncycastle:bcprov-jdk18on:1.83") implementation("org.bouncycastle:bcprov-jdk18on:1.83")
implementation("org.commonmark:commonmark:0.27.1")
implementation("org.commonmark:commonmark-ext-autolink:0.27.1")
implementation("org.commonmark:commonmark-ext-gfm-strikethrough:0.27.1")
implementation("org.commonmark:commonmark-ext-gfm-tables:0.27.1")
implementation("org.commonmark:commonmark-ext-task-list-items:0.27.1")
// CameraX (for node.invoke camera.* parity) // CameraX (for node.invoke camera.* parity)
implementation("androidx.camera:camera-core:1.5.2") implementation("androidx.camera:camera-core:1.5.2")

View File

@@ -3,10 +3,20 @@ package ai.openclaw.android.ui.chat
import android.graphics.BitmapFactory import android.graphics.BitmapFactory
import android.util.Base64 import android.util.Base64
import androidx.compose.foundation.Image import androidx.compose.foundation.Image
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.IntrinsicSize
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.text.selection.SelectionContainer import androidx.compose.foundation.text.selection.SelectionContainer
import androidx.compose.material3.Text import androidx.compose.material3.Text
import androidx.compose.runtime.Composable import androidx.compose.runtime.Composable
@@ -15,18 +25,23 @@ import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.AnnotatedString import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.text.SpanStyle import androidx.compose.ui.text.SpanStyle
import androidx.compose.ui.text.TextStyle
import androidx.compose.ui.text.buildAnnotatedString import androidx.compose.ui.text.buildAnnotatedString
import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontStyle import androidx.compose.ui.text.font.FontStyle
import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextDecoration
import androidx.compose.ui.text.withStyle import androidx.compose.ui.text.withStyle
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import ai.openclaw.android.ui.mobileAccent
import ai.openclaw.android.ui.mobileCallout import ai.openclaw.android.ui.mobileCallout
import ai.openclaw.android.ui.mobileCaption1 import ai.openclaw.android.ui.mobileCaption1
import ai.openclaw.android.ui.mobileCodeBg import ai.openclaw.android.ui.mobileCodeBg
@@ -34,159 +49,510 @@ import ai.openclaw.android.ui.mobileCodeText
import ai.openclaw.android.ui.mobileTextSecondary import ai.openclaw.android.ui.mobileTextSecondary
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import org.commonmark.Extension
import org.commonmark.ext.autolink.AutolinkExtension
import org.commonmark.ext.gfm.strikethrough.Strikethrough
import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension
import org.commonmark.ext.gfm.tables.TableBlock
import org.commonmark.ext.gfm.tables.TableBody
import org.commonmark.ext.gfm.tables.TableCell
import org.commonmark.ext.gfm.tables.TableHead
import org.commonmark.ext.gfm.tables.TableRow
import org.commonmark.ext.gfm.tables.TablesExtension
import org.commonmark.ext.task.list.items.TaskListItemMarker
import org.commonmark.ext.task.list.items.TaskListItemsExtension
import org.commonmark.node.BlockQuote
import org.commonmark.node.BulletList
import org.commonmark.node.Code
import org.commonmark.node.Document
import org.commonmark.node.Emphasis
import org.commonmark.node.FencedCodeBlock
import org.commonmark.node.Heading
import org.commonmark.node.HardLineBreak
import org.commonmark.node.HtmlBlock
import org.commonmark.node.HtmlInline
import org.commonmark.node.Image as MarkdownImage
import org.commonmark.node.IndentedCodeBlock
import org.commonmark.node.Link
import org.commonmark.node.ListItem
import org.commonmark.node.Node
import org.commonmark.node.OrderedList
import org.commonmark.node.Paragraph
import org.commonmark.node.SoftLineBreak
import org.commonmark.node.StrongEmphasis
import org.commonmark.node.Text as MarkdownTextNode
import org.commonmark.node.ThematicBreak
import org.commonmark.parser.Parser
private const val LIST_INDENT_DP = 14
private val dataImageRegex = Regex("^data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)$")
private val markdownParser: Parser by lazy {
val extensions: List<Extension> =
listOf(
AutolinkExtension.create(),
StrikethroughExtension.create(),
TablesExtension.create(),
TaskListItemsExtension.create(),
)
Parser.builder()
.extensions(extensions)
.build()
}
@Composable @Composable
fun ChatMarkdown(text: String, textColor: Color) { fun ChatMarkdown(text: String, textColor: Color) {
val blocks = remember(text) { splitMarkdown(text) } val document = remember(text) { markdownParser.parse(text) as Document }
val inlineCodeBg = mobileCodeBg val inlineStyles = InlineStyles(inlineCodeBg = mobileCodeBg, inlineCodeColor = mobileCodeText)
val inlineCodeColor = mobileCodeText
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
for (b in blocks) { RenderMarkdownBlocks(
when (b) { start = document.firstChild,
is ChatMarkdownBlock.Text -> { textColor = textColor,
val trimmed = b.text.trimEnd() inlineStyles = inlineStyles,
if (trimmed.isEmpty()) continue listDepth = 0,
)
}
}
@Composable
private fun RenderMarkdownBlocks(
start: Node?,
textColor: Color,
inlineStyles: InlineStyles,
listDepth: Int,
) {
var node = start
while (node != null) {
val current = node
when (current) {
is Paragraph -> {
RenderParagraph(current, textColor = textColor, inlineStyles = inlineStyles)
}
is Heading -> {
val headingText = remember(current) { buildInlineMarkdown(current.firstChild, inlineStyles) }
Text(
text = headingText,
style = headingStyle(current.level),
color = textColor,
)
}
is FencedCodeBlock -> {
SelectionContainer(modifier = Modifier.fillMaxWidth()) {
ChatCodeBlock(code = current.literal.orEmpty(), language = current.info?.trim()?.ifEmpty { null })
}
}
is IndentedCodeBlock -> {
SelectionContainer(modifier = Modifier.fillMaxWidth()) {
ChatCodeBlock(code = current.literal.orEmpty(), language = null)
}
}
is BlockQuote -> {
Row(
modifier = Modifier
.fillMaxWidth()
.height(IntrinsicSize.Min)
.padding(vertical = 2.dp),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.Top,
) {
Box(
modifier = Modifier
.width(2.dp)
.fillMaxHeight()
.background(mobileTextSecondary.copy(alpha = 0.35f)),
)
Column(
modifier = Modifier.weight(1f),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
RenderMarkdownBlocks(
start = current.firstChild,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
}
}
}
is BulletList -> {
RenderBulletList(
list = current,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
}
is OrderedList -> {
RenderOrderedList(
list = current,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
}
is TableBlock -> {
RenderTableBlock(
table = current,
textColor = textColor,
inlineStyles = inlineStyles,
)
}
is ThematicBreak -> {
Box(
modifier = Modifier
.fillMaxWidth()
.height(1.dp)
.background(mobileTextSecondary.copy(alpha = 0.25f)),
)
}
is HtmlBlock -> {
val literal = current.literal.orEmpty().trim()
if (literal.isNotEmpty()) {
Text( Text(
text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor), text = literal,
style = mobileCallout, style = mobileCallout.copy(fontFamily = FontFamily.Monospace),
color = textColor, color = textColor,
) )
} }
is ChatMarkdownBlock.Code -> { }
SelectionContainer(modifier = Modifier.fillMaxWidth()) { }
ChatCodeBlock(code = b.code, language = b.language) node = current.next
} }
} }
is ChatMarkdownBlock.InlineImage -> {
InlineBase64Image(base64 = b.base64, mimeType = b.mimeType) @Composable
private fun RenderParagraph(
paragraph: Paragraph,
textColor: Color,
inlineStyles: InlineStyles,
) {
val standaloneImage = remember(paragraph) { standaloneDataImage(paragraph) }
if (standaloneImage != null) {
InlineBase64Image(base64 = standaloneImage.base64, mimeType = standaloneImage.mimeType)
return
}
val annotated = remember(paragraph) { buildInlineMarkdown(paragraph.firstChild, inlineStyles) }
if (annotated.text.trimEnd().isEmpty()) {
return
}
Text(
text = annotated,
style = mobileCallout,
color = textColor,
)
}
@Composable
private fun RenderBulletList(
list: BulletList,
textColor: Color,
inlineStyles: InlineStyles,
listDepth: Int,
) {
Column(
modifier = Modifier.padding(start = (LIST_INDENT_DP * listDepth).dp),
verticalArrangement = Arrangement.spacedBy(6.dp),
) {
var item = list.firstChild
while (item != null) {
if (item is ListItem) {
RenderListItem(
item = item,
markerText = "",
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
}
item = item.next
}
}
}
@Composable
private fun RenderOrderedList(
list: OrderedList,
textColor: Color,
inlineStyles: InlineStyles,
listDepth: Int,
) {
Column(
modifier = Modifier.padding(start = (LIST_INDENT_DP * listDepth).dp),
verticalArrangement = Arrangement.spacedBy(6.dp),
) {
var index = list.markerStartNumber ?: 1
var item = list.firstChild
while (item != null) {
if (item is ListItem) {
RenderListItem(
item = item,
markerText = "$index.",
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
index += 1
}
item = item.next
}
}
}
@Composable
private fun RenderListItem(
item: ListItem,
markerText: String,
textColor: Color,
inlineStyles: InlineStyles,
listDepth: Int,
) {
var contentStart = item.firstChild
var marker = markerText
val task = contentStart as? TaskListItemMarker
if (task != null) {
marker = if (task.isChecked) "" else ""
contentStart = task.next
}
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.Top,
) {
Text(
text = marker,
style = mobileCallout.copy(fontWeight = FontWeight.SemiBold),
color = textColor,
modifier = Modifier.width(24.dp),
)
Column(
modifier = Modifier.weight(1f),
verticalArrangement = Arrangement.spacedBy(4.dp),
) {
RenderMarkdownBlocks(
start = contentStart,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth + 1,
)
}
}
}
@Composable
private fun RenderTableBlock(
table: TableBlock,
textColor: Color,
inlineStyles: InlineStyles,
) {
val rows = remember(table) { buildTableRows(table, inlineStyles) }
if (rows.isEmpty()) return
val maxCols = rows.maxOf { row -> row.cells.size }.coerceAtLeast(1)
val scrollState = rememberScrollState()
Column(
modifier = Modifier
.fillMaxWidth()
.horizontalScroll(scrollState)
.border(1.dp, mobileTextSecondary.copy(alpha = 0.25f)),
) {
for (row in rows) {
Row(
modifier = Modifier.fillMaxWidth(),
) {
for (index in 0 until maxCols) {
val cell = row.cells.getOrNull(index) ?: AnnotatedString("")
Text(
text = cell,
style = if (row.isHeader) mobileCaption1.copy(fontWeight = FontWeight.SemiBold) else mobileCallout,
color = textColor,
modifier = Modifier
.border(1.dp, mobileTextSecondary.copy(alpha = 0.22f))
.padding(horizontal = 8.dp, vertical = 6.dp)
.width(160.dp),
)
} }
} }
} }
} }
} }
private sealed interface ChatMarkdownBlock { private fun buildTableRows(table: TableBlock, inlineStyles: InlineStyles): List<TableRenderRow> {
data class Text(val text: String) : ChatMarkdownBlock val rows = mutableListOf<TableRenderRow>()
data class Code(val code: String, val language: String?) : ChatMarkdownBlock var child = table.firstChild
data class InlineImage(val mimeType: String?, val base64: String) : ChatMarkdownBlock while (child != null) {
} when (child) {
is TableHead -> rows.addAll(readTableSection(child, isHeader = true, inlineStyles = inlineStyles))
private fun splitMarkdown(raw: String): List<ChatMarkdownBlock> { is TableBody -> rows.addAll(readTableSection(child, isHeader = false, inlineStyles = inlineStyles))
if (raw.isEmpty()) return emptyList() is TableRow -> rows.add(readTableRow(child, isHeader = false, inlineStyles = inlineStyles))
val out = ArrayList<ChatMarkdownBlock>()
var idx = 0
while (idx < raw.length) {
val fenceStart = raw.indexOf("```", startIndex = idx)
if (fenceStart < 0) {
out.addAll(splitInlineImages(raw.substring(idx)))
break
} }
child = child.next
if (fenceStart > idx) {
out.addAll(splitInlineImages(raw.substring(idx, fenceStart)))
}
val langLineStart = fenceStart + 3
val langLineEnd = raw.indexOf('\n', startIndex = langLineStart).let { if (it < 0) raw.length else it }
val language = raw.substring(langLineStart, langLineEnd).trim().ifEmpty { null }
val codeStart = if (langLineEnd < raw.length && raw[langLineEnd] == '\n') langLineEnd + 1 else langLineEnd
val fenceEnd = raw.indexOf("```", startIndex = codeStart)
if (fenceEnd < 0) {
out.addAll(splitInlineImages(raw.substring(fenceStart)))
break
}
val code = raw.substring(codeStart, fenceEnd)
out.add(ChatMarkdownBlock.Code(code = code, language = language))
idx = fenceEnd + 3
} }
return rows
return out
} }
private fun splitInlineImages(text: String): List<ChatMarkdownBlock> { private fun readTableSection(section: Node, isHeader: Boolean, inlineStyles: InlineStyles): List<TableRenderRow> {
if (text.isEmpty()) return emptyList() val rows = mutableListOf<TableRenderRow>()
val regex = Regex("data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)") var row = section.firstChild
val out = ArrayList<ChatMarkdownBlock>() while (row != null) {
if (row is TableRow) {
var idx = 0 rows.add(readTableRow(row, isHeader = isHeader, inlineStyles = inlineStyles))
while (idx < text.length) {
val m = regex.find(text, startIndex = idx) ?: break
val start = m.range.first
val end = m.range.last + 1
if (start > idx) out.add(ChatMarkdownBlock.Text(text.substring(idx, start)))
val mime = "image/" + (m.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png")
val b64 = m.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty()
if (b64.isNotEmpty()) {
out.add(ChatMarkdownBlock.InlineImage(mimeType = mime, base64 = b64))
} }
idx = end row = row.next
} }
return rows
if (idx < text.length) out.add(ChatMarkdownBlock.Text(text.substring(idx)))
return out
} }
private fun parseInlineMarkdown( private fun readTableRow(row: TableRow, isHeader: Boolean, inlineStyles: InlineStyles): TableRenderRow {
text: String, val cells = mutableListOf<AnnotatedString>()
inlineCodeBg: androidx.compose.ui.graphics.Color, var cellNode = row.firstChild
inlineCodeColor: androidx.compose.ui.graphics.Color, while (cellNode != null) {
): AnnotatedString { if (cellNode is TableCell) {
if (text.isEmpty()) return AnnotatedString("") cells.add(buildInlineMarkdown(cellNode.firstChild, inlineStyles))
}
cellNode = cellNode.next
}
return TableRenderRow(isHeader = isHeader, cells = cells)
}
val out = buildAnnotatedString { private fun buildInlineMarkdown(start: Node?, inlineStyles: InlineStyles): AnnotatedString {
var i = 0 return buildAnnotatedString {
while (i < text.length) { appendInlineNode(
if (text.startsWith("**", startIndex = i)) { node = start,
val end = text.indexOf("**", startIndex = i + 2) inlineCodeBg = inlineStyles.inlineCodeBg,
if (end > i + 2) { inlineCodeColor = inlineStyles.inlineCodeColor,
withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) { )
append(text.substring(i + 2, end)) }
} }
i = end + 2
continue private fun AnnotatedString.Builder.appendInlineNode(
node: Node?,
inlineCodeBg: Color,
inlineCodeColor: Color,
) {
var current = node
while (current != null) {
when (current) {
is MarkdownTextNode -> append(current.literal)
is SoftLineBreak -> append('\n')
is HardLineBreak -> append('\n')
is Code -> {
withStyle(
SpanStyle(
fontFamily = FontFamily.Monospace,
background = inlineCodeBg,
color = inlineCodeColor,
),
) {
append(current.literal)
} }
} }
is Emphasis -> {
if (text[i] == '`') { withStyle(SpanStyle(fontStyle = FontStyle.Italic)) {
val end = text.indexOf('`', startIndex = i + 1) appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
if (end > i + 1) {
withStyle(
SpanStyle(
fontFamily = FontFamily.Monospace,
background = inlineCodeBg,
color = inlineCodeColor,
),
) {
append(text.substring(i + 1, end))
}
i = end + 1
continue
} }
} }
is StrongEmphasis -> {
if (text[i] == '*' && (i + 1 < text.length && text[i + 1] != '*')) { withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) {
val end = text.indexOf('*', startIndex = i + 1) appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
if (end > i + 1) {
withStyle(SpanStyle(fontStyle = FontStyle.Italic)) {
append(text.substring(i + 1, end))
}
i = end + 1
continue
} }
} }
is Strikethrough -> {
append(text[i]) withStyle(SpanStyle(textDecoration = TextDecoration.LineThrough)) {
i += 1 appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
}
}
is Link -> {
withStyle(
SpanStyle(
color = mobileAccent,
textDecoration = TextDecoration.Underline,
),
) {
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
}
}
is MarkdownImage -> {
val alt = buildPlainText(current.firstChild)
if (alt.isNotBlank()) {
append(alt)
} else {
append("image")
}
}
is HtmlInline -> {
if (!current.literal.isNullOrBlank()) {
append(current.literal)
}
}
else -> {
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
}
} }
current = current.next
} }
return out
} }
private fun buildPlainText(start: Node?): String {
val sb = StringBuilder()
var node = start
while (node != null) {
when (node) {
is MarkdownTextNode -> sb.append(node.literal)
is SoftLineBreak, is HardLineBreak -> sb.append('\n')
else -> sb.append(buildPlainText(node.firstChild))
}
node = node.next
}
return sb.toString()
}
private fun standaloneDataImage(paragraph: Paragraph): ParsedDataImage? {
val only = paragraph.firstChild as? MarkdownImage ?: return null
if (only.next != null) return null
return parseDataImageDestination(only.destination)
}
private fun parseDataImageDestination(destination: String?): ParsedDataImage? {
val raw = destination?.trim().orEmpty()
if (raw.isEmpty()) return null
val match = dataImageRegex.matchEntire(raw) ?: return null
val subtype = match.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png"
val base64 = match.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty()
if (base64.isEmpty()) return null
return ParsedDataImage(mimeType = "image/$subtype", base64 = base64)
}
private fun headingStyle(level: Int): TextStyle {
return when (level.coerceIn(1, 6)) {
1 -> mobileCallout.copy(fontSize = 22.sp, lineHeight = 28.sp, fontWeight = FontWeight.Bold)
2 -> mobileCallout.copy(fontSize = 20.sp, lineHeight = 26.sp, fontWeight = FontWeight.Bold)
3 -> mobileCallout.copy(fontSize = 18.sp, lineHeight = 24.sp, fontWeight = FontWeight.SemiBold)
4 -> mobileCallout.copy(fontSize = 16.sp, lineHeight = 22.sp, fontWeight = FontWeight.SemiBold)
else -> mobileCallout.copy(fontWeight = FontWeight.SemiBold)
}
}
private data class InlineStyles(
val inlineCodeBg: Color,
val inlineCodeColor: Color,
)
private data class TableRenderRow(
val isHeader: Boolean,
val cells: List<AnnotatedString>,
)
private data class ParsedDataImage(
val mimeType: String,
val base64: String,
)
@Composable @Composable
private fun InlineBase64Image(base64: String, mimeType: String?) { private fun InlineBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) } var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }