mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-08 22:43:58 +00:00
feat(android): render LaTeX display math in chat via bundled KaTeX (#101435)
* feat(android): render LaTeX display math in chat via bundled KaTeX
* fix(android): preserve fallback for invalid math
* docs(changelog): note Android display math
* Revert "docs(changelog): note Android display math"
This reverts commit 40356286a3.
This commit is contained in:
committed by
GitHub
parent
454fccd784
commit
16348ddec3
@@ -5851,7 +5851,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 315,
|
||||
"line": 326,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMarkdown.kt",
|
||||
"source": "$index.",
|
||||
"surface": "android",
|
||||
@@ -5859,7 +5859,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 718,
|
||||
"line": 729,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMarkdown.kt",
|
||||
"source": "Image unavailable",
|
||||
"surface": "android",
|
||||
|
||||
110
apps/android/THIRD_PARTY_LICENSES/openclaw/licenses/KaTeX.txt
Normal file
110
apps/android/THIRD_PARTY_LICENSES/openclaw/licenses/KaTeX.txt
Normal file
@@ -0,0 +1,110 @@
|
||||
KaTeX is licensed under the MIT License. KaTeX bundles fonts under the SIL Open Font License.
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2013-2020 Khan Academy and other contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
-----------------------------------------------------------
|
||||
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
||||
-----------------------------------------------------------
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting -- in part or in whole -- any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created using
|
||||
the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
37
apps/android/app/src/main/assets/katex/README.md
Normal file
37
apps/android/app/src/main/assets/katex/README.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# KaTeX Android assets
|
||||
|
||||
- Version: KaTeX 0.17.0 (`v0.17.0`, released 2026-05-22)
|
||||
- Source: https://github.com/KaTeX/KaTeX/releases/tag/v0.17.0
|
||||
- Release archive: https://github.com/KaTeX/KaTeX/releases/download/v0.17.0/katex.tar.gz
|
||||
- Published and verified archive SHA-256: `f77cf2555a40e879a4784e43e229d41867f106125bc05b15161de3e761b64b88`
|
||||
|
||||
Only the minified runtime, its stylesheet, and the WOFF2 fonts referenced by that stylesheet are copied from the release archive. `index.html` and `renderer.js` are OpenClaw's local rendering shell. No auto-render extension, mhchem extension, source map, WOFF, or TTF file is bundled.
|
||||
|
||||
## SHA-256
|
||||
|
||||
```text
|
||||
0cdd387c9590a1a9f9794560022dbb59654a7d86f187aa0c81495ad42d3a7308 fonts/KaTeX_AMS-Regular.woff2
|
||||
de7701e42cf1f4cf0b766c03fb27977207eee2f4fd5d76fa82188406da43ea4c fonts/KaTeX_Caligraphic-Bold.woff2
|
||||
5d53e70ad607c2352162dec9e0923fb54ecdafaccbf604cd8dcf7d00facb989b fonts/KaTeX_Caligraphic-Regular.woff2
|
||||
74444efd593c005e3f4573b44524704c0af0a937fe911cca9e94068d0d140d3f fonts/KaTeX_Fraktur-Bold.woff2
|
||||
51814d270d06ff0255dba0799994fa4d8c84d11f09951d47595f4abb1f3602dc fonts/KaTeX_Fraktur-Regular.woff2
|
||||
0f60d1b897938ec918c8ce073092411baf9438f6739465693ff18b0f9d20b021 fonts/KaTeX_Main-Bold.woff2
|
||||
99cd42a3c072d918f2f44984a807cf7aa16e13545fd0875fc07c6c65f99e715b fonts/KaTeX_Main-BoldItalic.woff2
|
||||
97479ca6cce906abc961ecac96faa5f9ca2e61b8e7670d475826bcdee9a7c267 fonts/KaTeX_Main-Italic.woff2
|
||||
c2342cd8b869e01752a9321dc17213fc40d4d04c79688c1d43f2cf316abd7866 fonts/KaTeX_Main-Regular.woff2
|
||||
dc47344dbb6cb5b655c8460d561f4df5f501b90c804ad3c6cec65fe322351ab1 fonts/KaTeX_Math-BoldItalic.woff2
|
||||
7af58c5ec8f132a2ddde9027c6d7814decce4d3b822a11192a42a20e2e973264 fonts/KaTeX_Math-Italic.woff2
|
||||
e99ae51144bf1232efcc1bfe5add36262c6866b0faab24fa75740e1b98577a62 fonts/KaTeX_SansSerif-Bold.woff2
|
||||
00b26ac825e2095056396e0553b8ac26d3f8ad158c3826e28b4c45b385c4714a fonts/KaTeX_SansSerif-Italic.woff2
|
||||
68e8c73ef42afd3ccec58bf0fba302cce448938e7fc020a5e31f8a952eee1342 fonts/KaTeX_SansSerif-Regular.woff2
|
||||
036d4e95149b69ff9bcc0cd55771efeb25ffa3947293e69acd78d5ac328c684b fonts/KaTeX_Script-Regular.woff2
|
||||
6b47c40166b6dbe21a5dfca7718413f2147fd2399be1ba605d8ad39cedf25dfe fonts/KaTeX_Size1-Regular.woff2
|
||||
d04c54219f9eaec6d4d4fd42dfb28785975a4794d6b2fc71e566b9cd6db842dd fonts/KaTeX_Size2-Regular.woff2
|
||||
73d591271b1604960cb10bb90fee021670af7297017e0e98480b332d11f51995 fonts/KaTeX_Size3-Regular.woff2
|
||||
a4af7d414440a1c1790825cfb700cf9cf43b0f2c4b04f0ebc523011ad9853ec0 fonts/KaTeX_Size4-Regular.woff2
|
||||
71d517d67827787cfabdf186914cc3358eda539e37931941f2b2fd4a21f68c0b fonts/KaTeX_Typewriter-Regular.woff2
|
||||
e73bea6d899a3f89595d37f5713cb5ddabc4084fc428a5931782628ee14799e2 index.html
|
||||
a34ad8fc188e8f5a3af7ceaa2a58d7210c6c9171335a15bff2b48ebcd6a6f5b0 katex.min.css
|
||||
45fbe318fea878fdc0a111913dc1f87894b2c439360d0228c086ef313f213efc katex.min.js
|
||||
3cefe6fba565e189311b29265eb842484a637e1a80b0b5e766ab05bfc9ad452f renderer.js
|
||||
```
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
20
apps/android/app/src/main/assets/katex/index.html
Normal file
20
apps/android/app/src/main/assets/katex/index.html
Normal file
@@ -0,0 +1,20 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta
|
||||
http-equiv="Content-Security-Policy"
|
||||
content="default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'"
|
||||
>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
|
||||
<link rel="stylesheet" href="katex.min.css">
|
||||
<style>
|
||||
html, body { margin: 0; padding: 0; overflow: hidden; background: transparent; }
|
||||
#math { box-sizing: border-box; display: inline-block; min-width: 100%; width: max-content; padding: 2px 1px; }
|
||||
.katex-display { margin: 0; }
|
||||
</style>
|
||||
<script defer src="katex.min.js"></script>
|
||||
<script defer src="renderer.js"></script>
|
||||
</head>
|
||||
<body><div id="math"></div></body>
|
||||
</html>
|
||||
1
apps/android/app/src/main/assets/katex/katex.min.css
vendored
Normal file
1
apps/android/app/src/main/assets/katex/katex.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
1
apps/android/app/src/main/assets/katex/katex.min.js
vendored
Normal file
1
apps/android/app/src/main/assets/katex/katex.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
30
apps/android/app/src/main/assets/katex/renderer.js
Normal file
30
apps/android/app/src/main/assets/katex/renderer.js
Normal file
@@ -0,0 +1,30 @@
|
||||
"use strict";
|
||||
|
||||
window.renderMath = async (job) => {
|
||||
const container = document.getElementById("math");
|
||||
try {
|
||||
document.body.style.width = `${job.widthCssPx}px`;
|
||||
container.style.color = job.color;
|
||||
container.style.fontSize = `${job.fontSizeCssPx}px`;
|
||||
container.replaceChildren();
|
||||
katex.render(job.latex, container, {
|
||||
displayMode: true,
|
||||
maxExpand: 1000,
|
||||
maxSize: 10,
|
||||
strict: "ignore",
|
||||
throwOnError: true,
|
||||
trust: false,
|
||||
});
|
||||
await document.fonts.ready;
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
const initialBounds = container.getBoundingClientRect();
|
||||
const width = Math.ceil(Math.max(initialBounds.width, container.scrollWidth));
|
||||
document.body.style.width = `${width}px`;
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
const finalBounds = container.getBoundingClientRect();
|
||||
const height = Math.ceil(Math.max(finalBounds.height, container.scrollHeight));
|
||||
ChatMathBridge.onRenderComplete(job.id, width, height, true);
|
||||
} catch (_) {
|
||||
ChatMathBridge.onRenderComplete(job.id, 0, 0, false);
|
||||
}
|
||||
};
|
||||
@@ -74,6 +74,7 @@ import org.commonmark.node.Paragraph
|
||||
import org.commonmark.node.SoftLineBreak
|
||||
import org.commonmark.node.StrongEmphasis
|
||||
import org.commonmark.node.ThematicBreak
|
||||
import org.commonmark.parser.IncludeSourceSpans
|
||||
import org.commonmark.parser.Parser
|
||||
import java.net.URI
|
||||
import java.util.Locale
|
||||
@@ -95,6 +96,7 @@ private val markdownParser: Parser by lazy {
|
||||
Parser
|
||||
.builder()
|
||||
.extensions(extensions)
|
||||
.includeSourceSpans(IncludeSourceSpans.BLOCKS_AND_INLINES)
|
||||
.build()
|
||||
}
|
||||
|
||||
@@ -105,18 +107,27 @@ fun ChatMarkdown(
|
||||
textColor: Color,
|
||||
isStreaming: Boolean = false,
|
||||
) {
|
||||
val document = remember(text) { parseChatMarkdown(text) }
|
||||
val blocks = remember(text, isStreaming) { segmentChatMarkdown(text, isStreaming) }
|
||||
val inlineStyles =
|
||||
InlineStyles(inlineCodeBg = mobileCodeBg, inlineCodeColor = mobileCodeText, linkColor = mobileAccent, baseCallout = mobileCallout)
|
||||
|
||||
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
|
||||
RenderMarkdownBlocks(
|
||||
start = document.firstChild,
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = 0,
|
||||
isStreaming = isStreaming,
|
||||
)
|
||||
for (block in blocks) {
|
||||
when (block) {
|
||||
is ChatMarkdownSourceBlock.Markdown -> {
|
||||
val document = remember(block.source) { parseChatMarkdown(block.source) }
|
||||
RenderMarkdownBlocks(
|
||||
start = document.firstChild,
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = 0,
|
||||
isStreaming = isStreaming,
|
||||
)
|
||||
}
|
||||
is ChatMarkdownSourceBlock.Math -> ChatMathBlock(latex = block.latex, textColor = textColor)
|
||||
is ChatMarkdownSourceBlock.MathFallback -> ChatMathFallback(latex = block.latex)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,670 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.app.Activity
|
||||
import android.app.Application
|
||||
import android.content.Context
|
||||
import android.content.ContextWrapper
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.LruCache
|
||||
import android.view.View
|
||||
import android.view.ViewGroup
|
||||
import android.webkit.JavascriptInterface
|
||||
import android.webkit.RenderProcessGoneDetail
|
||||
import android.webkit.WebResourceRequest
|
||||
import android.webkit.WebResourceResponse
|
||||
import android.webkit.WebSettings
|
||||
import android.webkit.WebView
|
||||
import android.webkit.WebViewClient
|
||||
import androidx.compose.foundation.Image
|
||||
import androidx.compose.foundation.horizontalScroll
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.BoxWithConstraints
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.graphics.luminance
|
||||
import androidx.compose.ui.graphics.toArgb
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.platform.LocalDensity
|
||||
import androidx.compose.ui.unit.sp
|
||||
import androidx.core.graphics.createBitmap
|
||||
import androidx.core.net.toUri
|
||||
import org.json.JSONObject
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.lang.ref.WeakReference
|
||||
import java.util.Locale
|
||||
import android.graphics.Color as AndroidColor
|
||||
|
||||
private const val MATH_BITMAP_CACHE_BYTES = 4 * 1024 * 1024
|
||||
private const val MATH_NEGATIVE_CACHE_ENTRIES = 64
|
||||
private const val MATH_RENDER_TIMEOUT_MS = 3000L
|
||||
private const val MATH_WIDTH_BUCKET_PX = 64
|
||||
private const val MATH_MAX_BITMAP_DIMENSION = 8192
|
||||
private const val MATH_MAX_BITMAP_PIXELS = MATH_BITMAP_CACHE_BYTES / 4
|
||||
private const val KATEX_ASSET_ROOT = "file:///android_asset/katex/"
|
||||
private const val KATEX_SHELL_URL = "${KATEX_ASSET_ROOT}index.html"
|
||||
|
||||
internal data class ChatMathRenderKey(
|
||||
val latex: String,
|
||||
val widthBucket: Int,
|
||||
val darkMode: Boolean,
|
||||
)
|
||||
|
||||
internal data class ChatMathRenderRequest(
|
||||
val key: ChatMathRenderKey,
|
||||
val textColor: Int,
|
||||
val fontSizePx: Float,
|
||||
val density: Float,
|
||||
) {
|
||||
companion object {
|
||||
fun create(
|
||||
latex: String,
|
||||
widthPx: Int,
|
||||
darkMode: Boolean,
|
||||
textColor: Int,
|
||||
fontSizePx: Float,
|
||||
density: Float,
|
||||
): ChatMathRenderRequest {
|
||||
val boundedWidth = widthPx.coerceAtLeast(1)
|
||||
val widthBucket = ((boundedWidth / MATH_WIDTH_BUCKET_PX) * MATH_WIDTH_BUCKET_PX).coerceAtLeast(MATH_WIDTH_BUCKET_PX)
|
||||
return ChatMathRenderRequest(
|
||||
key = ChatMathRenderKey(latex = latex, widthBucket = widthBucket, darkMode = darkMode),
|
||||
textColor = textColor,
|
||||
fontSizePx = fontSizePx,
|
||||
density = density,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed interface ChatMathRenderResult<out T> {
|
||||
data class Success<T>(
|
||||
val value: T,
|
||||
) : ChatMathRenderResult<T>
|
||||
|
||||
data object Failure : ChatMathRenderResult<Nothing>
|
||||
|
||||
data object TransientFailure : ChatMathRenderResult<Nothing>
|
||||
}
|
||||
|
||||
internal sealed interface ChatMathCacheEntry<out T> {
|
||||
data class Success<T>(
|
||||
val value: T,
|
||||
) : ChatMathCacheEntry<T>
|
||||
|
||||
data object Failure : ChatMathCacheEntry<Nothing>
|
||||
|
||||
data object Missing : ChatMathCacheEntry<Nothing>
|
||||
}
|
||||
|
||||
internal interface ChatMathRenderCache<T> {
|
||||
fun get(request: ChatMathRenderRequest): ChatMathCacheEntry<T>
|
||||
|
||||
fun put(
|
||||
request: ChatMathRenderRequest,
|
||||
result: ChatMathRenderResult<T>,
|
||||
)
|
||||
}
|
||||
|
||||
internal fun interface ChatMathRenderBackend<T> {
|
||||
fun render(
|
||||
request: ChatMathRenderRequest,
|
||||
completion: (ChatMathRenderResult<T>) -> Unit,
|
||||
)
|
||||
}
|
||||
|
||||
internal fun interface ChatMathTimeout {
|
||||
fun cancel()
|
||||
}
|
||||
|
||||
internal fun interface ChatMathTimeoutScheduler {
|
||||
fun schedule(
|
||||
delayMs: Long,
|
||||
action: () -> Unit,
|
||||
): ChatMathTimeout
|
||||
}
|
||||
|
||||
internal fun interface ChatMathRenderSubscription {
|
||||
fun cancel()
|
||||
}
|
||||
|
||||
/** Serializes one renderer backend, deduplicates in-flight keys, and owns timeout fallback. */
|
||||
internal class ChatMathRenderCoordinator<T>(
|
||||
private val backend: ChatMathRenderBackend<T>,
|
||||
private val cache: ChatMathRenderCache<T>,
|
||||
private val timeoutScheduler: ChatMathTimeoutScheduler,
|
||||
) {
|
||||
private val queued = LinkedHashMap<ChatMathRenderRequest, PendingRender<T>>()
|
||||
private var active: PendingRender<T>? = null
|
||||
private var callbackId = 0L
|
||||
private var renderAttemptId = 0L
|
||||
private var activeAttemptId = 0L
|
||||
private var timeout: ChatMathTimeout? = null
|
||||
|
||||
fun render(
|
||||
request: ChatMathRenderRequest,
|
||||
completion: (ChatMathRenderResult<T>) -> Unit,
|
||||
): ChatMathRenderSubscription {
|
||||
when (val cached = cache.get(request)) {
|
||||
is ChatMathCacheEntry.Success -> {
|
||||
completion(ChatMathRenderResult.Success(cached.value))
|
||||
return ChatMathRenderSubscription {}
|
||||
}
|
||||
ChatMathCacheEntry.Failure -> {
|
||||
completion(ChatMathRenderResult.Failure)
|
||||
return ChatMathRenderSubscription {}
|
||||
}
|
||||
ChatMathCacheEntry.Missing -> Unit
|
||||
}
|
||||
|
||||
callbackId += 1
|
||||
val id = callbackId
|
||||
val pending = active?.takeIf { item -> item.request == request } ?: queued[request]
|
||||
if (pending != null) {
|
||||
pending.callbacks[id] = completion
|
||||
} else {
|
||||
queued[request] = PendingRender(request = request, callbacks = linkedMapOf(id to completion))
|
||||
pump()
|
||||
}
|
||||
return ChatMathRenderSubscription {
|
||||
active?.callbacks?.remove(id)
|
||||
val iterator = queued.iterator()
|
||||
while (iterator.hasNext()) {
|
||||
val item = iterator.next().value
|
||||
item.callbacks.remove(id)
|
||||
if (item.callbacks.isEmpty()) iterator.remove()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun pump() {
|
||||
if (active != null) return
|
||||
val next = queued.entries.firstOrNull() ?: return
|
||||
queued.remove(next.key)
|
||||
active = next.value
|
||||
renderAttemptId += 1
|
||||
activeAttemptId = renderAttemptId
|
||||
val attemptId = activeAttemptId
|
||||
timeout =
|
||||
timeoutScheduler.schedule(MATH_RENDER_TIMEOUT_MS) {
|
||||
finish(attemptId, ChatMathRenderResult.TransientFailure)
|
||||
}
|
||||
backend.render(next.value.request) { result -> finish(attemptId, result) }
|
||||
}
|
||||
|
||||
private fun finish(
|
||||
attemptId: Long,
|
||||
result: ChatMathRenderResult<T>,
|
||||
) {
|
||||
if (attemptId != activeAttemptId) return
|
||||
val current = active ?: return
|
||||
timeout?.cancel()
|
||||
timeout = null
|
||||
active = null
|
||||
cache.put(current.request, result)
|
||||
current.callbacks.values
|
||||
.toList()
|
||||
.forEach { callback -> callback(result) }
|
||||
pump()
|
||||
}
|
||||
|
||||
private data class PendingRender<T>(
|
||||
val request: ChatMathRenderRequest,
|
||||
val callbacks: MutableMap<Long, (ChatMathRenderResult<T>) -> Unit>,
|
||||
)
|
||||
}
|
||||
|
||||
private class ChatMathBitmapCache : ChatMathRenderCache<Bitmap> {
|
||||
private val bitmaps =
|
||||
object : LruCache<ChatMathRenderKey, CachedBitmap>(MATH_BITMAP_CACHE_BYTES) {
|
||||
override fun sizeOf(
|
||||
key: ChatMathRenderKey,
|
||||
value: CachedBitmap,
|
||||
): Int = value.bitmap.byteCount.coerceAtLeast(1)
|
||||
}
|
||||
private val failures = LinkedHashMap<ChatMathRenderKey, RenderStyle>(MATH_NEGATIVE_CACHE_ENTRIES, 0.75f, true)
|
||||
|
||||
override fun get(request: ChatMathRenderRequest): ChatMathCacheEntry<Bitmap> {
|
||||
val style = request.renderStyle
|
||||
bitmaps.get(request.key)?.let { cached ->
|
||||
if (cached.style == style) return ChatMathCacheEntry.Success(cached.bitmap)
|
||||
bitmaps.remove(request.key)
|
||||
}
|
||||
failures[request.key]?.let { failedStyle ->
|
||||
if (failedStyle == style) return ChatMathCacheEntry.Failure
|
||||
failures.remove(request.key)
|
||||
}
|
||||
return ChatMathCacheEntry.Missing
|
||||
}
|
||||
|
||||
override fun put(
|
||||
request: ChatMathRenderRequest,
|
||||
result: ChatMathRenderResult<Bitmap>,
|
||||
) {
|
||||
when (result) {
|
||||
is ChatMathRenderResult.Success -> {
|
||||
failures.remove(request.key)
|
||||
bitmaps.put(request.key, CachedBitmap(request.renderStyle, result.value))
|
||||
}
|
||||
ChatMathRenderResult.Failure -> {
|
||||
failures[request.key] = request.renderStyle
|
||||
while (failures.size > MATH_NEGATIVE_CACHE_ENTRIES) {
|
||||
failures.remove(failures.entries.first().key)
|
||||
}
|
||||
}
|
||||
ChatMathRenderResult.TransientFailure -> Unit
|
||||
}
|
||||
}
|
||||
|
||||
private data class CachedBitmap(
|
||||
val style: RenderStyle,
|
||||
val bitmap: Bitmap,
|
||||
)
|
||||
}
|
||||
|
||||
private data class RenderStyle(
|
||||
val textColor: Int,
|
||||
val fontSizePx: Float,
|
||||
val density: Float,
|
||||
)
|
||||
|
||||
private val ChatMathRenderRequest.renderStyle: RenderStyle
|
||||
get() = RenderStyle(textColor = textColor, fontSizePx = fontSizePx, density = density)
|
||||
|
||||
/** Process-singleton entry point. Its sole WebView and all queue state stay on the main thread. */
|
||||
internal object ChatMathRenderer {
|
||||
private val handler = Handler(Looper.getMainLooper())
|
||||
|
||||
// This is intentionally process-owned: its WebView uses the Application context, and the
|
||||
// activity host is weak, detached on destruction, then replaced on the next render.
|
||||
@SuppressLint("StaticFieldLeak")
|
||||
private var backend: ChatMathWebViewBackend? = null
|
||||
private var coordinator: ChatMathRenderCoordinator<Bitmap>? = null
|
||||
|
||||
fun render(
|
||||
context: Context,
|
||||
request: ChatMathRenderRequest,
|
||||
completion: (ChatMathRenderResult<Bitmap>) -> Unit,
|
||||
): ChatMathRenderSubscription {
|
||||
check(Looper.myLooper() == Looper.getMainLooper()) { "ChatMathRenderer must run on the main thread" }
|
||||
val host = context.findActivity()?.window?.decorView as? ViewGroup
|
||||
if (host == null) {
|
||||
completion(ChatMathRenderResult.TransientFailure)
|
||||
return ChatMathRenderSubscription {}
|
||||
}
|
||||
val renderBackend =
|
||||
backend
|
||||
?: ChatMathWebViewBackend(context.applicationContext as Application, host).also { created -> backend = created }
|
||||
renderBackend.updateHost(host)
|
||||
val renderer =
|
||||
coordinator
|
||||
?: ChatMathRenderCoordinator(
|
||||
backend = renderBackend,
|
||||
cache = ChatMathBitmapCache(),
|
||||
timeoutScheduler =
|
||||
ChatMathTimeoutScheduler { delayMs, action ->
|
||||
val runnable = Runnable(action)
|
||||
handler.postDelayed(runnable, delayMs)
|
||||
ChatMathTimeout { handler.removeCallbacks(runnable) }
|
||||
},
|
||||
).also { created -> coordinator = created }
|
||||
return renderer.render(request, completion)
|
||||
}
|
||||
}
|
||||
|
||||
private class ChatMathWebViewBackend(
|
||||
private val application: Application,
|
||||
host: ViewGroup,
|
||||
) : ChatMathRenderBackend<Bitmap> {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private var ready = false
|
||||
private var nextRenderId = 0L
|
||||
private var active: ActiveRender? = null
|
||||
private var host = WeakReference(host)
|
||||
private var webView = createWebView(application)
|
||||
private val activityCallbacks =
|
||||
object : Application.ActivityLifecycleCallbacks {
|
||||
override fun onActivityDestroyed(activity: Activity) {
|
||||
releaseHost(activity)
|
||||
}
|
||||
|
||||
override fun onActivityCreated(
|
||||
activity: Activity,
|
||||
savedInstanceState: Bundle?,
|
||||
) = Unit
|
||||
|
||||
override fun onActivityStarted(activity: Activity) = Unit
|
||||
|
||||
override fun onActivityResumed(activity: Activity) = Unit
|
||||
|
||||
override fun onActivityPaused(activity: Activity) = Unit
|
||||
|
||||
override fun onActivityStopped(activity: Activity) = Unit
|
||||
|
||||
override fun onActivitySaveInstanceState(
|
||||
activity: Activity,
|
||||
outState: Bundle,
|
||||
) = Unit
|
||||
}
|
||||
|
||||
init {
|
||||
application.registerActivityLifecycleCallbacks(activityCallbacks)
|
||||
attachWebView(webView)
|
||||
}
|
||||
|
||||
fun updateHost(host: ViewGroup) {
|
||||
this.host = WeakReference(host)
|
||||
attachWebView(webView)
|
||||
}
|
||||
|
||||
override fun render(
|
||||
request: ChatMathRenderRequest,
|
||||
completion: (ChatMathRenderResult<Bitmap>) -> Unit,
|
||||
) {
|
||||
if (host.get() == null) {
|
||||
completion(ChatMathRenderResult.TransientFailure)
|
||||
return
|
||||
}
|
||||
nextRenderId += 1
|
||||
active = ActiveRender(nextRenderId.toString(), request, completion)
|
||||
if (ready) evaluateActiveRender()
|
||||
}
|
||||
|
||||
// JavaScript is required only for the bundled KaTeX shell. The client blocks network loads,
|
||||
// rejects every URL outside the asset root, and receives LaTeX as a JSON value rather than HTML.
|
||||
@SuppressLint("SetJavaScriptEnabled")
|
||||
@Suppress("DEPRECATION")
|
||||
private fun createWebView(context: Context): WebView =
|
||||
WebView(context).apply {
|
||||
alpha = 0f
|
||||
visibility = View.VISIBLE
|
||||
isClickable = false
|
||||
setBackgroundColor(AndroidColor.TRANSPARENT)
|
||||
setLayerType(View.LAYER_TYPE_SOFTWARE, null)
|
||||
setWillNotDraw(false)
|
||||
settings.apply {
|
||||
javaScriptEnabled = true
|
||||
allowFileAccess = true
|
||||
allowContentAccess = false
|
||||
allowFileAccessFromFileURLs = false
|
||||
allowUniversalAccessFromFileURLs = false
|
||||
blockNetworkLoads = true
|
||||
domStorageEnabled = false
|
||||
databaseEnabled = false
|
||||
cacheMode = WebSettings.LOAD_NO_CACHE
|
||||
mixedContentMode = WebSettings.MIXED_CONTENT_NEVER_ALLOW
|
||||
offscreenPreRaster = true
|
||||
}
|
||||
addJavascriptInterface(RenderBridge(), "ChatMathBridge")
|
||||
webViewClient =
|
||||
object : WebViewClient() {
|
||||
override fun shouldInterceptRequest(
|
||||
view: WebView?,
|
||||
request: WebResourceRequest?,
|
||||
): WebResourceResponse? {
|
||||
val url = request?.url?.toString().orEmpty()
|
||||
return if (isAllowedAssetUrl(url)) null else emptyResponse()
|
||||
}
|
||||
|
||||
override fun shouldOverrideUrlLoading(
|
||||
view: WebView?,
|
||||
request: WebResourceRequest?,
|
||||
): Boolean = !isAllowedAssetUrl(request?.url?.toString().orEmpty())
|
||||
|
||||
override fun onPageFinished(
|
||||
view: WebView?,
|
||||
url: String?,
|
||||
) {
|
||||
if (url == KATEX_SHELL_URL) {
|
||||
ready = true
|
||||
evaluateActiveRender()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onRenderProcessGone(
|
||||
view: WebView,
|
||||
detail: RenderProcessGoneDetail,
|
||||
): Boolean {
|
||||
if (webView !== view) return true
|
||||
val interrupted = active
|
||||
active = null
|
||||
ready = false
|
||||
(view.parent as? ViewGroup)?.removeView(view)
|
||||
view.destroy()
|
||||
mainHandler.post {
|
||||
webView = createWebView(application)
|
||||
attachWebView(webView)
|
||||
interrupted?.completion?.invoke(ChatMathRenderResult.TransientFailure)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
loadUrl(KATEX_SHELL_URL)
|
||||
}
|
||||
|
||||
private fun attachWebView(view: WebView) {
|
||||
val currentHost = host.get() ?: return
|
||||
if (view.parent === currentHost) return
|
||||
(view.parent as? ViewGroup)?.removeView(view)
|
||||
// Android's visual-state contract requires a visible attached WebView. Keep this sole
|
||||
// process renderer transparent, 1 px, and behind the activity content; it never enters chat.
|
||||
currentHost.addView(view, 0, ViewGroup.LayoutParams(1, 1))
|
||||
}
|
||||
|
||||
private fun releaseHost(activity: Activity) {
|
||||
val currentHost = host.get() ?: return
|
||||
if (currentHost !== activity.window.decorView) return
|
||||
(webView.parent as? ViewGroup)?.removeView(webView)
|
||||
host.clear()
|
||||
val interrupted = active
|
||||
active = null
|
||||
interrupted?.completion?.invoke(ChatMathRenderResult.TransientFailure)
|
||||
}
|
||||
|
||||
private fun evaluateActiveRender() {
|
||||
val render = active ?: return
|
||||
val viewportWidth = render.request.key.widthBucket
|
||||
val density = render.request.density
|
||||
webView.measure(
|
||||
View.MeasureSpec.makeMeasureSpec(viewportWidth, View.MeasureSpec.EXACTLY),
|
||||
View.MeasureSpec.makeMeasureSpec(1, View.MeasureSpec.EXACTLY),
|
||||
)
|
||||
webView.layout(0, 0, viewportWidth, 1)
|
||||
webView.scrollTo(0, 0)
|
||||
|
||||
val payload =
|
||||
JSONObject()
|
||||
.put("id", render.id)
|
||||
.put("latex", render.request.key.latex)
|
||||
.put("widthCssPx", viewportWidth / density)
|
||||
.put("fontSizeCssPx", render.request.fontSizePx / density)
|
||||
.put("color", cssColor(render.request.textColor))
|
||||
webView.evaluateJavascript("window.renderMath($payload);", null)
|
||||
}
|
||||
|
||||
private inner class RenderBridge {
|
||||
@JavascriptInterface
|
||||
fun onRenderComplete(
|
||||
id: String,
|
||||
widthCssPx: Double,
|
||||
heightCssPx: Double,
|
||||
success: Boolean,
|
||||
) {
|
||||
mainHandler.post {
|
||||
val render = active?.takeIf { item -> item.id == id } ?: return@post
|
||||
val density = render.request.density
|
||||
if (!success) {
|
||||
active = null
|
||||
render.completion(ChatMathRenderResult.Failure)
|
||||
return@post
|
||||
}
|
||||
val width =
|
||||
kotlin.math
|
||||
.ceil(widthCssPx * density)
|
||||
.toInt()
|
||||
.coerceAtLeast(1)
|
||||
val height =
|
||||
kotlin.math
|
||||
.ceil(heightCssPx * density)
|
||||
.toInt()
|
||||
.coerceAtLeast(1)
|
||||
if (!isSafeBitmapSize(width, height)) {
|
||||
active = null
|
||||
render.completion(ChatMathRenderResult.Failure)
|
||||
return@post
|
||||
}
|
||||
|
||||
webView.measure(
|
||||
View.MeasureSpec.makeMeasureSpec(width, View.MeasureSpec.EXACTLY),
|
||||
View.MeasureSpec.makeMeasureSpec(height, View.MeasureSpec.EXACTLY),
|
||||
)
|
||||
webView.layout(0, 0, width, height)
|
||||
webView.scrollTo(0, 0)
|
||||
webView.postVisualStateCallback(
|
||||
id.toLong(),
|
||||
object : WebView.VisualStateCallback() {
|
||||
override fun onComplete(requestId: Long) {
|
||||
val current = active?.takeIf { item -> item.id == requestId.toString() } ?: return
|
||||
val bitmap =
|
||||
runCatching {
|
||||
webView.measure(
|
||||
View.MeasureSpec.makeMeasureSpec(width, View.MeasureSpec.EXACTLY),
|
||||
View.MeasureSpec.makeMeasureSpec(height, View.MeasureSpec.EXACTLY),
|
||||
)
|
||||
webView.layout(0, 0, width, height)
|
||||
createBitmap(width, height, Bitmap.Config.ARGB_8888).also { target ->
|
||||
target.eraseColor(AndroidColor.TRANSPARENT)
|
||||
webView.draw(Canvas(target))
|
||||
}
|
||||
}.getOrNull()
|
||||
active = null
|
||||
current.completion(
|
||||
bitmap?.let { target -> ChatMathRenderResult.Success(target) }
|
||||
?: ChatMathRenderResult.TransientFailure,
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private data class ActiveRender(
|
||||
val id: String,
|
||||
val request: ChatMathRenderRequest,
|
||||
val completion: (ChatMathRenderResult<Bitmap>) -> Unit,
|
||||
)
|
||||
}
|
||||
|
||||
private tailrec fun Context.findActivity(): Activity? =
|
||||
when (this) {
|
||||
is Activity -> this
|
||||
is ContextWrapper -> baseContext.findActivity()
|
||||
else -> null
|
||||
}
|
||||
|
||||
private fun isAllowedAssetUrl(url: String): Boolean =
|
||||
runCatching {
|
||||
val uri = url.toUri()
|
||||
uri.scheme == "file" &&
|
||||
uri.host.isNullOrEmpty() &&
|
||||
uri.path.orEmpty().startsWith("/android_asset/katex/") &&
|
||||
uri.pathSegments.none { segment -> segment == ".." }
|
||||
}.getOrDefault(false)
|
||||
|
||||
private fun emptyResponse(): WebResourceResponse = WebResourceResponse("text/plain", Charsets.UTF_8.name(), ByteArrayInputStream(ByteArray(0)))
|
||||
|
||||
private fun cssColor(argb: Int): String = String.format(Locale.US, "#%02x%02x%02x", AndroidColor.red(argb), AndroidColor.green(argb), AndroidColor.blue(argb))
|
||||
|
||||
private fun isSafeBitmapSize(
|
||||
width: Int,
|
||||
height: Int,
|
||||
): Boolean =
|
||||
width <= MATH_MAX_BITMAP_DIMENSION &&
|
||||
height <= MATH_MAX_BITMAP_DIMENSION &&
|
||||
width.toLong() * height.toLong() <= MATH_MAX_BITMAP_PIXELS
|
||||
|
||||
@Composable
|
||||
internal fun ChatMathBlock(
|
||||
latex: String,
|
||||
textColor: Color,
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
val density = LocalDensity.current
|
||||
val darkMode = textColor.luminance() > 0.5f
|
||||
BoxWithConstraints(modifier = Modifier.fillMaxWidth()) {
|
||||
val widthPx = with(density) { maxWidth.roundToPx() }
|
||||
val fontSizePx = with(density) { 16.sp.toPx() }
|
||||
val densityScale = density.density
|
||||
val request =
|
||||
remember(latex, widthPx, darkMode, textColor, fontSizePx, densityScale) {
|
||||
ChatMathRenderRequest.create(
|
||||
latex = latex,
|
||||
widthPx = widthPx,
|
||||
darkMode = darkMode,
|
||||
textColor = textColor.toArgb(),
|
||||
fontSizePx = fontSizePx,
|
||||
density = densityScale,
|
||||
)
|
||||
}
|
||||
var bitmap by remember(request) { mutableStateOf<Bitmap?>(null) }
|
||||
var failed by remember(request) { mutableStateOf(false) }
|
||||
DisposableEffect(request) {
|
||||
val subscription =
|
||||
ChatMathRenderer.render(context, request) { result ->
|
||||
when (result) {
|
||||
is ChatMathRenderResult.Success -> {
|
||||
bitmap = result.value
|
||||
failed = false
|
||||
}
|
||||
ChatMathRenderResult.Failure,
|
||||
ChatMathRenderResult.TransientFailure,
|
||||
-> failed = true
|
||||
}
|
||||
}
|
||||
onDispose { subscription.cancel() }
|
||||
}
|
||||
|
||||
val rendered = bitmap
|
||||
if (rendered == null || failed) {
|
||||
ChatMathFallback(latex)
|
||||
} else {
|
||||
val scrollState = rememberScrollState()
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxWidth()
|
||||
.horizontalScroll(scrollState),
|
||||
) {
|
||||
Image(
|
||||
bitmap = rendered.asImageBitmap(),
|
||||
contentDescription = latex,
|
||||
modifier =
|
||||
Modifier
|
||||
.width(with(density) { rendered.width.toDp() })
|
||||
.height(with(density) { rendered.height.toDp() }),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun ChatMathFallback(latex: String) {
|
||||
ChatCodeBlock(code = latex, language = null)
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import org.commonmark.node.Code
|
||||
import org.commonmark.node.Document
|
||||
import org.commonmark.node.HardLineBreak
|
||||
import org.commonmark.node.Node
|
||||
import org.commonmark.node.Paragraph
|
||||
import org.commonmark.node.Image as MarkdownImage
|
||||
import org.commonmark.node.Link as MarkdownLink
|
||||
|
||||
internal const val CHAT_MATH_MAX_BYTES = 5000
|
||||
|
||||
internal sealed interface ChatMarkdownSourceBlock {
|
||||
data class Markdown(
|
||||
val source: String,
|
||||
) : ChatMarkdownSourceBlock
|
||||
|
||||
data class Math(
|
||||
val latex: String,
|
||||
) : ChatMarkdownSourceBlock
|
||||
|
||||
data class MathFallback(
|
||||
val latex: String,
|
||||
) : ChatMarkdownSourceBlock
|
||||
}
|
||||
|
||||
/** Extracts top-level display math while CommonMark owns code/container boundaries. */
|
||||
internal fun segmentChatMarkdown(
|
||||
source: String,
|
||||
isStreaming: Boolean,
|
||||
): List<ChatMarkdownSourceBlock> {
|
||||
if (!source.contains("$$") && !source.contains("\\[")) {
|
||||
return listOf(ChatMarkdownSourceBlock.Markdown(source))
|
||||
}
|
||||
|
||||
val document = parseChatMarkdown(source)
|
||||
val topLevelParagraphLines = mutableSetOf<Int>()
|
||||
val protectedInlineLines = mutableSetOf<Int>()
|
||||
var child = document.firstChild
|
||||
while (child != null) {
|
||||
if (child is Paragraph) {
|
||||
child.sourceSpans.forEach { span -> topLevelParagraphLines.add(span.lineIndex) }
|
||||
collectProtectedInlineLines(child.firstChild, protectedInlineLines)
|
||||
}
|
||||
child = child.next
|
||||
}
|
||||
|
||||
val lines = source.split('\n')
|
||||
val extractions = mutableListOf<MathExtraction>()
|
||||
var lineIndex = 0
|
||||
while (lineIndex < lines.size) {
|
||||
val opener =
|
||||
if (lineIndex in topLevelParagraphLines && lineIndex !in protectedInlineLines) {
|
||||
MathDelimiter.parse(lines[lineIndex])
|
||||
} else {
|
||||
null
|
||||
}
|
||||
if (opener == null) {
|
||||
lineIndex += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (opener.sameLineLatex != null) {
|
||||
extractions.add(MathExtraction(lineIndex..lineIndex, opener.sameLineLatex))
|
||||
lineIndex += 1
|
||||
continue
|
||||
}
|
||||
|
||||
var closeIndex = lineIndex + 1
|
||||
while (
|
||||
closeIndex < lines.size &&
|
||||
(
|
||||
closeIndex !in topLevelParagraphLines ||
|
||||
closeIndex in protectedInlineLines ||
|
||||
!opener.isClose(lines[closeIndex])
|
||||
)
|
||||
) {
|
||||
closeIndex += 1
|
||||
}
|
||||
val closed = closeIndex < lines.size
|
||||
if (!closed && isStreaming) {
|
||||
// One unmatched opener owns the remaining stream; later delimiter-looking lines stay text.
|
||||
break
|
||||
}
|
||||
|
||||
val contentEnd = if (closed) closeIndex else lines.size
|
||||
val latex = lines.subList(lineIndex + 1, contentEnd).joinToString("\n").trim()
|
||||
val extractionEnd = if (closed) closeIndex else lines.lastIndex
|
||||
extractions.add(MathExtraction(lineIndex..extractionEnd, latex))
|
||||
lineIndex = extractionEnd + 1
|
||||
}
|
||||
|
||||
if (extractions.isEmpty()) {
|
||||
return listOf(ChatMarkdownSourceBlock.Markdown(source))
|
||||
}
|
||||
if (containsReferenceStyleLink(document, source)) {
|
||||
// Splitting would separate a reference link from its definition and change CommonMark semantics.
|
||||
return listOf(ChatMarkdownSourceBlock.Markdown(source))
|
||||
}
|
||||
|
||||
val blocks = mutableListOf<ChatMarkdownSourceBlock>()
|
||||
var proseStart = 0
|
||||
for (extraction in extractions) {
|
||||
appendMarkdownBlock(lines, proseStart, extraction.lines.first, blocks)
|
||||
blocks.add(
|
||||
if (extraction.latex.toByteArray(Charsets.UTF_8).size <= CHAT_MATH_MAX_BYTES) {
|
||||
ChatMarkdownSourceBlock.Math(extraction.latex)
|
||||
} else {
|
||||
ChatMarkdownSourceBlock.MathFallback(extraction.latex)
|
||||
},
|
||||
)
|
||||
proseStart = extraction.lines.last + 1
|
||||
}
|
||||
appendMarkdownBlock(lines, proseStart, lines.size, blocks)
|
||||
return blocks
|
||||
}
|
||||
|
||||
private fun containsReferenceStyleLink(
|
||||
document: Document,
|
||||
source: String,
|
||||
): Boolean {
|
||||
fun search(start: Node?): Boolean {
|
||||
var node = start
|
||||
while (node != null) {
|
||||
if (node is MarkdownLink || node is MarkdownImage) {
|
||||
val spans = node.sourceSpans
|
||||
val startIndex = spans.minOfOrNull { span -> span.inputIndex }
|
||||
val endIndex = spans.maxOfOrNull { span -> span.inputIndex + span.length }
|
||||
if (startIndex != null && endIndex != null && startIndex >= 0 && endIndex <= source.length) {
|
||||
if (source.substring(startIndex, endIndex).trimEnd().endsWith(']')) return true
|
||||
}
|
||||
}
|
||||
if (search(node.firstChild)) return true
|
||||
node = node.next
|
||||
}
|
||||
return false
|
||||
}
|
||||
return search(document.firstChild)
|
||||
}
|
||||
|
||||
private fun collectProtectedInlineLines(
|
||||
start: Node?,
|
||||
lines: MutableSet<Int>,
|
||||
) {
|
||||
var node = start
|
||||
while (node != null) {
|
||||
if (node is Code) {
|
||||
node.sourceSpans.forEach { span -> lines.add(span.lineIndex) }
|
||||
}
|
||||
if (node is HardLineBreak) {
|
||||
node.sourceSpans.forEach { span -> lines.add(span.lineIndex + 1) }
|
||||
}
|
||||
val spannedLines = node.sourceSpans.map { span -> span.lineIndex }.distinct()
|
||||
if (spannedLines.size > 1) {
|
||||
lines.addAll(checkNotNull(spannedLines.minOrNull())..checkNotNull(spannedLines.maxOrNull()))
|
||||
}
|
||||
collectProtectedInlineLines(node.firstChild, lines)
|
||||
node = node.next
|
||||
}
|
||||
}
|
||||
|
||||
private fun appendMarkdownBlock(
|
||||
lines: List<String>,
|
||||
start: Int,
|
||||
end: Int,
|
||||
blocks: MutableList<ChatMarkdownSourceBlock>,
|
||||
) {
|
||||
var contentStart = start
|
||||
var contentEnd = end
|
||||
while (contentStart < contentEnd && lines[contentStart].isBlank()) contentStart += 1
|
||||
while (contentEnd > contentStart && lines[contentEnd - 1].isBlank()) contentEnd -= 1
|
||||
if (contentStart < contentEnd) {
|
||||
blocks.add(ChatMarkdownSourceBlock.Markdown(lines.subList(contentStart, contentEnd).joinToString("\n")))
|
||||
}
|
||||
}
|
||||
|
||||
private data class MathExtraction(
|
||||
val lines: IntRange,
|
||||
val latex: String,
|
||||
)
|
||||
|
||||
private data class MathDelimiter(
|
||||
val close: String,
|
||||
val sameLineLatex: String?,
|
||||
) {
|
||||
fun isClose(line: String): Boolean = line.trim() == close
|
||||
|
||||
companion object {
|
||||
fun parse(line: String): MathDelimiter? {
|
||||
val indent = line.indexOfFirst { char -> char != ' ' }.let { index -> if (index == -1) line.length else index }
|
||||
if (indent > 3 || indent == line.length) return null
|
||||
val suffix = line.substring(indent)
|
||||
val (open, close) =
|
||||
when {
|
||||
suffix.startsWith("$$") -> "$$" to "$$"
|
||||
suffix.startsWith("\\[") -> "\\[" to "\\]"
|
||||
else -> return null
|
||||
}
|
||||
val remainder = suffix.removePrefix(open)
|
||||
if (remainder.isBlank()) return MathDelimiter(close = close, sameLineLatex = null)
|
||||
val closeIndex = remainder.lastIndexOf(close)
|
||||
if (closeIndex < 0 || remainder.substring(closeIndex + close.length).isNotBlank()) return null
|
||||
return MathDelimiter(close = close, sameLineLatex = remainder.substring(0, closeIndex).trim())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ class AndroidLicenseNoticesTest {
|
||||
"Bouncy Castle Provider",
|
||||
"CommonMark Java",
|
||||
"dnsjava",
|
||||
"KaTeX",
|
||||
"Kotlin Libraries",
|
||||
"Manrope",
|
||||
"nibor autolink",
|
||||
|
||||
@@ -15,6 +15,89 @@ import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
class ChatMarkdownTest {
|
||||
@Test
|
||||
fun displayMathSegmentsOwnLineAndSameLineDollarBlocks() {
|
||||
val sameLine = segmentChatMarkdown("before\n$$ x^2 + y^2 $$\nafter", isStreaming = false)
|
||||
val ownLine = segmentChatMarkdown("$$\nx + y\n$$", isStreaming = false)
|
||||
|
||||
assertEquals(
|
||||
listOf(
|
||||
ChatMarkdownSourceBlock.Markdown("before"),
|
||||
ChatMarkdownSourceBlock.Math("x^2 + y^2"),
|
||||
ChatMarkdownSourceBlock.Markdown("after"),
|
||||
),
|
||||
sameLine,
|
||||
)
|
||||
assertEquals(listOf(ChatMarkdownSourceBlock.Math("x + y")), ownLine)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun displayMathSegmentsBracketBlocks() {
|
||||
assertEquals(
|
||||
listOf(ChatMarkdownSourceBlock.Math("\\frac{a}{b}")),
|
||||
segmentChatMarkdown("\\[\\frac{a}{b}\\]", isStreaming = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun displayMathIgnoresFencedAndInlineCode() {
|
||||
val fenced = "```tex\n$$\nx + y\n$$\n```"
|
||||
val inline = "`$$ x + y $$`"
|
||||
|
||||
assertEquals(listOf(ChatMarkdownSourceBlock.Markdown(fenced)), segmentChatMarkdown(fenced, isStreaming = false))
|
||||
assertEquals(listOf(ChatMarkdownSourceBlock.Markdown(inline)), segmentChatMarkdown(inline, isStreaming = false))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun fencedDelimiterCannotCloseStreamingDisplayMath() {
|
||||
val source = "$$\nx\n```text\n$$\n```"
|
||||
|
||||
assertEquals(
|
||||
listOf(ChatMarkdownSourceBlock.Markdown(source)),
|
||||
segmentChatMarkdown(source, isStreaming = true),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun displayMathDoesNotSplitSpanningInlineMarkup() {
|
||||
val emphasized = "*before\n$$ x + y $$\nafter*"
|
||||
|
||||
assertEquals(
|
||||
listOf(ChatMarkdownSourceBlock.Markdown(emphasized)),
|
||||
segmentChatMarkdown(emphasized, isStreaming = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun displayMathDoesNotExposeHardBreakEscape() {
|
||||
val hardBreak = "before\\\n$$ x + y $$\nafter"
|
||||
|
||||
assertEquals(
|
||||
listOf(ChatMarkdownSourceBlock.Markdown(hardBreak)),
|
||||
segmentChatMarkdown(hardBreak, isStreaming = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun unclosedStreamingDisplayMathStaysMarkdown() {
|
||||
val source = "before\n$$\nx + y"
|
||||
|
||||
assertEquals(
|
||||
listOf(ChatMarkdownSourceBlock.Markdown(source)),
|
||||
segmentChatMarkdown(source, isStreaming = true),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun oversizedDisplayMathUsesCodeFallback() {
|
||||
val latex = "é".repeat(CHAT_MATH_MAX_BYTES / 2 + 1)
|
||||
|
||||
assertEquals(
|
||||
listOf(ChatMarkdownSourceBlock.MathFallback(latex)),
|
||||
segmentChatMarkdown("$$\n$latex\n$$", isStreaming = false),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun bareUrlsCarryClickableUrlAnnotations() {
|
||||
val url = "https://www.amazon.it/GAZEBO-CANOPY-ACCIAIO-BIANCO-IMPERMEABILE/dp/B01G5R9FCK"
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.RuntimeEnvironment
|
||||
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
class ChatMathAssetsTest {
|
||||
@Test
|
||||
fun rendererReportsMalformedKatexAsFailure() {
|
||||
val assets = RuntimeEnvironment.getApplication().assets
|
||||
val renderer = assets.open("katex/renderer.js").bufferedReader().use { reader -> reader.readText() }
|
||||
|
||||
assertTrue(Regex("""throwOnError:\s*true""").containsMatchIn(renderer))
|
||||
assertFalse(Regex("""throwOnError:\s*false""").containsMatchIn(renderer))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNotEquals
|
||||
import org.junit.Test
|
||||
|
||||
class ChatMathRendererTest {
|
||||
@Test
|
||||
fun queuePreservesOrderAndDeduplicatesMatchingJobs() {
|
||||
val harness = RenderHarness()
|
||||
val results = mutableListOf<String>()
|
||||
val first = request("first")
|
||||
val second = request("second")
|
||||
|
||||
harness.coordinator.render(first) { result -> results.add("a:${result.value()}") }
|
||||
harness.coordinator.render(first) { result -> results.add("b:${result.value()}") }
|
||||
harness.coordinator.render(second) { result -> results.add("c:${result.value()}") }
|
||||
|
||||
assertEquals(listOf(first), harness.backend.requests)
|
||||
harness.backend.complete(ChatMathRenderResult.Success("one"))
|
||||
assertEquals(listOf(first, second), harness.backend.requests)
|
||||
assertEquals(listOf("a:one", "b:one"), results)
|
||||
harness.backend.complete(ChatMathRenderResult.Success("two"))
|
||||
assertEquals(listOf("a:one", "b:one", "c:two"), results)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun cacheKeyBucketsWidthAndIncludesDarkMode() {
|
||||
val lightA = request("x", widthPx = 321, darkMode = false)
|
||||
val lightB = request("x", widthPx = 350, darkMode = false)
|
||||
val dark = request("x", widthPx = 321, darkMode = true)
|
||||
|
||||
assertEquals(lightA.key, lightB.key)
|
||||
assertNotEquals(lightA.key, dark.key)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun presentationChangeWithSameKeyRendersAgain() {
|
||||
val harness = RenderHarness()
|
||||
val first = request("x")
|
||||
val recolored = first.copy(textColor = 0xffffffff.toInt())
|
||||
|
||||
harness.coordinator.render(first) {}
|
||||
harness.backend.complete(ChatMathRenderResult.Success("first"))
|
||||
harness.coordinator.render(recolored) {}
|
||||
|
||||
assertEquals(first.key, recolored.key)
|
||||
assertEquals(listOf(first, recolored), harness.backend.requests)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun negativeResultsAreCachedWithoutAnotherBackendJob() {
|
||||
val harness = RenderHarness()
|
||||
val request = request("bad")
|
||||
val results = mutableListOf<ChatMathRenderResult<String>>()
|
||||
|
||||
harness.coordinator.render(request, results::add)
|
||||
harness.backend.complete(ChatMathRenderResult.Failure)
|
||||
harness.coordinator.render(request, results::add)
|
||||
|
||||
assertEquals(1, harness.backend.requests.size)
|
||||
assertEquals(listOf(ChatMathRenderResult.Failure, ChatMathRenderResult.Failure), results)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun transientFailuresCanRetryTheSameKey() {
|
||||
val harness = RenderHarness()
|
||||
val request = request("retry")
|
||||
|
||||
harness.coordinator.render(request) {}
|
||||
harness.backend.complete(ChatMathRenderResult.TransientFailure)
|
||||
harness.coordinator.render(request) {}
|
||||
|
||||
assertEquals(listOf(request, request), harness.backend.requests)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun cancelDropsQueuedJobWithoutInterruptingActiveCacheWarmup() {
|
||||
val harness = RenderHarness()
|
||||
val first = request("first")
|
||||
val canceled = request("canceled")
|
||||
val last = request("last")
|
||||
|
||||
harness.coordinator.render(first) {}
|
||||
harness.coordinator.render(canceled) {}.cancel()
|
||||
harness.coordinator.render(last) {}
|
||||
harness.backend.complete(ChatMathRenderResult.Success("one"))
|
||||
|
||||
assertEquals(listOf(first, last), harness.backend.requests)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun timeoutFailsCurrentJobAndAdvancesQueue() {
|
||||
val harness = RenderHarness()
|
||||
val results = mutableListOf<ChatMathRenderResult<String>>()
|
||||
val first = request("first")
|
||||
val second = request("second")
|
||||
|
||||
harness.coordinator.render(first, results::add)
|
||||
harness.coordinator.render(second, results::add)
|
||||
harness.scheduler.fire()
|
||||
|
||||
assertEquals(listOf(ChatMathRenderResult.TransientFailure), results)
|
||||
assertEquals(listOf(first, second), harness.backend.requests)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun staleCompletionAfterTimeoutCannotCompleteRetry() {
|
||||
val harness = RenderHarness()
|
||||
val firstResults = mutableListOf<ChatMathRenderResult<String>>()
|
||||
val retryResults = mutableListOf<ChatMathRenderResult<String>>()
|
||||
val request = request("retry")
|
||||
|
||||
harness.coordinator.render(request, firstResults::add)
|
||||
val staleCompletion = harness.backend.completions.removeAt(0)
|
||||
harness.scheduler.fire()
|
||||
harness.coordinator.render(request, retryResults::add)
|
||||
staleCompletion(ChatMathRenderResult.Success("stale"))
|
||||
|
||||
assertEquals(listOf(ChatMathRenderResult.TransientFailure), firstResults)
|
||||
assertEquals(emptyList<ChatMathRenderResult<String>>(), retryResults)
|
||||
harness.backend.complete(ChatMathRenderResult.Success("fresh"))
|
||||
assertEquals(listOf(ChatMathRenderResult.Success("fresh")), retryResults)
|
||||
}
|
||||
|
||||
private class RenderHarness {
|
||||
val backend = FakeBackend()
|
||||
val cache = FakeCache()
|
||||
val scheduler = FakeScheduler()
|
||||
val coordinator = ChatMathRenderCoordinator(backend, cache, scheduler)
|
||||
}
|
||||
|
||||
private class FakeBackend : ChatMathRenderBackend<String> {
|
||||
val requests = mutableListOf<ChatMathRenderRequest>()
|
||||
val completions = mutableListOf<(ChatMathRenderResult<String>) -> Unit>()
|
||||
|
||||
override fun render(
|
||||
request: ChatMathRenderRequest,
|
||||
completion: (ChatMathRenderResult<String>) -> Unit,
|
||||
) {
|
||||
requests.add(request)
|
||||
completions.add(completion)
|
||||
}
|
||||
|
||||
fun complete(result: ChatMathRenderResult<String>) {
|
||||
completions.removeAt(0).invoke(result)
|
||||
}
|
||||
}
|
||||
|
||||
private class FakeCache : ChatMathRenderCache<String> {
|
||||
private val entries = mutableMapOf<ChatMathRenderRequest, ChatMathCacheEntry<String>>()
|
||||
|
||||
override fun get(request: ChatMathRenderRequest): ChatMathCacheEntry<String> = entries[request] ?: ChatMathCacheEntry.Missing
|
||||
|
||||
override fun put(
|
||||
request: ChatMathRenderRequest,
|
||||
result: ChatMathRenderResult<String>,
|
||||
) {
|
||||
entries[request] =
|
||||
when (result) {
|
||||
is ChatMathRenderResult.Success -> ChatMathCacheEntry.Success(result.value)
|
||||
ChatMathRenderResult.Failure -> ChatMathCacheEntry.Failure
|
||||
ChatMathRenderResult.TransientFailure -> ChatMathCacheEntry.Missing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class FakeScheduler : ChatMathTimeoutScheduler {
|
||||
private var action: (() -> Unit)? = null
|
||||
|
||||
override fun schedule(
|
||||
delayMs: Long,
|
||||
action: () -> Unit,
|
||||
): ChatMathTimeout {
|
||||
this.action = action
|
||||
return ChatMathTimeout { if (this.action === action) this.action = null }
|
||||
}
|
||||
|
||||
fun fire() {
|
||||
val pending = action
|
||||
action = null
|
||||
checkNotNull(pending).invoke()
|
||||
}
|
||||
}
|
||||
|
||||
private fun ChatMathRenderResult<String>.value(): String =
|
||||
when (this) {
|
||||
is ChatMathRenderResult.Success -> value
|
||||
ChatMathRenderResult.Failure -> "failure"
|
||||
ChatMathRenderResult.TransientFailure -> "transient failure"
|
||||
}
|
||||
|
||||
private fun request(
|
||||
latex: String,
|
||||
widthPx: Int = 321,
|
||||
darkMode: Boolean = false,
|
||||
): ChatMathRenderRequest =
|
||||
ChatMathRenderRequest.create(
|
||||
latex = latex,
|
||||
widthPx = widthPx,
|
||||
darkMode = darkMode,
|
||||
textColor = 0xff000000.toInt(),
|
||||
fontSizePx = 16f,
|
||||
density = 1f,
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user