let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

A2UI: share web UI and action bridge

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-18 11:38:32 +01:00
parent 8a343aedf2
commit c61bd6c84d
24 changed files with 809 additions and 18655 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18021
apps/android/app/src/main/assets/canvas_a2ui/a2ui.bundle.js
View File

File diff suppressed because one or more lines are too long

23
apps/android/app/src/main/assets/canvas_a2ui/index.html
View File

@@ -1,23 +0,0 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Canvas</title>
<style>
:root { color-scheme: light dark; }
html, body { height: 100%; margin: 0; }
body {
font: 13px -apple-system, system-ui;
background: #0b1020;
color: #e5e7eb;
overflow: hidden;
}
clawdis-a2ui-host { display: block; height: 100%; }
</style>
</head>
<body>
<clawdis-a2ui-host></clawdis-a2ui-host>
<script src="a2ui.bundle.js"></script>
</body>
</html>

4
apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt
View File

@@ -97,6 +97,10 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.disconnect()
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
runtime.handleCanvasA2UIActionFromWebView(payloadJson)
}
fun loadChat(sessionKey: String = "main") {
runtime.loadChat(sessionKey)
}

93
apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt
View File

@@ -18,6 +18,7 @@ import com.steipete.clawdis.node.node.CameraCaptureManager
import com.steipete.clawdis.node.node.CanvasController
import com.steipete.clawdis.node.protocol.ClawdisCapability
import com.steipete.clawdis.node.protocol.ClawdisCameraCommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UIAction
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UICommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasCommand
import com.steipete.clawdis.node.voice.VoiceWakeManager
@@ -364,6 +365,79 @@ class NodeRuntime(context: Context) {
session.disconnect()
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
scope.launch {
val trimmed = payloadJson.trim()
if (trimmed.isEmpty()) return@launch
val root =
try {
json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch
} catch (_: Throwable) {
return@launch
}
val userActionObj = (root["userAction"] as? JsonObject) ?: root
val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
java.util.UUID.randomUUID().toString()
}
val name = (userActionObj["name"] as? JsonPrimitive)?.content?.trim().orEmpty()
if (name.isEmpty()) return@launch
val surfaceId =
(userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
val sourceComponentId =
(userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" }
val contextJson = (userActionObj["context"] as? JsonObject)?.toString()
val sessionKey = "main"
val message =
ClawdisCanvasA2UIAction.formatAgentMessage(
actionName = name,
sessionKey = sessionKey,
surfaceId = surfaceId,
sourceComponentId = sourceComponentId,
host = displayName.value,
instanceId = instanceId.value.lowercase(),
contextJson = contextJson,
)
val connected = isConnected.value
var error: String? = null
if (connected) {
try {
session.sendEvent(
event = "agent.request",
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(message))
put("sessionKey", JsonPrimitive(sessionKey))
put("thinking", JsonPrimitive("low"))
put("deliver", JsonPrimitive(false))
put("key", JsonPrimitive(actionId))
}.toString(),
)
} catch (e: Throwable) {
error = e.message ?: "send failed"
}
} else {
error = "bridge not connected"
}
try {
canvas.eval(
ClawdisCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId = actionId,
ok = connected && error == null,
error = error,
),
)
} catch (_: Throwable) {
// ignore
}
}
}
fun loadChat(sessionKey: String = "main") {
chat.load(sessionKey)
}
@@ -581,11 +655,15 @@ class NodeRuntime(context: Context) {
val raw = paramsJson?.trim().orEmpty()
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
if (command == ClawdisCanvasA2UICommand.PushJSONL.rawValue) {
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val jsonl = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
val hasMessagesArray = obj["messages"] is JsonArray
if (command == ClawdisCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) {
val jsonl = jsonlField
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
val messages =
jsonl
@@ -604,9 +682,6 @@ class NodeRuntime(context: Context) {
return JsonArray(messages).toString()
}
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
val out =
arr.mapIndexed { idx, el ->
@@ -638,7 +713,7 @@ class NodeRuntime(context: Context) {
private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
private const val a2uiIndexUrl: String = "file:///android_asset/canvas_a2ui/index.html"
private const val a2uiIndexUrl: String = "file:///android_asset/CanvasA2UI/index.html"
private const val a2uiReadyCheckJS: String =
"""

159
apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt
View File

@@ -20,6 +20,8 @@ class CanvasController {
@Volatile private var mode: Mode = Mode.CANVAS
@Volatile private var url: String = ""
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
fun attach(webView: WebView) {
this.webView = webView
reload()
@@ -59,7 +61,7 @@ class CanvasController {
if (trimmed.isBlank()) return@withWebViewOnMain
wv.loadUrl(trimmed)
}
Mode.CANVAS -> wv.loadDataWithBaseURL(null, canvasHtml, "text/html", "utf-8", null)
Mode.CANVAS -> wv.loadUrl(scaffoldAssetUrl)
}
}
}
@@ -143,158 +145,3 @@ class CanvasController {
}
}
}
private val canvasHtml =
"""
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
<title>Canvas</title>
<style>
:root { color-scheme: dark; }
@media (prefers-reduced-motion: reduce) {
body::before, body::after { animation: none !important; }
}
html,body { height:100%; margin:0; }
body {
background: radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.18), rgba(0,0,0,0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.14), rgba(0,0,0,0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.10), rgba(0,0,0,0) 60%),
#000;
overflow: hidden;
}
body::before {
content:"";
position: fixed;
inset: -20%;
background:
repeating-linear-gradient(0deg, rgba(255,255,255,0.03) 0, rgba(255,255,255,0.03) 1px,
transparent 1px, transparent 48px),
repeating-linear-gradient(90deg, rgba(255,255,255,0.03) 0, rgba(255,255,255,0.03) 1px,
transparent 1px, transparent 48px);
transform: translate3d(0,0,0) rotate(-7deg);
will-change: transform, opacity;
-webkit-backface-visibility: hidden;
backface-visibility: hidden;
opacity: 0.45;
pointer-events: none;
animation: clawdis-grid-drift 140s ease-in-out infinite alternate;
}
body::after {
content:"";
position: fixed;
inset: -35%;
background:
radial-gradient(900px 700px at 30% 30%, rgba(42,113,255,0.16), rgba(0,0,0,0) 60%),
radial-gradient(800px 650px at 70% 35%, rgba(255,0,138,0.12), rgba(0,0,0,0) 62%),
radial-gradient(900px 800px at 55% 75%, rgba(0,209,255,0.10), rgba(0,0,0,0) 62%);
filter: blur(28px);
opacity: 0.52;
will-change: transform, opacity;
-webkit-backface-visibility: hidden;
backface-visibility: hidden;
transform: translate3d(0,0,0);
pointer-events: none;
animation: clawdis-glow-drift 110s ease-in-out infinite alternate;
}
@supports (mix-blend-mode: screen) {
body::after { mix-blend-mode: screen; }
}
@supports not (mix-blend-mode: screen) {
body::after { opacity: 0.70; }
}
@keyframes clawdis-grid-drift {
0% { transform: translate3d(-12px, 8px, 0) rotate(-7deg); opacity: 0.40; }
50% { transform: translate3d( 10px,-7px, 0) rotate(-6.6deg); opacity: 0.56; }
100% { transform: translate3d(-8px, 6px, 0) rotate(-7.2deg); opacity: 0.42; }
}
@keyframes clawdis-glow-drift {
0% { transform: translate3d(-18px, 12px, 0) scale(1.02); opacity: 0.40; }
50% { transform: translate3d( 14px,-10px, 0) scale(1.05); opacity: 0.52; }
100% { transform: translate3d(-10px, 8px, 0) scale(1.03); opacity: 0.43; }
}
canvas {
display:block;
width:100vw;
height:100vh;
touch-action: none;
}
#clawdis-status {
position: fixed;
inset: 0;
display: grid;
place-items: center;
pointer-events: none;
}
#clawdis-status .card {
text-align: center;
padding: 16px 18px;
border-radius: 14px;
background: rgba(18, 18, 22, 0.42);
border: 1px solid rgba(255,255,255,0.08);
box-shadow: 0 18px 60px rgba(0,0,0,0.55);
-webkit-backdrop-filter: blur(14px);
backdrop-filter: blur(14px);
}
#clawdis-status .title {
font: 600 20px -apple-system, BlinkMacSystemFont, "SF Pro Display", "SF Pro Text", system-ui, sans-serif;
letter-spacing: 0.2px;
color: rgba(255,255,255,0.92);
text-shadow: 0 0 22px rgba(42, 113, 255, 0.35);
}
#clawdis-status .subtitle {
margin-top: 6px;
font: 500 12px -apple-system, BlinkMacSystemFont, "SF Pro Text", system-ui, sans-serif;
color: rgba(255,255,255,0.58);
}
</style>
</head>
<body>
<canvas id="clawdis-canvas"></canvas>
<div id="clawdis-status">
<div class="card">
<div class="title" id="clawdis-status-title">Ready</div>
<div class="subtitle" id="clawdis-status-subtitle">Waiting for agent</div>
</div>
</div>
<script>
(() => {
const canvas = document.getElementById('clawdis-canvas');
const ctx = canvas.getContext('2d');
const statusEl = document.getElementById('clawdis-status');
const titleEl = document.getElementById('clawdis-status-title');
const subtitleEl = document.getElementById('clawdis-status-subtitle');
function resize() {
const dpr = window.devicePixelRatio || 1;
const w = Math.max(1, Math.floor(window.innerWidth * dpr));
const h = Math.max(1, Math.floor(window.innerHeight * dpr));
canvas.width = w;
canvas.height = h;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
}
window.addEventListener('resize', resize);
resize();
window.__clawdis = {
canvas,
ctx,
setStatus: (title, subtitle) => {
if (!statusEl) return;
if (!title && !subtitle) {
statusEl.style.display = 'none';
return;
}
statusEl.style.display = 'grid';
if (titleEl && typeof title === 'string') titleEl.textContent = title;
if (subtitleEl && typeof subtitle === 'string') subtitleEl.textContent = subtitle;
}
};
})();
</script>
</body>
</html>
""".trimIndent()

49
apps/android/app/src/main/java/com/steipete/clawdis/node/protocol/ClawdisCanvasA2UIAction.kt Normal file
View File

@@ -0,0 +1,49 @@
package com.steipete.clawdis.node.protocol
object ClawdisCanvasA2UIAction {
fun sanitizeTagValue(value: String): String {
val trimmed = value.trim().ifEmpty { "-" }
val normalized = trimmed.replace(" ", "_")
val out = StringBuilder(normalized.length)
for (c in normalized) {
val ok =
c.isLetterOrDigit() ||
c == '_' ||
c == '-' ||
c == '.' ||
c == ':'
out.append(if (ok) c else '_')
}
return out.toString()
}
fun formatAgentMessage(
actionName: String,
sessionKey: String,
surfaceId: String,
sourceComponentId: String,
host: String,
instanceId: String,
contextJson: String?,
): String {
val ctxSuffix = contextJson?.takeIf { it.isNotBlank() }?.let { " ctx=$it" }.orEmpty()
return listOf(
"CANVAS_A2UI",
"action=${sanitizeTagValue(actionName)}",
"session=${sanitizeTagValue(sessionKey)}",
"surface=${sanitizeTagValue(surfaceId)}",
"component=${sanitizeTagValue(sourceComponentId)}",
"host=${sanitizeTagValue(host)}",
"instance=${sanitizeTagValue(instanceId)}$ctxSuffix",
"default=update_canvas",
).joinToString(separator = " ")
}
fun jsDispatchA2UIActionStatus(actionId: String, ok: Boolean, error: String?): String {
val err = (error ?: "").replace("\\", "\\\\").replace("\"", "\\\"")
val okLiteral = if (ok) "true" else "false"
val idEscaped = actionId.replace("\\", "\\\\").replace("\"", "\\\"")
return "window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));"
}
}

87
apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt
View File

@@ -6,13 +6,13 @@ import android.content.pm.PackageManager
import android.graphics.Color
import android.util.Log
import android.view.View
import android.webkit.JavascriptInterface
import android.webkit.WebView
import android.webkit.WebSettings
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebViewClient
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
@@ -24,12 +24,10 @@ import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.foundation.background
import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.FilledTonalIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.ModalBottomSheet
import androidx.compose.material3.rememberModalBottomSheetState
import androidx.compose.material.icons.Icons
@@ -42,10 +40,7 @@ import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color as ComposeColor
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
@@ -80,7 +75,6 @@ fun RootScreen(viewModel: MainViewModel) {
PackageManager.PERMISSION_GRANTED
Box(modifier = Modifier.fillMaxSize()) {
CanvasBackdrop(modifier = Modifier.fillMaxSize())
CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize())
}
@@ -187,73 +181,30 @@ private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier)
)
}
}
setBackgroundColor(Color.TRANSPARENT)
setBackgroundResource(0)
// WebView transparency + HW acceleration can render as solid black on some Android/WebView builds.
// Prefer correct alpha blending since we render the idle backdrop in Compose underneath.
setLayerType(View.LAYER_TYPE_SOFTWARE, null)
setBackgroundColor(Color.BLACK)
setLayerType(View.LAYER_TYPE_HARDWARE, null)
addJavascriptInterface(
CanvasA2UIActionBridge { payload ->
viewModel.handleCanvasA2UIActionFromWebView(payload)
},
CanvasA2UIActionBridge.interfaceName,
)
viewModel.canvas.attach(this)
}
},
)
}
@Composable
private fun CanvasBackdrop(modifier: Modifier = Modifier) {
val base = MaterialTheme.colorScheme.background
private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) {
@JavascriptInterface
fun postMessage(payload: String?) {
val msg = payload?.trim().orEmpty()
if (msg.isEmpty()) return
onMessage(msg)
}
Canvas(modifier = modifier.background(base)) {
// Subtle idle backdrop; also acts as fallback when WebView content is transparent or fails to load.
drawRect(
brush =
Brush.linearGradient(
colors =
listOf(
ComposeColor(0xFF0A2034),
ComposeColor(0xFF070A10),
ComposeColor(0xFF250726),
),
start = Offset(0f, 0f),
end = Offset(size.width, size.height),
),
)
val step = 48f * density
val lineColor = ComposeColor.White.copy(alpha = 0.028f)
var x = -step
while (x < size.width + step) {
drawLine(color = lineColor, start = Offset(x, 0f), end = Offset(x, size.height), strokeWidth = 1f)
x += step
}
var y = -step
while (y < size.height + step) {
drawLine(color = lineColor, start = Offset(0f, y), end = Offset(size.width, y), strokeWidth = 1f)
y += step
}
drawRect(
brush =
Brush.radialGradient(
colors = listOf(ComposeColor(0xFF2A71FF).copy(alpha = 0.22f), ComposeColor.Transparent),
center = Offset(size.width * 0.15f, size.height * 0.20f),
radius = size.minDimension * 0.9f,
),
)
drawRect(
brush =
Brush.radialGradient(
colors = listOf(ComposeColor(0xFFFF008A).copy(alpha = 0.18f), ComposeColor.Transparent),
center = Offset(size.width * 0.85f, size.height * 0.30f),
radius = size.minDimension * 0.75f,
),
)
drawRect(
brush =
Brush.radialGradient(
colors = listOf(ComposeColor(0xFF00D1FF).copy(alpha = 0.14f), ComposeColor.Transparent),
center = Offset(size.width * 0.60f, size.height * 0.90f),
radius = size.minDimension * 0.85f,
),
)
companion object {
const val interfaceName: String = "clawdisCanvasA2UIAction"
}
}