let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Android: add canvas.a2ui push/reset

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-18 10:44:50 +01:00
parent 6f58a9d643
commit cfb36525ab
7 changed files with 18248 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18021
apps/android/app/src/main/assets/canvas_a2ui/a2ui.bundle.js Normal file
View File

File diff suppressed because one or more lines are too long

23
apps/android/app/src/main/assets/canvas_a2ui/index.html Normal file
View File

@@ -0,0 +1,23 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Canvas</title>
<style>
:root { color-scheme: light dark; }
html, body { height: 100%; margin: 0; }
body {
font: 13px -apple-system, system-ui;
background: #0b1020;
color: #e5e7eb;
overflow: hidden;
}
clawdis-a2ui-host { display: block; height: 100%; }
</style>
</head>
<body>
<clawdis-a2ui-host></clawdis-a2ui-host>
<script src="a2ui.bundle.js"></script>
</body>
</html>

161
apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt
View File

@@ -18,6 +18,7 @@ import com.steipete.clawdis.node.node.CameraCaptureManager
import com.steipete.clawdis.node.node.CanvasController import com.steipete.clawdis.node.node.CanvasController
import com.steipete.clawdis.node.protocol.ClawdisCapability import com.steipete.clawdis.node.protocol.ClawdisCapability
import com.steipete.clawdis.node.protocol.ClawdisCameraCommand import com.steipete.clawdis.node.protocol.ClawdisCameraCommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasA2UICommand
import com.steipete.clawdis.node.protocol.ClawdisCanvasCommand import com.steipete.clawdis.node.protocol.ClawdisCanvasCommand
import com.steipete.clawdis.node.voice.VoiceWakeManager import com.steipete.clawdis.node.voice.VoiceWakeManager
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
@@ -268,15 +269,17 @@ class NodeRuntime(context: Context) {
val invokeCommands = val invokeCommands =
buildList { buildList {
add("canvas.show") add(ClawdisCanvasCommand.Show.rawValue)
add("canvas.hide") add(ClawdisCanvasCommand.Hide.rawValue)
add("canvas.setMode") add(ClawdisCanvasCommand.SetMode.rawValue)
add("canvas.navigate") add(ClawdisCanvasCommand.Navigate.rawValue)
add("canvas.eval") add(ClawdisCanvasCommand.Eval.rawValue)
add("canvas.snapshot") add(ClawdisCanvasCommand.Snapshot.rawValue)
add(ClawdisCanvasA2UICommand.Push.rawValue)
add(ClawdisCanvasA2UICommand.Reset.rawValue)
if (cameraEnabled.value) { if (cameraEnabled.value) {
add("camera.snap") add(ClawdisCameraCommand.Snap.rawValue)
add("camera.clip") add(ClawdisCameraCommand.Clip.rawValue)
} }
} }
val resolved = val resolved =
@@ -447,6 +450,7 @@ class NodeRuntime(context: Context) {
private suspend fun handleInvoke(command: String, paramsJson: String?): BridgeSession.InvokeResult { private suspend fun handleInvoke(command: String, paramsJson: String?): BridgeSession.InvokeResult {
if ( if (
command.startsWith(ClawdisCanvasCommand.NamespacePrefix) || command.startsWith(ClawdisCanvasCommand.NamespacePrefix) ||
command.startsWith(ClawdisCanvasA2UICommand.NamespacePrefix) ||
command.startsWith(ClawdisCameraCommand.NamespacePrefix) command.startsWith(ClawdisCameraCommand.NamespacePrefix)
) { ) {
if (!isForeground.value) { if (!isForeground.value) {
@@ -507,6 +511,29 @@ class NodeRuntime(context: Context) {
} }
BridgeSession.InvokeResult.ok("""{"format":"png","base64":"$base64"}""") BridgeSession.InvokeResult.ok("""{"format":"png","base64":"$base64"}""")
} }
ClawdisCanvasA2UICommand.Reset.rawValue -> {
val ready = ensureA2uiReady()
if (!ready) {
return BridgeSession.InvokeResult.error(code = "A2UI_NOT_READY", message = "A2UI not ready")
}
val res = canvas.eval(a2uiResetJS)
BridgeSession.InvokeResult.ok(res)
}
ClawdisCanvasA2UICommand.Push.rawValue, ClawdisCanvasA2UICommand.PushJSONL.rawValue -> {
val messages =
try {
decodeA2uiMessages(command, paramsJson)
} catch (err: Throwable) {
return BridgeSession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload")
}
val ready = ensureA2uiReady()
if (!ready) {
return BridgeSession.InvokeResult.error(code = "A2UI_NOT_READY", message = "A2UI not ready")
}
val js = a2uiApplyMessagesJS(messages)
val res = canvas.eval(js)
BridgeSession.InvokeResult.ok(res)
}
ClawdisCameraCommand.Snap.rawValue -> { ClawdisCameraCommand.Snap.rawValue -> {
val res = camera.snap(paramsJson) val res = camera.snap(paramsJson)
BridgeSession.InvokeResult.ok(res.payloadJson) BridgeSession.InvokeResult.ok(res.payloadJson)
@@ -528,10 +555,128 @@ class NodeRuntime(context: Context) {
) )
} }
} }
private suspend fun ensureA2uiReady(): Boolean {
try {
val already = canvas.eval(a2uiReadyCheckJS)
if (already == "true") return true
} catch (_: Throwable) {
// ignore
}
canvas.navigate(a2uiIndexUrl)
repeat(50) {
try {
val ready = canvas.eval(a2uiReadyCheckJS)
if (ready == "true") return true
} catch (_: Throwable) {
// ignore
}
delay(120)
}
return false
}
private fun decodeA2uiMessages(command: String, paramsJson: String?): String {
val raw = paramsJson?.trim().orEmpty()
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
if (command == ClawdisCanvasA2UICommand.PushJSONL.rawValue) {
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val jsonl = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
val messages =
jsonl
.lineSequence()
.map { it.trim() }
.filter { it.isNotBlank() }
.mapIndexed { idx, line ->
val el = json.parseToJsonElement(line)
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
.toList()
return JsonArray(messages).toString()
}
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
val out =
arr.mapIndexed { idx, el ->
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
return JsonArray(out).toString()
}
private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
if (msg.containsKey("createSurface")) {
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
)
}
val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
val matched = msg.keys.filter { allowed.contains(it) }
if (matched.size != 1) {
val found = msg.keys.sorted().joinToString(", ")
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
)
}
}
} }
private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D) private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
private const val a2uiIndexUrl: String = "file:///android_asset/canvas_a2ui/index.html"
private const val a2uiReadyCheckJS: String =
"""
(() => {
try {
return !!globalThis.clawdisA2UI && typeof globalThis.clawdisA2UI.applyMessages === 'function';
} catch (_) {
return false;
}
})()
"""
private const val a2uiResetJS: String =
"""
(() => {
try {
if (!globalThis.clawdisA2UI) return { ok: false, error: "missing clawdisA2UI" };
return globalThis.clawdisA2UI.reset();
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
"""
private fun a2uiApplyMessagesJS(messagesJson: String): String {
return """
(() => {
try {
if (!globalThis.clawdisA2UI) return { ok: false, error: "missing clawdisA2UI" };
const messages = $messagesJson;
return globalThis.clawdisA2UI.applyMessages(messages);
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
""".trimIndent()
}
private fun String.toJsonString(): String { private fun String.toJsonString(): String {
val escaped = val escaped =
this.replace("\\", "\\\\") this.replace("\\", "\\\\")

63
apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt
View File

@@ -5,6 +5,7 @@ import android.os.Build
import android.graphics.Canvas import android.graphics.Canvas
import android.os.Looper import android.os.Looper
import android.webkit.WebView import android.webkit.WebView
import org.json.JSONObject
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
@@ -31,6 +32,10 @@ class CanvasController {
fun navigate(url: String) { fun navigate(url: String) {
this.url = url this.url = url
if (url.trim().isNotBlank()) {
// `canvas.navigate` is expected to show web content; default to WEB mode to match iOS.
this.mode = Mode.WEB
}
reload() reload()
} }
@@ -100,47 +105,41 @@ class CanvasController {
companion object { companion object {
fun parseMode(paramsJson: String?): Mode { fun parseMode(paramsJson: String?): Mode {
val raw = paramsJson ?: return Mode.CANVAS val obj = parseParamsObject(paramsJson) ?: return Mode.CANVAS
return if (raw.contains("\"web\"")) Mode.WEB else Mode.CANVAS return if (obj.optString("mode", "").equals("web", ignoreCase = true)) {
Mode.WEB
} else {
Mode.CANVAS
}
} }
fun parseNavigateUrl(paramsJson: String?): String? { fun parseNavigateUrl(paramsJson: String?): String? {
val raw = paramsJson ?: return null val obj = parseParamsObject(paramsJson) ?: return null
val key = "\"url\"" val url = obj.optString("url", "").trim()
val idx = raw.indexOf(key) return url.takeIf { it.isNotBlank() }
if (idx < 0) return null
val start = raw.indexOf('"', idx + key.length)
if (start < 0) return null
val end = raw.indexOf('"', start + 1)
if (end < 0) return null
return raw.substring(start + 1, end)
} }
fun parseEvalJs(paramsJson: String?): String? { fun parseEvalJs(paramsJson: String?): String? {
val raw = paramsJson ?: return null val obj = parseParamsObject(paramsJson) ?: return null
val key = "\"javaScript\"" val js = obj.optString("javaScript", "")
val idx = raw.indexOf(key) return js.takeIf { it.isNotBlank() }
if (idx < 0) return null
val start = raw.indexOf('"', idx + key.length)
if (start < 0) return null
val end = raw.lastIndexOf('"')
if (end <= start) return null
return raw.substring(start + 1, end)
.replace("\\n", "\n")
.replace("\\\"", "\"")
.replace("\\\\", "\\")
} }
fun parseSnapshotMaxWidth(paramsJson: String?): Int? { fun parseSnapshotMaxWidth(paramsJson: String?): Int? {
val raw = paramsJson ?: return null val obj = parseParamsObject(paramsJson) ?: return null
val key = "\"maxWidth\"" if (!obj.has("maxWidth")) return null
val idx = raw.indexOf(key) val width = obj.optInt("maxWidth", 0)
if (idx < 0) return null return width.takeIf { it > 0 }
val colon = raw.indexOf(':', idx + key.length) }
if (colon < 0) return null
val tail = raw.substring(colon + 1).trimStart() private fun parseParamsObject(paramsJson: String?): JSONObject? {
val num = tail.takeWhile { it.isDigit() } val raw = paramsJson?.trim() ?: return null
return num.toIntOrNull() if (raw.isBlank()) return null
return try {
JSONObject(raw)
} catch (_: Throwable) {
null
}
} }
} }
} }

11
apps/android/app/src/main/java/com/steipete/clawdis/node/protocol/ClawdisProtocolConstants.kt
View File

@@ -20,6 +20,17 @@ enum class ClawdisCanvasCommand(val rawValue: String) {
} }
} }
enum class ClawdisCanvasA2UICommand(val rawValue: String) {
Push("canvas.a2ui.push"),
PushJSONL("canvas.a2ui.pushJSONL"),
Reset("canvas.a2ui.reset"),
;
companion object {
const val NamespacePrefix: String = "canvas.a2ui."
}
}
enum class ClawdisCameraCommand(val rawValue: String) { enum class ClawdisCameraCommand(val rawValue: String) {
Snap("camera.snap"), Snap("camera.snap"),
Clip("camera.clip"), Clip("camera.clip"),

7
apps/android/app/src/test/java/com/steipete/clawdis/node/protocol/ClawdisProtocolConstantsTest.kt
View File

@@ -14,6 +14,13 @@ class ClawdisProtocolConstantsTest {
assertEquals("canvas.snapshot", ClawdisCanvasCommand.Snapshot.rawValue) assertEquals("canvas.snapshot", ClawdisCanvasCommand.Snapshot.rawValue)
} }
@Test
fun a2uiCommandsUseStableStrings() {
assertEquals("canvas.a2ui.push", ClawdisCanvasA2UICommand.Push.rawValue)
assertEquals("canvas.a2ui.pushJSONL", ClawdisCanvasA2UICommand.PushJSONL.rawValue)
assertEquals("canvas.a2ui.reset", ClawdisCanvasA2UICommand.Reset.rawValue)
}
@Test @Test
fun capabilitiesUseStableStrings() { fun capabilitiesUseStableStrings() {
assertEquals("canvas", ClawdisCapability.Canvas.rawValue) assertEquals("canvas", ClawdisCapability.Canvas.rawValue)

3
docs/android/connect.md
View File

@@ -102,7 +102,8 @@ The Android node’s Chat sheet uses the gateway’s **primary session key** (`m
## 7) Canvas + camera ## 7) Canvas + camera
Canvas commands (foreground only): Canvas commands (foreground only):
- `canvas.eval`, `canvas.snapshot`, `canvas.navigate`, `canvas.setMode` - `canvas.eval`, `canvas.snapshot`, `canvas.navigate` (switches to web mode), `canvas.setMode` (use `"canvas"` to return)
- A2UI: `canvas.a2ui.push`, `canvas.a2ui.reset` (`canvas.a2ui.pushJSONL` legacy alias)
Camera commands (foreground only; permission-gated): Camera commands (foreground only; permission-gated):
- `camera.snap` (jpg) - `camera.snap` (jpg)