diff --git a/apps/android/.gitignore b/apps/android/.gitignore new file mode 100644 index 000000000..68bfc099e --- /dev/null +++ b/apps/android/.gitignore @@ -0,0 +1,5 @@ +.gradle/ +**/build/ +local.properties +.idea/ +**/*.iml diff --git a/apps/android/README.md b/apps/android/README.md new file mode 100644 index 000000000..576f01e7c --- /dev/null +++ b/apps/android/README.md @@ -0,0 +1,10 @@ +## Clawdis Node (Android) (internal) + +Prototype Android “node” app (Iris parity): connects to the Gateway-owned bridge (`_clawdis-bridge._tcp`) over TCP and exposes Canvas + Chat + Camera. + +### Open in Android Studio +- Open the folder `apps/android`. + +### Run +- `./gradlew :app:installDebug` + diff --git a/apps/android/app/build.gradle.kts b/apps/android/app/build.gradle.kts new file mode 100644 index 000000000..5ace327a8 --- /dev/null +++ b/apps/android/app/build.gradle.kts @@ -0,0 +1,79 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + id("org.jetbrains.kotlin.plugin.serialization") +} + +android { + namespace = "com.steipete.clawdis.node" + compileSdk = 34 + + defaultConfig { + applicationId = "com.steipete.clawdis.node" + minSdk = 31 + targetSdk = 34 + versionCode = 1 + versionName = "0.1" + } + + buildTypes { + release { + isMinifyEnabled = false + } + } + + buildFeatures { + compose = true + } + + composeOptions { + kotlinCompilerExtensionVersion = "1.5.14" + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + kotlinOptions { + jvmTarget = "17" + } + + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + } + } +} + +dependencies { + val composeBom = platform("androidx.compose:compose-bom:2024.06.00") + implementation(composeBom) + androidTestImplementation(composeBom) + + implementation("androidx.core:core-ktx:1.13.1") + implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.8.4") + implementation("androidx.activity:activity-compose:1.9.1") + + implementation("androidx.compose.ui:ui") + implementation("androidx.compose.ui:ui-tooling-preview") + implementation("androidx.compose.material3:material3") + implementation("androidx.navigation:navigation-compose:2.7.7") + + debugImplementation("androidx.compose.ui:ui-tooling") + + // Material Components (XML theme + resources) + implementation("com.google.android.material:material:1.12.0") + + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3") + + implementation("androidx.security:security-crypto:1.1.0-alpha06") + + // CameraX (for node.invoke camera.* parity) + implementation("androidx.camera:camera-core:1.3.4") + implementation("androidx.camera:camera-camera2:1.3.4") + implementation("androidx.camera:camera-lifecycle:1.3.4") + implementation("androidx.camera:camera-video:1.3.4") + implementation("androidx.camera:camera-view:1.3.4") +} diff --git a/apps/android/app/src/main/AndroidManifest.xml b/apps/android/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..21bb8c8a4 --- /dev/null +++ b/apps/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/MainActivity.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/MainActivity.kt new file mode 100644 index 000000000..ee940b042 --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/MainActivity.kt @@ -0,0 +1,62 @@ +package com.steipete.clawdis.node + +import android.Manifest +import android.os.Bundle +import android.os.Build +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.activity.viewModels +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.ui.Modifier +import androidx.core.content.ContextCompat +import com.steipete.clawdis.node.ui.RootScreen + +class MainActivity : ComponentActivity() { + private val viewModel: MainViewModel by viewModels() + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + requestDiscoveryPermissionsIfNeeded() + viewModel.camera.attachLifecycleOwner(this) + setContent { + MaterialTheme { + Surface(modifier = Modifier) { + RootScreen(viewModel = viewModel) + } + } + } + } + + override fun onStart() { + super.onStart() + viewModel.setForeground(true) + } + + override fun onStop() { + viewModel.setForeground(false) + super.onStop() + } + + private fun requestDiscoveryPermissionsIfNeeded() { + if (Build.VERSION.SDK_INT >= 33) { + val ok = + ContextCompat.checkSelfPermission( + this, + Manifest.permission.NEARBY_WIFI_DEVICES, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!ok) { + requestPermissions(arrayOf(Manifest.permission.NEARBY_WIFI_DEVICES), 100) + } + } else { + val ok = + ContextCompat.checkSelfPermission( + this, + Manifest.permission.ACCESS_FINE_LOCATION, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!ok) { + requestPermissions(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION), 101) + } + } + } +} diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt new file mode 100644 index 000000000..058ca528b --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt @@ -0,0 +1,403 @@ +package com.steipete.clawdis.node + +import android.app.Application +import androidx.lifecycle.AndroidViewModel +import androidx.lifecycle.viewModelScope +import com.steipete.clawdis.node.bridge.BridgeDiscovery +import com.steipete.clawdis.node.bridge.BridgeEndpoint +import com.steipete.clawdis.node.bridge.BridgePairingClient +import com.steipete.clawdis.node.bridge.BridgeSession +import com.steipete.clawdis.node.node.CameraCaptureManager +import com.steipete.clawdis.node.node.CanvasController +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive + +class MainViewModel(app: Application) : AndroidViewModel(app) { + private val prefs = SecurePrefs(app) + + val canvas = CanvasController() + val camera = CameraCaptureManager(app) + private val json = Json { ignoreUnknownKeys = true } + + private val discovery = BridgeDiscovery(app) + val bridges: StateFlow> = discovery.bridges + + private val _isConnected = MutableStateFlow(false) + val isConnected: StateFlow = _isConnected.asStateFlow() + + private val _statusText = MutableStateFlow("Not connected") + val statusText: StateFlow = _statusText.asStateFlow() + + private val _serverName = MutableStateFlow(null) + val serverName: StateFlow = _serverName.asStateFlow() + + private val _remoteAddress = MutableStateFlow(null) + val remoteAddress: StateFlow = _remoteAddress.asStateFlow() + + private val _isForeground = MutableStateFlow(true) + val isForeground: StateFlow = _isForeground.asStateFlow() + + private val session = + BridgeSession( + scope = viewModelScope, + onConnected = { name, remote -> + _statusText.value = "Connected" + _serverName.value = name + _remoteAddress.value = remote + _isConnected.value = true + }, + onDisconnected = { message -> + _statusText.value = message + _serverName.value = null + _remoteAddress.value = null + _isConnected.value = false + }, + onEvent = { event, payloadJson -> + handleBridgeEvent(event, payloadJson) + }, + onInvoke = { req -> + handleInvoke(req.command, req.paramsJson) + }, + ) + + val instanceId: StateFlow = prefs.instanceId + val displayName: StateFlow = prefs.displayName + val cameraEnabled: StateFlow = prefs.cameraEnabled + val manualEnabled: StateFlow = prefs.manualEnabled + val manualHost: StateFlow = prefs.manualHost + val manualPort: StateFlow = prefs.manualPort + val lastDiscoveredStableId: StateFlow = prefs.lastDiscoveredStableId + + private var didAutoConnect = false + + init { + viewModelScope.launch(Dispatchers.Default) { + bridges.collect { list -> + if (list.isNotEmpty()) { + // Persist the last discovered bridge (best-effort UX parity with iOS). + prefs.setLastDiscoveredStableId(list.last().stableId) + } + + if (didAutoConnect) return@collect + if (_isConnected.value) return@collect + + val token = prefs.loadBridgeToken() + if (token.isNullOrBlank()) return@collect + + if (manualEnabled.value) { + val host = manualHost.value.trim() + val port = manualPort.value + if (host.isNotEmpty() && port in 1..65535) { + didAutoConnect = true + connect(BridgeEndpoint.manual(host = host, port = port)) + } + return@collect + } + + val targetStableId = lastDiscoveredStableId.value.trim() + if (targetStableId.isEmpty()) return@collect + val target = list.firstOrNull { it.stableId == targetStableId } ?: return@collect + didAutoConnect = true + connect(target) + } + } + } + + fun setForeground(value: Boolean) { + _isForeground.value = value + } + + fun setDisplayName(value: String) { + prefs.setDisplayName(value) + } + + fun setCameraEnabled(value: Boolean) { + prefs.setCameraEnabled(value) + } + + fun setManualEnabled(value: Boolean) { + prefs.setManualEnabled(value) + } + + fun setManualHost(value: String) { + prefs.setManualHost(value) + } + + fun setManualPort(value: Int) { + prefs.setManualPort(value) + } + + fun connect(endpoint: BridgeEndpoint) { + viewModelScope.launch(Dispatchers.IO) { + _statusText.value = "Connecting…" + val token = prefs.loadBridgeToken() + val resolved = + if (token.isNullOrBlank()) { + _statusText.value = "Pairing…" + BridgePairingClient().pairAndHello( + endpoint = endpoint, + hello = BridgePairingClient.Hello( + nodeId = instanceId.value, + displayName = displayName.value, + token = null, + platform = "Android", + version = "dev", + ), + ) + } else { + BridgePairingClient.PairResult(ok = true, token = token.trim()) + } + + if (!resolved.ok || resolved.token.isNullOrBlank()) { + _statusText.value = "Failed: pairing required" + return@launch + } + + prefs.saveBridgeToken(resolved.token!!) + session.connect( + endpoint = endpoint, + hello = BridgeSession.Hello( + nodeId = instanceId.value, + displayName = displayName.value, + token = resolved.token, + platform = "Android", + version = "dev", + ), + ) + } + } + + fun connectManual() { + val host = manualHost.value.trim() + val port = manualPort.value + if (host.isEmpty() || port <= 0 || port > 65535) { + _statusText.value = "Failed: invalid manual host/port" + return + } + connect(BridgeEndpoint.manual(host = host, port = port)) + } + + fun disconnect() { + session.disconnect() + } + + data class ChatMessage(val id: String, val role: String, val text: String, val timestampMs: Long?) + + private val _chatMessages = MutableStateFlow>(emptyList()) + val chatMessages: StateFlow> = _chatMessages.asStateFlow() + + private val _chatError = MutableStateFlow(null) + val chatError: StateFlow = _chatError.asStateFlow() + + private val pendingRuns = mutableSetOf() + private val _pendingRunCount = MutableStateFlow(0) + val pendingRunCount: StateFlow = _pendingRunCount.asStateFlow() + + fun loadChat(sessionKey: String = "main") { + viewModelScope.launch(Dispatchers.IO) { + _chatError.value = null + try { + // Best-effort; push events are optional, but improve latency. + session.sendEvent("chat.subscribe", """{"sessionKey":"$sessionKey"}""") + } catch (_: Throwable) { + // ignore + } + + try { + val res = session.request("chat.history", """{"sessionKey":"$sessionKey"}""") + _chatMessages.value = parseHistory(res) + } catch (e: Exception) { + _chatError.value = e.message + } + } + } + + fun sendChat(sessionKey: String = "main", message: String, thinking: String = "off") { + val trimmed = message.trim() + if (trimmed.isEmpty()) return + viewModelScope.launch(Dispatchers.IO) { + _chatError.value = null + val idem = java.util.UUID.randomUUID().toString() + + _chatMessages.value = + _chatMessages.value + + ChatMessage( + id = java.util.UUID.randomUUID().toString(), + role = "user", + text = trimmed, + timestampMs = System.currentTimeMillis(), + ) + + try { + val params = + """{"sessionKey":"$sessionKey","message":${trimmed.toJsonString()},"thinking":"$thinking","timeoutMs":30000,"idempotencyKey":"$idem"}""" + val res = session.request("chat.send", params) + val runId = parseRunId(res) ?: idem + pendingRuns.add(runId) + _pendingRunCount.value = pendingRuns.size + } catch (e: Exception) { + _chatError.value = e.message + } + } + } + + private fun handleBridgeEvent(event: String, payloadJson: String?) { + if (event != "chat" || payloadJson.isNullOrBlank()) return + try { + val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return + val state = payload["state"].asStringOrNull() + val runId = payload["runId"].asStringOrNull() + if (!runId.isNullOrBlank()) { + pendingRuns.remove(runId) + _pendingRunCount.value = pendingRuns.size + } + + when (state) { + "final" -> { + val msgObj = payload["message"].asObjectOrNull() + val role = msgObj?.get("role").asStringOrNull() ?: "assistant" + val text = extractTextFromMessage(msgObj) + if (!text.isNullOrBlank()) { + _chatMessages.value = + _chatMessages.value + + ChatMessage( + id = java.util.UUID.randomUUID().toString(), + role = role, + text = text, + timestampMs = System.currentTimeMillis(), + ) + } + } + "error" -> { + _chatError.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed" + } + } + } catch (_: Throwable) { + // ignore + } + } + + private fun parseHistory(historyJson: String): List { + val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return emptyList() + val raw = root["messages"] ?: return emptyList() + val array = raw as? JsonArray ?: return emptyList() + return array.mapNotNull { item -> + val obj = item as? JsonObject ?: return@mapNotNull null + val role = obj["role"].asStringOrNull() ?: return@mapNotNull null + val text = extractTextFromMessage(obj) ?: return@mapNotNull null + ChatMessage( + id = java.util.UUID.randomUUID().toString(), + role = role, + text = text, + timestampMs = null, + ) + } + } + + private fun extractTextFromMessage(msgObj: JsonObject?): String? { + if (msgObj == null) return null + val content = msgObj["content"] ?: return null + return when (content) { + is JsonPrimitive -> content.asStringOrNull() + else -> { + val arr = (content as? JsonArray) ?: return null + arr.mapNotNull { part -> + val p = part as? JsonObject ?: return@mapNotNull null + p["text"].asStringOrNull() + }.joinToString("\n").trim().ifBlank { null } + } + } + } + + private fun parseRunId(resJson: String): String? { + return try { + json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull() + } catch (_: Throwable) { + null + } + } + + private suspend fun handleInvoke(command: String, paramsJson: String?): BridgeSession.InvokeResult { + if ((command.startsWith("screen.") || command.startsWith("camera.")) && !isForeground.value) { + return BridgeSession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: screen/camera commands require foreground", + ) + } + if (command.startsWith("camera.") && !cameraEnabled.value) { + return BridgeSession.InvokeResult.error( + code = "CAMERA_DISABLED", + message = "CAMERA_DISABLED: enable Camera in Settings", + ) + } + + return when (command) { + "screen.show" -> BridgeSession.InvokeResult.ok(null) + "screen.hide" -> BridgeSession.InvokeResult.ok(null) + "screen.setMode" -> { + val mode = CanvasController.parseMode(paramsJson) + canvas.setMode(mode) + BridgeSession.InvokeResult.ok(null) + } + "screen.navigate" -> { + val url = CanvasController.parseNavigateUrl(paramsJson) + if (url != null) canvas.navigate(url) + BridgeSession.InvokeResult.ok(null) + } + "screen.eval" -> { + val js = CanvasController.parseEvalJs(paramsJson) ?: return BridgeSession.InvokeResult.error( + code = "INVALID_REQUEST", + message = "INVALID_REQUEST: javaScript required", + ) + val result = canvas.eval(js) + BridgeSession.InvokeResult.ok("""{"result":${result.toJsonString()}}""") + } + "screen.snapshot" -> { + val maxWidth = CanvasController.parseSnapshotMaxWidth(paramsJson) + val base64 = canvas.snapshotPngBase64(maxWidth = maxWidth) + BridgeSession.InvokeResult.ok("""{"format":"png","base64":"$base64"}""") + } + "camera.snap" -> { + val res = camera.snap(paramsJson) + BridgeSession.InvokeResult.ok(res.payloadJson) + } + "camera.clip" -> { + val res = camera.clip(paramsJson) + BridgeSession.InvokeResult.ok(res.payloadJson) + } + else -> + BridgeSession.InvokeResult.error( + code = "INVALID_REQUEST", + message = "INVALID_REQUEST: unknown command", + ) + } + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun String.toJsonString(): String { + val escaped = + this.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + return "\"$escaped\"" +} diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/SecurePrefs.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/SecurePrefs.kt new file mode 100644 index 000000000..67abd6aca --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/SecurePrefs.kt @@ -0,0 +1,97 @@ +package com.steipete.clawdis.node + +import android.content.Context +import androidx.security.crypto.EncryptedSharedPreferences +import androidx.security.crypto.MasterKey +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import java.util.UUID + +class SecurePrefs(context: Context) { + private val masterKey = + MasterKey.Builder(context) + .setKeyScheme(MasterKey.KeyScheme.AES256_GCM) + .build() + + private val prefs = + EncryptedSharedPreferences.create( + context, + "clawdis.node.secure", + masterKey, + EncryptedSharedPreferences.PrefKeyEncryptionScheme.AES256_SIV, + EncryptedSharedPreferences.PrefValueEncryptionScheme.AES256_GCM, + ) + + private val _instanceId = MutableStateFlow(loadOrCreateInstanceId()) + val instanceId: StateFlow = _instanceId + + private val _displayName = MutableStateFlow(prefs.getString("node.displayName", "Android Node")!!) + val displayName: StateFlow = _displayName + + private val _cameraEnabled = MutableStateFlow(prefs.getBoolean("camera.enabled", true)) + val cameraEnabled: StateFlow = _cameraEnabled + + private val _manualEnabled = MutableStateFlow(prefs.getBoolean("bridge.manual.enabled", false)) + val manualEnabled: StateFlow = _manualEnabled + + private val _manualHost = MutableStateFlow(prefs.getString("bridge.manual.host", "")!!) + val manualHost: StateFlow = _manualHost + + private val _manualPort = MutableStateFlow(prefs.getInt("bridge.manual.port", 18790)) + val manualPort: StateFlow = _manualPort + + private val _lastDiscoveredStableId = + MutableStateFlow(prefs.getString("bridge.lastDiscoveredStableId", "")!!) + val lastDiscoveredStableId: StateFlow = _lastDiscoveredStableId + + fun setLastDiscoveredStableId(value: String) { + val trimmed = value.trim() + prefs.edit().putString("bridge.lastDiscoveredStableId", trimmed).apply() + _lastDiscoveredStableId.value = trimmed + } + + fun setDisplayName(value: String) { + val trimmed = value.trim() + prefs.edit().putString("node.displayName", trimmed).apply() + _displayName.value = trimmed + } + + fun setCameraEnabled(value: Boolean) { + prefs.edit().putBoolean("camera.enabled", value).apply() + _cameraEnabled.value = value + } + + fun setManualEnabled(value: Boolean) { + prefs.edit().putBoolean("bridge.manual.enabled", value).apply() + _manualEnabled.value = value + } + + fun setManualHost(value: String) { + val trimmed = value.trim() + prefs.edit().putString("bridge.manual.host", trimmed).apply() + _manualHost.value = trimmed + } + + fun setManualPort(value: Int) { + prefs.edit().putInt("bridge.manual.port", value).apply() + _manualPort.value = value + } + + fun loadBridgeToken(): String? { + val key = "bridge.token.${_instanceId.value}" + return prefs.getString(key, null) + } + + fun saveBridgeToken(token: String) { + val key = "bridge.token.${_instanceId.value}" + prefs.edit().putString(key, token.trim()).apply() + } + + private fun loadOrCreateInstanceId(): String { + val existing = prefs.getString("node.instanceId", null)?.trim() + if (!existing.isNullOrBlank()) return existing + val fresh = UUID.randomUUID().toString() + prefs.edit().putString("node.instanceId", fresh).apply() + return fresh + } +} diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeDiscovery.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeDiscovery.kt new file mode 100644 index 000000000..3d4307885 --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeDiscovery.kt @@ -0,0 +1,88 @@ +package com.steipete.clawdis.node.bridge + +import android.content.Context +import android.net.nsd.NsdManager +import android.net.nsd.NsdServiceInfo +import android.os.Build +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import java.util.concurrent.ConcurrentHashMap + +class BridgeDiscovery(context: Context) { + private val nsd = context.getSystemService(NsdManager::class.java) + private val serviceType = "_clawdis-bridge._tcp." + + private val byId = ConcurrentHashMap() + private val _bridges = MutableStateFlow>(emptyList()) + val bridges: StateFlow> = _bridges.asStateFlow() + + private val discoveryListener = + object : NsdManager.DiscoveryListener { + override fun onStartDiscoveryFailed(serviceType: String, errorCode: Int) {} + override fun onStopDiscoveryFailed(serviceType: String, errorCode: Int) {} + override fun onDiscoveryStarted(serviceType: String) {} + override fun onDiscoveryStopped(serviceType: String) {} + + override fun onServiceFound(serviceInfo: NsdServiceInfo) { + if (serviceInfo.serviceType != this@BridgeDiscovery.serviceType) return + resolve(serviceInfo) + } + + override fun onServiceLost(serviceInfo: NsdServiceInfo) { + val id = stableId(serviceInfo) + byId.remove(id) + publish() + } + } + + init { + try { + nsd.discoverServices(serviceType, NsdManager.PROTOCOL_DNS_SD, discoveryListener) + } catch (_: Throwable) { + // ignore (best-effort) + } + } + + private fun resolve(serviceInfo: NsdServiceInfo) { + nsd.resolveService( + serviceInfo, + object : NsdManager.ResolveListener { + override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {} + + override fun onServiceResolved(resolved: NsdServiceInfo) { + val host = resolved.host?.hostAddress ?: return + val port = resolved.port + if (port <= 0) return + + val displayName = txt(resolved, "displayName") ?: resolved.serviceName + val id = stableId(resolved) + byId[id] = BridgeEndpoint(stableId = id, name = displayName, host = host, port = port) + publish() + } + }, + ) + } + + private fun publish() { + _bridges.value = byId.values.sortedBy { it.name.lowercase() } + } + + private fun stableId(info: NsdServiceInfo): String { + return "${info.serviceType}|local.|${normalizeName(info.serviceName)}" + } + + private fun normalizeName(raw: String): String { + return raw.trim().split(Regex("\\s+")).joinToString(" ") + } + + private fun txt(info: NsdServiceInfo, key: String): String? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) return null + val bytes = info.attributes[key] ?: return null + return try { + String(bytes, Charsets.UTF_8).trim().ifEmpty { null } + } catch (_: Throwable) { + null + } + } +} diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeEndpoint.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeEndpoint.kt new file mode 100644 index 000000000..bd359e470 --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeEndpoint.kt @@ -0,0 +1,19 @@ +package com.steipete.clawdis.node.bridge + +data class BridgeEndpoint( + val stableId: String, + val name: String, + val host: String, + val port: Int, +) { + companion object { + fun manual(host: String, port: Int): BridgeEndpoint = + BridgeEndpoint( + stableId = "manual|$host|$port", + name = "$host:$port", + host = host, + port = port, + ) + } +} + diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgePairingClient.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgePairingClient.kt new file mode 100644 index 000000000..a52aa5dbd --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgePairingClient.kt @@ -0,0 +1,118 @@ +package com.steipete.clawdis.node.bridge + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.buildJsonObject +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.InputStreamReader +import java.io.OutputStreamWriter +import java.net.InetSocketAddress +import java.net.Socket + +class BridgePairingClient { + private val json = Json { ignoreUnknownKeys = true } + + data class Hello( + val nodeId: String, + val displayName: String?, + val token: String?, + val platform: String?, + val version: String?, + ) + + data class PairResult(val ok: Boolean, val token: String?, val error: String? = null) + + suspend fun pairAndHello(endpoint: BridgeEndpoint, hello: Hello): PairResult = + withContext(Dispatchers.IO) { + val socket = Socket() + socket.tcpNoDelay = true + socket.connect(InetSocketAddress(endpoint.host, endpoint.port), 8_000) + socket.soTimeout = 60_000 + + val reader = BufferedReader(InputStreamReader(socket.getInputStream(), Charsets.UTF_8)) + val writer = BufferedWriter(OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8)) + + fun send(line: String) { + writer.write(line) + writer.write("\n") + writer.flush() + } + + fun sendJson(obj: JsonObject) = send(obj.toString()) + + try { + sendJson( + buildJsonObject { + put("type", JsonPrimitive("hello")) + put("nodeId", JsonPrimitive(hello.nodeId)) + hello.displayName?.let { put("displayName", JsonPrimitive(it)) } + hello.token?.let { put("token", JsonPrimitive(it)) } + hello.platform?.let { put("platform", JsonPrimitive(it)) } + hello.version?.let { put("version", JsonPrimitive(it)) } + }, + ) + + val firstObj = json.parseToJsonElement(reader.readLine()).asObjectOrNull() + ?: return@withContext PairResult(ok = false, token = null, error = "unexpected bridge response") + when (firstObj["type"].asStringOrNull()) { + "hello-ok" -> PairResult(ok = true, token = hello.token) + "error" -> { + val code = firstObj["code"].asStringOrNull() ?: "UNAVAILABLE" + val message = firstObj["message"].asStringOrNull() ?: "pairing required" + if (code != "NOT_PAIRED" && code != "UNAUTHORIZED") { + return@withContext PairResult(ok = false, token = null, error = "$code: $message") + } + + sendJson( + buildJsonObject { + put("type", JsonPrimitive("pair-request")) + put("nodeId", JsonPrimitive(hello.nodeId)) + hello.displayName?.let { put("displayName", JsonPrimitive(it)) } + hello.platform?.let { put("platform", JsonPrimitive(it)) } + hello.version?.let { put("version", JsonPrimitive(it)) } + }, + ) + + while (true) { + val nextLine = reader.readLine() ?: break + val next = json.parseToJsonElement(nextLine).asObjectOrNull() ?: continue + when (next["type"].asStringOrNull()) { + "pair-ok" -> { + val token = next["token"].asStringOrNull() + return@withContext PairResult(ok = !token.isNullOrBlank(), token = token) + } + "error" -> { + val c = next["code"].asStringOrNull() ?: "UNAVAILABLE" + val m = next["message"].asStringOrNull() ?: "pairing failed" + return@withContext PairResult(ok = false, token = null, error = "$c: $m") + } + } + } + PairResult(ok = false, token = null, error = "pairing failed") + } + else -> PairResult(ok = false, token = null, error = "unexpected bridge response") + } + } finally { + try { + socket.close() + } catch (_: Throwable) { + // ignore + } + } + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeSession.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeSession.kt new file mode 100644 index 000000000..9f949f9d8 --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeSession.kt @@ -0,0 +1,303 @@ +package com.steipete.clawdis.node.bridge + +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.InputStreamReader +import java.io.OutputStreamWriter +import java.net.InetSocketAddress +import java.net.Socket +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap + +class BridgeSession( + private val scope: CoroutineScope, + private val onConnected: (serverName: String, remoteAddress: String?) -> Unit, + private val onDisconnected: (message: String) -> Unit, + private val onEvent: (event: String, payloadJson: String?) -> Unit, + private val onInvoke: suspend (InvokeRequest) -> InvokeResult, +) { + data class Hello( + val nodeId: String, + val displayName: String?, + val token: String?, + val platform: String?, + val version: String?, + ) + + data class InvokeRequest(val id: String, val command: String, val paramsJson: String?) + + data class InvokeResult(val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) { + companion object { + fun ok(payloadJson: String?) = InvokeResult(ok = true, payloadJson = payloadJson, error = null) + fun error(code: String, message: String) = + InvokeResult(ok = false, payloadJson = null, error = ErrorShape(code = code, message = message)) + } + } + + data class ErrorShape(val code: String, val message: String) + + private val json = Json { ignoreUnknownKeys = true } + private val writeLock = Mutex() + private val pending = ConcurrentHashMap>() + + private var desired: Pair? = null + private var job: Job? = null + + fun connect(endpoint: BridgeEndpoint, hello: Hello) { + desired = endpoint to hello + if (job == null) { + job = scope.launch(Dispatchers.IO) { runLoop() } + } + } + + fun disconnect() { + desired = null + scope.launch(Dispatchers.IO) { + job?.cancelAndJoin() + job = null + onDisconnected("Disconnected") + } + } + + suspend fun sendEvent(event: String, payloadJson: String?) { + val conn = currentConnection ?: return + conn.sendJson( + buildJsonObject { + put("type", JsonPrimitive("event")) + put("event", JsonPrimitive(event)) + if (payloadJson != null) put("payloadJSON", JsonPrimitive(payloadJson)) else put("payloadJSON", JsonNull) + }, + ) + } + + suspend fun request(method: String, paramsJson: String?): String { + val conn = currentConnection ?: throw IllegalStateException("not connected") + val id = UUID.randomUUID().toString() + val deferred = CompletableDeferred() + pending[id] = deferred + conn.sendJson( + buildJsonObject { + put("type", JsonPrimitive("req")) + put("id", JsonPrimitive(id)) + put("method", JsonPrimitive(method)) + if (paramsJson != null) put("paramsJSON", JsonPrimitive(paramsJson)) else put("paramsJSON", JsonNull) + }, + ) + val res = deferred.await() + if (res.ok) return res.payloadJson ?: "" + val err = res.error + throw IllegalStateException("${err?.code ?: "UNAVAILABLE"}: ${err?.message ?: "request failed"}") + } + + private data class RpcResponse(val id: String, val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) + + private class Connection(private val socket: Socket, private val reader: BufferedReader, private val writer: BufferedWriter, private val writeLock: Mutex) { + val remoteAddress: String? = + socket.inetAddress?.hostAddress?.takeIf { it.isNotBlank() }?.let { "${it}:${socket.port}" } + + suspend fun sendJson(obj: JsonObject) { + writeLock.withLock { + writer.write(obj.toString()) + writer.write("\n") + writer.flush() + } + } + + fun closeQuietly() { + try { + socket.close() + } catch (_: Throwable) { + // ignore + } + } + } + + @Volatile private var currentConnection: Connection? = null + + private suspend fun runLoop() { + var attempt = 0 + while (scope.isActive) { + val target = desired + if (target == null) { + currentConnection?.closeQuietly() + currentConnection = null + delay(250) + continue + } + + val (endpoint, hello) = target + try { + onDisconnected(if (attempt == 0) "Connecting…" else "Reconnecting…") + connectOnce(endpoint, hello) + attempt = 0 + } catch (err: Throwable) { + attempt += 1 + onDisconnected("Bridge error: ${err.message ?: err::class.java.simpleName}") + val sleepMs = minOf(8_000L, (350.0 * Math.pow(1.7, attempt.toDouble())).toLong()) + delay(sleepMs) + } + } + } + + private fun invokeErrorFromThrowable(err: Throwable): InvokeResult { + val msg = err.message?.trim().takeIf { !it.isNullOrEmpty() } ?: err::class.java.simpleName + val parts = msg.split(":", limit = 2) + if (parts.size == 2) { + val code = parts[0].trim() + val rest = parts[1].trim() + if (code.isNotEmpty() && code.all { it.isUpperCase() || it == '_' }) { + return InvokeResult.error(code = code, message = rest.ifEmpty { msg }) + } + } + return InvokeResult.error(code = "UNAVAILABLE", message = msg) + } + + private suspend fun connectOnce(endpoint: BridgeEndpoint, hello: Hello) = + withContext(Dispatchers.IO) { + val socket = Socket() + socket.tcpNoDelay = true + socket.connect(InetSocketAddress(endpoint.host, endpoint.port), 8_000) + socket.soTimeout = 0 + + val reader = BufferedReader(InputStreamReader(socket.getInputStream(), Charsets.UTF_8)) + val writer = BufferedWriter(OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8)) + + val conn = Connection(socket, reader, writer, writeLock) + currentConnection = conn + + try { + conn.sendJson( + buildJsonObject { + put("type", JsonPrimitive("hello")) + put("nodeId", JsonPrimitive(hello.nodeId)) + hello.displayName?.let { put("displayName", JsonPrimitive(it)) } + hello.token?.let { put("token", JsonPrimitive(it)) } + hello.platform?.let { put("platform", JsonPrimitive(it)) } + hello.version?.let { put("version", JsonPrimitive(it)) } + }, + ) + + val firstLine = reader.readLine() ?: throw IllegalStateException("bridge closed connection") + val first = json.parseToJsonElement(firstLine).asObjectOrNull() + ?: throw IllegalStateException("unexpected bridge response") + when (first["type"].asStringOrNull()) { + "hello-ok" -> { + val name = first["serverName"].asStringOrNull() ?: "Bridge" + onConnected(name, conn.remoteAddress) + } + "error" -> { + val code = first["code"].asStringOrNull() ?: "UNAVAILABLE" + val msg = first["message"].asStringOrNull() ?: "connect failed" + throw IllegalStateException("$code: $msg") + } + else -> throw IllegalStateException("unexpected bridge response") + } + + while (scope.isActive) { + val line = reader.readLine() ?: break + val frame = json.parseToJsonElement(line).asObjectOrNull() ?: continue + when (frame["type"].asStringOrNull()) { + "event" -> { + val event = frame["event"].asStringOrNull() ?: return@withContext + val payload = frame["payloadJSON"].asStringOrNull() + onEvent(event, payload) + } + "ping" -> { + val id = frame["id"].asStringOrNull() ?: "" + conn.sendJson(buildJsonObject { put("type", JsonPrimitive("pong")); put("id", JsonPrimitive(id)) }) + } + "res" -> { + val id = frame["id"].asStringOrNull() ?: continue + val ok = frame["ok"].asBooleanOrNull() ?: false + val payloadJson = frame["payloadJSON"].asStringOrNull() + val error = + frame["error"]?.let { + val obj = it.asObjectOrNull() ?: return@let null + val code = obj["code"].asStringOrNull() ?: "UNAVAILABLE" + val msg = obj["message"].asStringOrNull() ?: "request failed" + ErrorShape(code, msg) + } + pending.remove(id)?.complete(RpcResponse(id, ok, payloadJson, error)) + } + "invoke" -> { + val id = frame["id"].asStringOrNull() ?: continue + val command = frame["command"].asStringOrNull() ?: "" + val params = frame["paramsJSON"].asStringOrNull() + val result = + try { + onInvoke(InvokeRequest(id, command, params)) + } catch (err: Throwable) { + invokeErrorFromThrowable(err) + } + conn.sendJson( + buildJsonObject { + put("type", JsonPrimitive("invoke-res")) + put("id", JsonPrimitive(id)) + put("ok", JsonPrimitive(result.ok)) + if (result.payloadJson != null) put("payloadJSON", JsonPrimitive(result.payloadJson)) + if (result.error != null) { + put( + "error", + buildJsonObject { + put("code", JsonPrimitive(result.error.code)) + put("message", JsonPrimitive(result.error.message)) + }, + ) + } + }, + ) + } + "invoke-res" -> { + // gateway->node only (ignore) + } + } + } + } finally { + currentConnection = null + for ((_, waiter) in pending) { + waiter.cancel() + } + pending.clear() + conn.closeQuietly() + } + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun JsonElement?.asBooleanOrNull(): Boolean? = + when (this) { + is JsonPrimitive -> { + val c = content.trim() + when { + c.equals("true", ignoreCase = true) -> true + c.equals("false", ignoreCase = true) -> false + else -> null + } + } + else -> null + } diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CameraCaptureManager.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CameraCaptureManager.kt new file mode 100644 index 000000000..d80732f36 --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CameraCaptureManager.kt @@ -0,0 +1,235 @@ +package com.steipete.clawdis.node.node + +import android.Manifest +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.util.Base64 +import android.content.pm.PackageManager +import androidx.lifecycle.LifecycleOwner +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageCapture +import androidx.camera.core.ImageCaptureException +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.video.FileOutputOptions +import androidx.camera.video.Recorder +import androidx.camera.video.Recording +import androidx.camera.video.VideoCapture +import androidx.camera.video.VideoRecordEvent +import androidx.core.content.ContextCompat +import androidx.core.content.ContextCompat.checkSelfPermission +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.concurrent.Executor +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +class CameraCaptureManager(private val context: Context) { + data class Payload(val payloadJson: String) + + @Volatile private var lifecycleOwner: LifecycleOwner? = null + + fun attachLifecycleOwner(owner: LifecycleOwner) { + lifecycleOwner = owner + } + + private fun requireCameraPermission() { + val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED + if (!granted) throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") + } + + private fun requireMicPermission() { + val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED + if (!granted) throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + } + + suspend fun snap(paramsJson: String?): Payload = + withContext(Dispatchers.Main) { + requireCameraPermission() + val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") + val facing = parseFacing(paramsJson) ?: "front" + val quality = (parseQuality(paramsJson) ?: 0.9).coerceIn(0.1, 1.0) + val maxWidth = parseMaxWidth(paramsJson) + + val provider = context.cameraProvider() + val capture = ImageCapture.Builder().build() + val selector = + if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA + + provider.unbindAll() + provider.bindToLifecycle(owner, selector, capture) + + val bytes = capture.takeJpegBytes(context.mainExecutor()) + val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) + ?: throw IllegalStateException("UNAVAILABLE: failed to decode captured image") + val scaled = + if (maxWidth != null && maxWidth > 0 && decoded.width > maxWidth) { + val h = + (decoded.height.toDouble() * (maxWidth.toDouble() / decoded.width.toDouble())) + .toInt() + .coerceAtLeast(1) + Bitmap.createScaledBitmap(decoded, maxWidth, h, true) + } else { + decoded + } + + val out = ByteArrayOutputStream() + val jpegQuality = (quality * 100.0).toInt().coerceIn(10, 100) + if (!scaled.compress(Bitmap.CompressFormat.JPEG, jpegQuality, out)) { + throw IllegalStateException("UNAVAILABLE: failed to encode JPEG") + } + val base64 = Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) + Payload( + """{"format":"jpg","base64":"$base64","width":${scaled.width},"height":${scaled.height}}""", + ) + } + + suspend fun clip(paramsJson: String?): Payload = + withContext(Dispatchers.Main) { + requireCameraPermission() + val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") + val facing = parseFacing(paramsJson) ?: "front" + val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 45_000) + val includeAudio = parseIncludeAudio(paramsJson) ?: true + if (includeAudio) requireMicPermission() + + val provider = context.cameraProvider() + val recorder = Recorder.Builder().build() + val videoCapture = VideoCapture.withOutput(recorder) + val selector = + if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA + + provider.unbindAll() + provider.bindToLifecycle(owner, selector, videoCapture) + + val file = File.createTempFile("clawdis-clip-", ".mp4") + val outputOptions = FileOutputOptions.Builder(file).build() + + val finalized = kotlinx.coroutines.CompletableDeferred() + val recording: Recording = + videoCapture.output + .prepareRecording(context, outputOptions) + .apply { + if (includeAudio) withAudioEnabled() + } + .start(context.mainExecutor()) { event -> + if (event is VideoRecordEvent.Finalize) { + finalized.complete(event) + } + } + + try { + kotlinx.coroutines.delay(durationMs.toLong()) + } finally { + recording.stop() + } + + val finalizeEvent = + try { + withTimeout(10_000) { finalized.await() } + } catch (err: Throwable) { + file.delete() + throw IllegalStateException("UNAVAILABLE: camera clip finalize timed out") + } + if (finalizeEvent.hasError()) { + file.delete() + throw IllegalStateException("UNAVAILABLE: camera clip failed") + } + + val bytes = file.readBytes() + file.delete() + val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) + Payload( + """{"format":"mp4","base64":"$base64","durationMs":$durationMs,"hasAudio":${includeAudio}}""", + ) + } + + private fun parseFacing(paramsJson: String?): String? = + when { + paramsJson?.contains("\"front\"") == true -> "front" + paramsJson?.contains("\"back\"") == true -> "back" + else -> null + } + + private fun parseQuality(paramsJson: String?): Double? = + parseNumber(paramsJson, key = "quality")?.toDoubleOrNull() + + private fun parseMaxWidth(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "maxWidth")?.toIntOrNull() + + private fun parseDurationMs(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "durationMs")?.toIntOrNull() + + private fun parseIncludeAudio(paramsJson: String?): Boolean? { + val raw = paramsJson ?: return null + val key = "\"includeAudio\"" + val idx = raw.indexOf(key) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + key.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return when { + tail.startsWith("true") -> true + tail.startsWith("false") -> false + else -> null + } + } + + private fun parseNumber(paramsJson: String?, key: String): String? { + val raw = paramsJson ?: return null + val needle = "\"$key\"" + val idx = raw.indexOf(needle) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + needle.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return tail.takeWhile { it.isDigit() || it == '.' } + } + + private fun Context.mainExecutor(): Executor = ContextCompat.getMainExecutor(this) +} + +private suspend fun Context.cameraProvider(): ProcessCameraProvider = + suspendCancellableCoroutine { cont -> + val future = ProcessCameraProvider.getInstance(this) + future.addListener( + { + try { + cont.resume(future.get()) + } catch (e: Exception) { + cont.resumeWithException(e) + } + }, + ContextCompat.getMainExecutor(this), + ) + } + +private suspend fun ImageCapture.takeJpegBytes(executor: Executor): ByteArray = + suspendCancellableCoroutine { cont -> + val file = File.createTempFile("clawdis-snap-", ".jpg") + val options = ImageCapture.OutputFileOptions.Builder(file).build() + takePicture( + options, + executor, + object : ImageCapture.OnImageSavedCallback { + override fun onError(exception: ImageCaptureException) { + cont.resumeWithException(exception) + } + + override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) { + try { + val bytes = file.readBytes() + cont.resume(bytes) + } catch (e: Exception) { + cont.resumeWithException(e) + } finally { + file.delete() + } + } + }, + ) + } diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt new file mode 100644 index 000000000..1155c489b --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt @@ -0,0 +1,242 @@ +package com.steipete.clawdis.node.node + +import android.graphics.Bitmap +import android.os.Build +import android.graphics.Canvas +import android.webkit.WebView +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import android.util.Base64 +import kotlin.coroutines.resume + +class CanvasController { + enum class Mode { CANVAS, WEB } + + @Volatile private var webView: WebView? = null + @Volatile private var mode: Mode = Mode.CANVAS + @Volatile private var url: String = "" + + fun attach(webView: WebView) { + this.webView = webView + reload() + } + + fun setMode(mode: Mode) { + this.mode = mode + reload() + } + + fun navigate(url: String) { + this.url = url + reload() + } + + private fun reload() { + val wv = webView ?: return + when (mode) { + Mode.WEB -> wv.loadUrl(url.trim()) + Mode.CANVAS -> wv.loadDataWithBaseURL(null, canvasHtml, "text/html", "utf-8", null) + } + } + + suspend fun eval(javaScript: String): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + suspendCancellableCoroutine { cont -> + wv.evaluateJavascript(javaScript) { result -> + cont.resume(result ?: "") + } + } + } + + suspend fun snapshotPngBase64(maxWidth: Int?): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + val bmp = wv.captureBitmap() + val scaled = + if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { + val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) + Bitmap.createScaledBitmap(bmp, maxWidth, h, true) + } else { + bmp + } + + val out = ByteArrayOutputStream() + scaled.compress(Bitmap.CompressFormat.PNG, 100, out) + Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) + } + + private suspend fun WebView.captureBitmap(): Bitmap = + suspendCancellableCoroutine { cont -> + val width = width.coerceAtLeast(1) + val height = height.coerceAtLeast(1) + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + + // WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable + // cross-version snapshot for this lightweight "canvas" use-case. + draw(Canvas(bitmap)) + cont.resume(bitmap) + } + + companion object { + fun parseMode(paramsJson: String?): Mode { + val raw = paramsJson ?: return Mode.CANVAS + return if (raw.contains("\"web\"")) Mode.WEB else Mode.CANVAS + } + + fun parseNavigateUrl(paramsJson: String?): String? { + val raw = paramsJson ?: return null + val key = "\"url\"" + val idx = raw.indexOf(key) + if (idx < 0) return null + val start = raw.indexOf('"', idx + key.length) + if (start < 0) return null + val end = raw.indexOf('"', start + 1) + if (end < 0) return null + return raw.substring(start + 1, end) + } + + fun parseEvalJs(paramsJson: String?): String? { + val raw = paramsJson ?: return null + val key = "\"javaScript\"" + val idx = raw.indexOf(key) + if (idx < 0) return null + val start = raw.indexOf('"', idx + key.length) + if (start < 0) return null + val end = raw.lastIndexOf('"') + if (end <= start) return null + return raw.substring(start + 1, end) + .replace("\\n", "\n") + .replace("\\\"", "\"") + .replace("\\\\", "\\") + } + + fun parseSnapshotMaxWidth(paramsJson: String?): Int? { + val raw = paramsJson ?: return null + val key = "\"maxWidth\"" + val idx = raw.indexOf(key) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + key.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + val num = tail.takeWhile { it.isDigit() } + return num.toIntOrNull() + } + } +} + +private val canvasHtml = + """ + + + + + + Canvas + + + + +
+
+
Ready
+
Waiting for agent
+
+
+ + + + """.trimIndent() diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/ChatSheet.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/ChatSheet.kt new file mode 100644 index 000000000..d8c1f1616 --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/ChatSheet.kt @@ -0,0 +1,73 @@ +package com.steipete.clawdis.node.ui + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material3.Button +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import com.steipete.clawdis.node.MainViewModel + +@Composable +fun ChatSheet(viewModel: MainViewModel) { + val messages by viewModel.chatMessages.collectAsState() + val error by viewModel.chatError.collectAsState() + val pendingRunCount by viewModel.pendingRunCount.collectAsState() + var input by remember { mutableStateOf("") } + + LaunchedEffect(Unit) { + viewModel.loadChat("main") + } + + Column(modifier = Modifier.fillMaxSize().padding(16.dp), verticalArrangement = Arrangement.spacedBy(12.dp)) { + Text("Clawd Chat · session main") + + if (!error.isNullOrBlank()) { + Text("Error: $error") + } + + LazyColumn(modifier = Modifier.fillMaxWidth().weight(1f, fill = true)) { + items(messages) { msg -> + Text("${msg.role}: ${msg.text}") + } + if (pendingRunCount > 0) { + item { Text("assistant: …") } + } + } + + Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) { + OutlinedTextField( + value = input, + onValueChange = { input = it }, + modifier = Modifier.weight(1f), + label = { Text("Message") }, + ) + Button( + onClick = { + val text = input + input = "" + viewModel.sendChat("main", text) + }, + enabled = input.trim().isNotEmpty(), + ) { + Text("Send") + } + } + } +} diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt new file mode 100644 index 000000000..0a7207377 --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt @@ -0,0 +1,80 @@ +package com.steipete.clawdis.node.ui + +import android.annotation.SuppressLint +import android.webkit.WebView +import android.webkit.WebViewClient +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Button +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.ModalBottomSheet +import androidx.compose.material3.Text +import androidx.compose.material3.rememberModalBottomSheetState +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import com.steipete.clawdis.node.MainViewModel + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun RootScreen(viewModel: MainViewModel) { + var sheet by remember { mutableStateOf(null) } + val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) + + Box(modifier = Modifier.fillMaxSize()) { + CanvasView(viewModel = viewModel) + + Box(modifier = Modifier.align(Alignment.TopEnd).padding(12.dp)) { + Button(onClick = { sheet = Sheet.Settings }) { Text("Settings") } + } + + Box(modifier = Modifier.align(Alignment.TopStart).padding(12.dp)) { + Button(onClick = { sheet = Sheet.Chat }) { Text("Chat") } + } + } + + if (sheet != null) { + ModalBottomSheet( + onDismissRequest = { sheet = null }, + sheetState = sheetState, + ) { + when (sheet) { + Sheet.Chat -> ChatSheet(viewModel = viewModel) + Sheet.Settings -> SettingsSheet(viewModel = viewModel) + null -> {} + } + } + } +} + +private enum class Sheet { + Chat, + Settings, +} + +@SuppressLint("SetJavaScriptEnabled") +@Composable +private fun CanvasView(viewModel: MainViewModel) { + val context = LocalContext.current + AndroidView( + modifier = Modifier.fillMaxSize(), + factory = { + WebView(context).apply { + settings.javaScriptEnabled = true + settings.domStorageEnabled = false + webViewClient = WebViewClient() + setBackgroundColor(0x00000000) + viewModel.canvas.attach(this) + } + }, + ) +} + diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/SettingsSheet.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/SettingsSheet.kt new file mode 100644 index 000000000..9f61e2ff5 --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/SettingsSheet.kt @@ -0,0 +1,146 @@ +package com.steipete.clawdis.node.ui + +import android.Manifest +import android.content.pm.PackageManager +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material3.Button +import androidx.compose.material3.Divider +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Switch +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat +import com.steipete.clawdis.node.MainViewModel + +@Composable +fun SettingsSheet(viewModel: MainViewModel) { + val context = LocalContext.current + val instanceId by viewModel.instanceId.collectAsState() + val displayName by viewModel.displayName.collectAsState() + val cameraEnabled by viewModel.cameraEnabled.collectAsState() + val manualEnabled by viewModel.manualEnabled.collectAsState() + val manualHost by viewModel.manualHost.collectAsState() + val manualPort by viewModel.manualPort.collectAsState() + val statusText by viewModel.statusText.collectAsState() + val serverName by viewModel.serverName.collectAsState() + val remoteAddress by viewModel.remoteAddress.collectAsState() + val bridges by viewModel.bridges.collectAsState() + + val permissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms -> + val cameraOk = perms[Manifest.permission.CAMERA] == true + viewModel.setCameraEnabled(cameraOk) + } + + Column(modifier = Modifier.fillMaxWidth().padding(16.dp), verticalArrangement = Arrangement.spacedBy(14.dp)) { + Text("Node") + OutlinedTextField( + value = displayName, + onValueChange = viewModel::setDisplayName, + label = { Text("Name") }, + modifier = Modifier.fillMaxWidth(), + ) + Text("Instance ID: $instanceId") + + Divider() + + Text("Camera") + Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) { + Switch( + checked = cameraEnabled, + onCheckedChange = { enabled -> + if (!enabled) { + viewModel.setCameraEnabled(false) + return@Switch + } + + val cameraOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == + PackageManager.PERMISSION_GRANTED + if (cameraOk) { + viewModel.setCameraEnabled(true) + } else { + permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO)) + } + }, + ) + Text(if (cameraEnabled) "Allow Camera" else "Camera Disabled") + } + Text("Tip: grant Microphone permission for video clips with audio.") + + Divider() + + Text("Bridge") + Text("Status: $statusText") + if (serverName != null) Text("Server: $serverName") + if (remoteAddress != null) Text("Address: $remoteAddress") + + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + Button(onClick = viewModel::disconnect) { Text("Disconnect") } + } + + Divider() + + Text("Advanced") + Row(horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.fillMaxWidth()) { + Switch(checked = manualEnabled, onCheckedChange = viewModel::setManualEnabled) + Text(if (manualEnabled) "Manual Bridge Enabled" else "Manual Bridge Disabled") + } + OutlinedTextField( + value = manualHost, + onValueChange = viewModel::setManualHost, + label = { Text("Host") }, + modifier = Modifier.fillMaxWidth(), + enabled = manualEnabled, + ) + OutlinedTextField( + value = manualPort.toString(), + onValueChange = { v -> viewModel.setManualPort(v.toIntOrNull() ?: 0) }, + label = { Text("Port") }, + modifier = Modifier.fillMaxWidth(), + enabled = manualEnabled, + ) + Button(onClick = viewModel::connectManual, enabled = manualEnabled) { Text("Connect (Manual)") } + + Divider() + + Text("Discovered Bridges") + if (bridges.isEmpty()) { + Text("No bridges found yet.") + } else { + LazyColumn(modifier = Modifier.fillMaxWidth().height(240.dp)) { + items(bridges) { bridge -> + Row( + modifier = Modifier.fillMaxWidth().padding(vertical = 8.dp), + horizontalArrangement = Arrangement.SpaceBetween, + ) { + Column(modifier = Modifier.weight(1f)) { + Text(bridge.name) + Text("${bridge.host}:${bridge.port}") + } + Spacer(modifier = Modifier.padding(4.dp)) + Button(onClick = { viewModel.connect(bridge) }) { Text("Connect") } + } + Divider() + } + } + } + + Spacer(modifier = Modifier.height(20.dp)) + } +} diff --git a/apps/android/app/src/main/res/values/strings.xml b/apps/android/app/src/main/res/values/strings.xml new file mode 100644 index 000000000..4299a7813 --- /dev/null +++ b/apps/android/app/src/main/res/values/strings.xml @@ -0,0 +1,4 @@ + + Clawdis Node + + diff --git a/apps/android/app/src/main/res/values/themes.xml b/apps/android/app/src/main/res/values/themes.xml new file mode 100644 index 000000000..86d2e2f3b --- /dev/null +++ b/apps/android/app/src/main/res/values/themes.xml @@ -0,0 +1,8 @@ + + + + diff --git a/apps/android/build.gradle.kts b/apps/android/build.gradle.kts new file mode 100644 index 000000000..8c8f431e9 --- /dev/null +++ b/apps/android/build.gradle.kts @@ -0,0 +1,6 @@ +plugins { + id("com.android.application") version "8.5.2" apply false + id("org.jetbrains.kotlin.android") version "1.9.24" apply false + id("org.jetbrains.kotlin.plugin.serialization") version "1.9.24" apply false +} + diff --git a/apps/android/gradle.properties b/apps/android/gradle.properties new file mode 100644 index 000000000..47d0e718d --- /dev/null +++ b/apps/android/gradle.properties @@ -0,0 +1,4 @@ +org.gradle.jvmargs=-Xmx3g -Dfile.encoding=UTF-8 +android.useAndroidX=true +android.nonTransitiveRClass=true + diff --git a/apps/android/gradle/wrapper/gradle-wrapper.jar b/apps/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..e6441136f Binary files /dev/null and b/apps/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/apps/android/gradle/wrapper/gradle-wrapper.properties b/apps/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..b82aa23a4 --- /dev/null +++ b/apps/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/apps/android/gradlew b/apps/android/gradlew new file mode 100755 index 000000000..8f7d028a3 --- /dev/null +++ b/apps/android/gradlew @@ -0,0 +1,276 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +# Android Gradle Plugin requires a supported JDK (typically 17). +# On macOS, prefer JDK 17 when JAVA_HOME isn't set. +if [ "$darwin" = "true" ] && [ -z "$JAVA_HOME" ] ; then + if [ -x "/usr/libexec/java_home" ] ; then + jdk17=$(/usr/libexec/java_home -v 17 2>/dev/null) + if [ -n "$jdk17" ] && [ -d "$jdk17" ] ; then + JAVA_HOME=$jdk17 + export JAVA_HOME + fi + fi +fi + +# If the Android SDK isn't configured, try common default locations. +if [ -z "$ANDROID_SDK_ROOT" ] ; then + if [ -d "$HOME/Library/Android/sdk" ] ; then + ANDROID_SDK_ROOT="$HOME/Library/Android/sdk" + export ANDROID_SDK_ROOT + elif [ -d "$HOME/Android/Sdk" ] ; then + ANDROID_SDK_ROOT="$HOME/Android/Sdk" + export ANDROID_SDK_ROOT + fi +fi +if [ -z "$ANDROID_HOME" ] && [ -n "$ANDROID_SDK_ROOT" ] ; then + ANDROID_HOME="$ANDROID_SDK_ROOT" + export ANDROID_HOME +fi + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='-Dfile.encoding=UTF-8 "-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/apps/android/gradlew.bat b/apps/android/gradlew.bat new file mode 100644 index 000000000..16e26a115 --- /dev/null +++ b/apps/android/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS=-Dfile.encoding=UTF-8 "-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/apps/android/settings.gradle.kts b/apps/android/settings.gradle.kts new file mode 100644 index 000000000..7d67ee7a7 --- /dev/null +++ b/apps/android/settings.gradle.kts @@ -0,0 +1,19 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "ClawdisNodeAndroid" +include(":app") + diff --git a/docs/android/connect.md b/docs/android/connect.md new file mode 100644 index 000000000..f9e24a9ba --- /dev/null +++ b/docs/android/connect.md @@ -0,0 +1,94 @@ +--- +summary: "Runbook: connect/pair the Android node to a Clawdis Gateway and use Canvas/Chat/Camera" +read_when: + - Pairing or reconnecting the Android node + - Debugging Android bridge discovery or auth + - Verifying chat history parity across clients +--- + +# Android Node Connection Runbook + +Android node app ⇄ (mDNS/NSD + TCP bridge) ⇄ **Gateway bridge** ⇄ (loopback WS) ⇄ **Gateway** + +The Gateway WebSocket stays loopback-only (`ws://127.0.0.1:18789`). Android talks to the LAN-facing **bridge** (default `tcp://0.0.0.0:18790`) and uses Gateway-owned pairing. + +## Prerequisites + +- You can run the Gateway on the “master” machine. +- Android device/emulator is on the same LAN (mDNS must work) or you know the gateway’s LAN IP for manual connect. +- You can run the CLI (`clawdis`) on the gateway machine (or via SSH). + +## 1) Start the Gateway (with bridge enabled) + +Bridge is enabled by default (disable via `CLAWDIS_BRIDGE_ENABLED=0`). + +```bash +pnpm clawdis gateway --port 18789 --verbose +``` + +Confirm in logs you see something like: +- `bridge listening on tcp://0.0.0.0:18790 (Iris)` + +## 2) Verify discovery (optional) + +From the gateway machine: + +```bash +dns-sd -B _clawdis-bridge._tcp local. +``` + +More debugging notes: `docs/bonjour.md`. + +## 3) Connect from Android + +In the Android app: + +- Open **Settings**. +- Under **Discovered Bridges**, select your gateway and hit **Connect**. +- If mDNS is blocked, use **Advanced → Manual Bridge** (host + port) and **Connect (Manual)**. + +After the first successful pairing, Android auto-reconnects on launch: +- Manual endpoint (if enabled), otherwise +- The last discovered bridge (best-effort). + +## 4) Approve pairing (CLI) + +On the gateway machine: + +```bash +clawdis nodes pending +clawdis nodes approve +``` + +Pairing details: `docs/gateway/pairing.md`. + +## 5) Verify the node is connected + +- Via nodes list: + ```bash + clawdis nodes list + ``` +- Via Gateway: + ```bash + clawdis gateway call node.list --params "{}" + ``` + +## 6) Chat + history + +The Android node’s Chat sheet uses the gateway’s **primary session key** (`main`), so history and replies are shared with WebChat and other clients: + +- History: `chat.history` +- Send: `chat.send` +- Push updates (best-effort): `chat.subscribe` → `event:"chat"` + +## 7) Canvas + camera + +Canvas commands (foreground only): +- `screen.eval`, `screen.snapshot`, `screen.navigate`, `screen.setMode` + +Camera commands (foreground only; permission-gated): +- `camera.snap` (jpg) +- `camera.clip` (mp4) + +See `docs/camera.md` for parameters and CLI helpers. + diff --git a/docs/camera.md b/docs/camera.md index dc5ab93db..23fbab6e5 100644 --- a/docs/camera.md +++ b/docs/camera.md @@ -10,6 +10,7 @@ read_when: Clawdis supports **camera capture** for agent workflows: - **iOS node** (paired via Gateway): capture a **photo** (`jpg`) or **short video clip** (`mp4`, with optional audio) via `node.invoke`. +- **Android node** (paired via Gateway): capture a **photo** (`jpg`) or **short video clip** (`mp4`, with optional audio) via `node.invoke`. - **macOS app** (local control socket): capture a **photo** (`jpg`) or **short video clip** (`mp4`, with optional audio) via `clawdis-mac`. All camera access is gated behind **user-controlled settings**. @@ -68,6 +69,26 @@ Notes: - `nodes camera snap` defaults to **both** facings to give the agent both views. - Output files are temporary (in the OS temp directory) unless you build your own wrapper. +## Android node + +### User setting (default on) + +- Android Settings sheet → **Camera** → **Allow Camera** (`camera.enabled`) + - Default: **on** (missing key is treated as enabled). + - When off: `camera.*` commands return `CAMERA_DISABLED`. + +### Permissions + +- Android requires runtime permissions: + - `CAMERA` for both `camera.snap` and `camera.clip`. + - `RECORD_AUDIO` for `camera.clip` when `includeAudio=true`. + +If permissions are denied, `camera.*` requests fail with a `*_PERMISSION_REQUIRED` error. + +### Foreground requirement + +Like `screen.*`, the Android node only allows `camera.*` commands in the **foreground**. Background invocations return `NODE_BACKGROUND_UNAVAILABLE`. + ## macOS app ### User setting (default off) @@ -95,4 +116,3 @@ clawdis-mac camera clip --no-audio - Camera and microphone access trigger the usual OS permission prompts (and require usage strings in Info.plist). - Video clips are intentionally short to avoid oversized bridge payloads (base64 overhead + WebSocket message limits). -