From 06f71d883cc74f71c75fb921b97f2dcffcad59e0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 18 Dec 2025 23:32:07 +0100 Subject: [PATCH] Android: JPEG canvas snapshots + camera permission prompts --- .../com/steipete/clawdis/node/MainActivity.kt | 3 + .../com/steipete/clawdis/node/NodeRuntime.kt | 10 +- .../clawdis/node/PermissionRequester.kt | 57 ++++++++++ .../clawdis/node/node/CameraCaptureManager.kt | 38 +++++-- .../clawdis/node/node/CanvasController.kt | 104 ++++++++++++++++-- .../CanvasControllerSnapshotParamsTest.kt | 43 ++++++++ 6 files changed, 233 insertions(+), 22 deletions(-) create mode 100644 apps/android/app/src/main/java/com/steipete/clawdis/node/PermissionRequester.kt create mode 100644 apps/android/app/src/test/java/com/steipete/clawdis/node/node/CanvasControllerSnapshotParamsTest.kt diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/MainActivity.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/MainActivity.kt index 1b10e1e63..6478f7fba 100644 --- a/apps/android/app/src/main/java/com/steipete/clawdis/node/MainActivity.kt +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/MainActivity.kt @@ -24,6 +24,7 @@ import kotlinx.coroutines.launch class MainActivity : ComponentActivity() { private val viewModel: MainViewModel by viewModels() + private lateinit var permissionRequester: PermissionRequester override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -33,7 +34,9 @@ class MainActivity : ComponentActivity() { requestDiscoveryPermissionsIfNeeded() requestNotificationPermissionIfNeeded() NodeForegroundService.start(this) + permissionRequester = PermissionRequester(this) viewModel.camera.attachLifecycleOwner(this) + viewModel.camera.attachPermissionRequester(permissionRequester) lifecycleScope.launch { repeatOnLifecycle(Lifecycle.State.STARTED) { diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt index 6ebef6e2d..4e431bf8e 100644 --- a/apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt @@ -576,17 +576,21 @@ class NodeRuntime(context: Context) { BridgeSession.InvokeResult.ok("""{"result":${result.toJsonString()}}""") } ClawdisCanvasCommand.Snapshot.rawValue -> { - val maxWidth = CanvasController.parseSnapshotMaxWidth(paramsJson) + val snapshotParams = CanvasController.parseSnapshotParams(paramsJson) val base64 = try { - canvas.snapshotPngBase64(maxWidth = maxWidth) + canvas.snapshotBase64( + format = snapshotParams.format, + quality = snapshotParams.quality, + maxWidth = snapshotParams.maxWidth, + ) } catch (err: Throwable) { return BridgeSession.InvokeResult.error( code = "NODE_BACKGROUND_UNAVAILABLE", message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", ) } - BridgeSession.InvokeResult.ok("""{"format":"png","base64":"$base64"}""") + BridgeSession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""") } ClawdisCanvasA2UICommand.Reset.rawValue -> { val ready = ensureA2uiReady() diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/PermissionRequester.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/PermissionRequester.kt new file mode 100644 index 000000000..b95e51a2b --- /dev/null +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/PermissionRequester.kt @@ -0,0 +1,57 @@ +package com.steipete.clawdis.node + +import android.content.pm.PackageManager +import androidx.activity.ComponentActivity +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.core.content.ContextCompat +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext + +class PermissionRequester(private val activity: ComponentActivity) { + private val mutex = Mutex() + private var pending: CompletableDeferred>? = null + + private val launcher: ActivityResultLauncher> = + activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result -> + val p = pending + pending = null + p?.complete(result) + } + + suspend fun requestIfMissing( + permissions: List, + timeoutMs: Long = 20_000, + ): Map = + mutex.withLock { + val missing = + permissions.filter { perm -> + ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED + } + if (missing.isEmpty()) { + return permissions.associateWith { true } + } + + val deferred = CompletableDeferred>() + pending = deferred + withContext(Dispatchers.Main) { + launcher.launch(missing.toTypedArray()) + } + + val result = + withContext(Dispatchers.Default) { + kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() } + } + + // Merge: if something was already granted, treat it as granted even if launcher omitted it. + return permissions.associateWith { perm -> + val nowGranted = + ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED + result[perm] == true || nowGranted + } + } +} + diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CameraCaptureManager.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CameraCaptureManager.kt index d80732f36..b3a11b2bf 100644 --- a/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CameraCaptureManager.kt +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CameraCaptureManager.kt @@ -18,6 +18,7 @@ import androidx.camera.video.VideoCapture import androidx.camera.video.VideoRecordEvent import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat.checkSelfPermission +import com.steipete.clawdis.node.PermissionRequester import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.withTimeout @@ -32,24 +33,43 @@ class CameraCaptureManager(private val context: Context) { data class Payload(val payloadJson: String) @Volatile private var lifecycleOwner: LifecycleOwner? = null + @Volatile private var permissionRequester: PermissionRequester? = null fun attachLifecycleOwner(owner: LifecycleOwner) { lifecycleOwner = owner } - private fun requireCameraPermission() { - val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED - if (!granted) throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") + fun attachPermissionRequester(requester: PermissionRequester) { + permissionRequester = requester } - private fun requireMicPermission() { + private suspend fun ensureCameraPermission() { + val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED + if (granted) return + + val requester = permissionRequester + ?: throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") + val results = requester.requestIfMissing(listOf(Manifest.permission.CAMERA)) + if (results[Manifest.permission.CAMERA] != true) { + throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") + } + } + + private suspend fun ensureMicPermission() { val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED - if (!granted) throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + if (granted) return + + val requester = permissionRequester + ?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + val results = requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO)) + if (results[Manifest.permission.RECORD_AUDIO] != true) { + throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + } } suspend fun snap(paramsJson: String?): Payload = withContext(Dispatchers.Main) { - requireCameraPermission() + ensureCameraPermission() val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") val facing = parseFacing(paramsJson) ?: "front" val quality = (parseQuality(paramsJson) ?: 0.9).coerceIn(0.1, 1.0) @@ -90,12 +110,12 @@ class CameraCaptureManager(private val context: Context) { suspend fun clip(paramsJson: String?): Payload = withContext(Dispatchers.Main) { - requireCameraPermission() + ensureCameraPermission() val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") val facing = parseFacing(paramsJson) ?: "front" - val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 45_000) + val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 60_000) val includeAudio = parseIncludeAudio(paramsJson) ?: true - if (includeAudio) requireMicPermission() + if (includeAudio) ensureMicPermission() val provider = context.cameraProvider() val recorder = Recorder.Builder().build() diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt index 2635cfbd5..b0f4d4b25 100644 --- a/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/node/CanvasController.kt @@ -5,20 +5,33 @@ import android.os.Build import android.graphics.Canvas import android.os.Looper import android.webkit.WebView -import org.json.JSONObject import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.withContext import java.io.ByteArrayOutputStream import android.util.Base64 +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive import kotlin.coroutines.resume class CanvasController { + enum class SnapshotFormat(val rawValue: String) { + Png("png"), + Jpeg("jpeg"), + } + @Volatile private var webView: WebView? = null @Volatile private var url: String? = null private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html" + private fun clampJpegQuality(quality: Double?): Int { + val q = (quality ?: 0.82).coerceIn(0.1, 1.0) + return (q * 100.0).toInt().coerceIn(1, 100) + } + fun attach(webView: WebView) { this.webView = webView reload() @@ -77,6 +90,28 @@ class CanvasController { Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) } + suspend fun snapshotBase64(format: SnapshotFormat, quality: Double?, maxWidth: Int?): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + val bmp = wv.captureBitmap() + val scaled = + if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { + val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) + Bitmap.createScaledBitmap(bmp, maxWidth, h, true) + } else { + bmp + } + + val out = ByteArrayOutputStream() + val (compressFormat, compressQuality) = + when (format) { + SnapshotFormat.Png -> Bitmap.CompressFormat.PNG to 100 + SnapshotFormat.Jpeg -> Bitmap.CompressFormat.JPEG to clampJpegQuality(quality) + } + scaled.compress(compressFormat, compressQuality, out) + Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) + } + private suspend fun WebView.captureBitmap(): Bitmap = suspendCancellableCoroutine { cont -> val width = width.coerceAtLeast(1) @@ -90,32 +125,81 @@ class CanvasController { } companion object { + data class SnapshotParams(val format: SnapshotFormat, val quality: Double?, val maxWidth: Int?) + fun parseNavigateUrl(paramsJson: String?): String { - val obj = parseParamsObject(paramsJson) - return obj?.optString("url", "")?.trim().orEmpty() + val obj = parseParamsObject(paramsJson) ?: return "" + return obj.string("url").trim() } fun parseEvalJs(paramsJson: String?): String? { val obj = parseParamsObject(paramsJson) ?: return null - val js = obj.optString("javaScript", "") + val js = obj.string("javaScript").trim() return js.takeIf { it.isNotBlank() } } fun parseSnapshotMaxWidth(paramsJson: String?): Int? { val obj = parseParamsObject(paramsJson) ?: return null - if (!obj.has("maxWidth")) return null - val width = obj.optInt("maxWidth", 0) + if (!obj.containsKey("maxWidth")) return null + val width = obj.int("maxWidth") ?: 0 return width.takeIf { it > 0 } } - private fun parseParamsObject(paramsJson: String?): JSONObject? { - val raw = paramsJson?.trim() ?: return null - if (raw.isBlank()) return null + fun parseSnapshotFormat(paramsJson: String?): SnapshotFormat { + val obj = parseParamsObject(paramsJson) ?: return SnapshotFormat.Jpeg + val raw = obj.string("format").trim().lowercase() + return when (raw) { + "png" -> SnapshotFormat.Png + "jpeg", "jpg" -> SnapshotFormat.Jpeg + "" -> SnapshotFormat.Jpeg + else -> SnapshotFormat.Jpeg + } + } + + fun parseSnapshotQuality(paramsJson: String?): Double? { + val obj = parseParamsObject(paramsJson) ?: return null + if (!obj.containsKey("quality")) return null + val q = obj.double("quality") ?: Double.NaN + if (!q.isFinite()) return null + return q.coerceIn(0.1, 1.0) + } + + fun parseSnapshotParams(paramsJson: String?): SnapshotParams { + return SnapshotParams( + format = parseSnapshotFormat(paramsJson), + quality = parseSnapshotQuality(paramsJson), + maxWidth = parseSnapshotMaxWidth(paramsJson), + ) + } + + private val json = Json { ignoreUnknownKeys = true } + + private fun parseParamsObject(paramsJson: String?): JsonObject? { + val raw = paramsJson?.trim().orEmpty() + if (raw.isEmpty()) return null return try { - JSONObject(raw) + json.parseToJsonElement(raw).asObjectOrNull() } catch (_: Throwable) { null } } + + private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + + private fun JsonObject.string(key: String): String { + val prim = this[key] as? JsonPrimitive ?: return "" + val raw = prim.content + return raw.takeIf { it != "null" }.orEmpty() + } + + private fun JsonObject.int(key: String): Int? { + val prim = this[key] as? JsonPrimitive ?: return null + return prim.content.toIntOrNull() + } + + private fun JsonObject.double(key: String): Double? { + val prim = this[key] as? JsonPrimitive ?: return null + return prim.content.toDoubleOrNull() + } } } diff --git a/apps/android/app/src/test/java/com/steipete/clawdis/node/node/CanvasControllerSnapshotParamsTest.kt b/apps/android/app/src/test/java/com/steipete/clawdis/node/node/CanvasControllerSnapshotParamsTest.kt new file mode 100644 index 000000000..b7a135196 --- /dev/null +++ b/apps/android/app/src/test/java/com/steipete/clawdis/node/node/CanvasControllerSnapshotParamsTest.kt @@ -0,0 +1,43 @@ +package com.steipete.clawdis.node.node + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class CanvasControllerSnapshotParamsTest { + @Test + fun parseSnapshotParamsDefaultsToJpeg() { + val params = CanvasController.parseSnapshotParams(null) + assertEquals(CanvasController.SnapshotFormat.Jpeg, params.format) + assertNull(params.quality) + assertNull(params.maxWidth) + } + + @Test + fun parseSnapshotParamsParsesPng() { + val params = CanvasController.parseSnapshotParams("""{"format":"png","maxWidth":900}""") + assertEquals(CanvasController.SnapshotFormat.Png, params.format) + assertEquals(900, params.maxWidth) + } + + @Test + fun parseSnapshotParamsParsesJpegAliases() { + assertEquals( + CanvasController.SnapshotFormat.Jpeg, + CanvasController.parseSnapshotParams("""{"format":"jpeg"}""").format, + ) + assertEquals( + CanvasController.SnapshotFormat.Jpeg, + CanvasController.parseSnapshotParams("""{"format":"jpg"}""").format, + ) + } + + @Test + fun parseSnapshotParamsClampsQuality() { + val low = CanvasController.parseSnapshotParams("""{"quality":0.01}""") + assertEquals(0.1, low.quality) + + val high = CanvasController.parseSnapshotParams("""{"quality":5}""") + assertEquals(1.0, high.quality) + } +}