let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: extend status activity indicators

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-29 23:42:22 +01:00
parent 3c338d1858
commit 303954ae8c
9 changed files with 241 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@@ -14,6 +14,8 @@
- iOS/Android nodes: bridge auto-connect refreshes stale tokens and settings now show richer bridge/device details.
- iOS/Android nodes: status pill now surfaces camera activity instead of overlay toasts.
- iOS/Android/macOS nodes: camera snaps recompress to keep base64 payloads under 5 MB.
- iOS/Android nodes: status pill now surfaces pairing, screen recording, voice wake, and foreground-required states.
- macOS menu: top status line now shows pending node pairing approvals (incl. repairs).
- CLI: avoid spurious gateway close errors after successful request/response cycles.
- Agent runtime: clamp tool-result images to the 5MB Anthropic limit to avoid hard request rejections.
- Tests: add Swift Testing coverage for camera errors and Kotest coverage for Android bridge endpoints.

3
apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt
View File

@@ -23,9 +23,11 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val statusText: StateFlow<String> = runtime.statusText
val serverName: StateFlow<String?> = runtime.serverName
val remoteAddress: StateFlow<String?> = runtime.remoteAddress
val isForeground: StateFlow<Boolean> = runtime.isForeground
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
val instanceId: StateFlow<String> = runtime.instanceId
val displayName: StateFlow<String> = runtime.displayName
@@ -34,6 +36,7 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val wakeWords: StateFlow<List<String>> = runtime.wakeWords
val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening
val talkEnabled: StateFlow<Boolean> = runtime.talkEnabled
val talkStatusText: StateFlow<String> = runtime.talkStatusText

25
apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt
View File

@@ -111,6 +111,9 @@ class NodeRuntime(context: Context) {
private val _cameraFlashToken = MutableStateFlow(0L)
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
private val _screenRecordActive = MutableStateFlow(false)
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
private val _serverName = MutableStateFlow<String?>(null)
val serverName: StateFlow<String?> = _serverName.asStateFlow()
@@ -756,14 +759,20 @@ class NodeRuntime(context: Context) {
}
}
ClawdisScreenCommand.Record.rawValue -> {
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
BridgeSession.InvokeResult.ok(res.payloadJson)
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
_screenRecordActive.value = true
try {
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return BridgeSession.InvokeResult.error(code = code, message = message)
}
BridgeSession.InvokeResult.ok(res.payloadJson)
} finally {
_screenRecordActive.value = false
}
}
else ->
BridgeSession.InvokeResult.error(

81
apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt
View File

@@ -36,6 +36,10 @@ import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.Error
import androidx.compose.material.icons.filled.FiberManualRecord
import androidx.compose.material.icons.filled.PhotoCamera
import androidx.compose.material.icons.filled.RecordVoiceOver
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Report
import androidx.compose.material.icons.filled.ScreenShare
import androidx.compose.material.icons.filled.Settings
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
@@ -65,39 +69,100 @@ fun RootScreen(viewModel: MainViewModel) {
val statusText by viewModel.statusText.collectAsState()
val cameraHud by viewModel.cameraHud.collectAsState()
val cameraFlashToken by viewModel.cameraFlashToken.collectAsState()
val screenRecordActive by viewModel.screenRecordActive.collectAsState()
val isForeground by viewModel.isForeground.collectAsState()
val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState()
val activity =
remember(cameraHud) {
// Status pill owns transient capture state so it doesn't overlap the connection indicator.
cameraHud?.let { hud ->
when (hud.kind) {
remember(cameraHud, screenRecordActive, isForeground, statusText, voiceWakeStatusText) {
// Status pill owns transient activity state so it doesn't overlap the connection indicator.
if (!isForeground) {
return@remember StatusActivity(
title = "Foreground required",
icon = Icons.Default.Report,
contentDescription = "Foreground required",
)
}
val lowerStatus = statusText.lowercase()
if (lowerStatus.contains("repair")) {
return@remember StatusActivity(
title = "Repairing…",
icon = Icons.Default.Refresh,
contentDescription = "Repairing",
)
}
if (lowerStatus.contains("pairing") || lowerStatus.contains("approval")) {
return@remember StatusActivity(
title = "Approval pending",
icon = Icons.Default.RecordVoiceOver,
contentDescription = "Approval pending",
)
}
if (lowerStatus.contains("reconnecting") || lowerStatus.contains("connecting")) {
return@remember StatusActivity(
title = "Gateway reconnecting…",
icon = Icons.Default.Refresh,
contentDescription = "Gateway reconnecting",
)
}
if (screenRecordActive) {
return@remember StatusActivity(
title = "Recording screen…",
icon = Icons.Default.ScreenShare,
contentDescription = "Recording screen",
tint = androidx.compose.ui.graphics.Color.Red,
)
}
if (cameraHud != null) {
return@remember when (cameraHud.kind) {
CameraHudKind.Photo ->
StatusActivity(
title = hud.message,
title = cameraHud.message,
icon = Icons.Default.PhotoCamera,
contentDescription = "Taking photo",
)
CameraHudKind.Recording ->
StatusActivity(
title = hud.message,
title = cameraHud.message,
icon = Icons.Default.FiberManualRecord,
contentDescription = "Recording",
tint = androidx.compose.ui.graphics.Color.Red,
)
CameraHudKind.Success ->
StatusActivity(
title = hud.message,
title = cameraHud.message,
icon = Icons.Default.CheckCircle,
contentDescription = "Capture finished",
)
CameraHudKind.Error ->
StatusActivity(
title = hud.message,
title = cameraHud.message,
icon = Icons.Default.Error,
contentDescription = "Capture failed",
tint = androidx.compose.ui.graphics.Color.Red,
)
}
}
if (voiceWakeStatusText.contains("Microphone permission", ignoreCase = true)) {
return@remember StatusActivity(
title = "Mic permission",
icon = Icons.Default.Error,
contentDescription = "Mic permission required",
)
}
if (voiceWakeStatusText == "Paused") {
val suffix = if (!isForeground) " (background)" else ""
return@remember StatusActivity(
title = "Voice Wake paused$suffix",
icon = Icons.Default.RecordVoiceOver,
contentDescription = "Voice Wake paused",
)
}
null
}
val bridgeState =

4
apps/ios/Sources/Model/NodeAppModel.swift
View File

@@ -36,6 +36,7 @@ final class NodeAppModel {
var cameraHUDText: String?
var cameraHUDKind: CameraHUDKind?
var cameraFlashNonce: Int = 0
var screenRecordActive: Bool = false
init() {
self.voiceWake.configure { [weak self] cmd in
@@ -598,6 +599,9 @@ final class NodeAppModel {
NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4",
])
}
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
self.screenRecordActive = true
defer { self.screenRecordActive = false }
let path = try await self.screenRecorder.record(
screenIndex: params.screenIndex,
durationMs: params.durationMs,

74
apps/ios/Sources/RootCanvas.swift
View File

@@ -119,6 +119,7 @@ struct RootCanvas: View {
}
private struct CanvasContent: View {
@Environment(NodeAppModel.self) private var appModel
var systemColorScheme: ColorScheme
var bridgeStatus: StatusPill.BridgeState
var voiceWakeEnabled: Bool
@@ -173,28 +174,63 @@ private struct CanvasContent: View {
}
private var statusActivity: StatusPill.Activity? {
// Status pill owns transient capture state so it doesn't overlap the connection indicator.
guard let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind else { return nil }
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
// Status pill owns transient activity state so it doesn't overlap the connection indicator.
if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let bridgeLower = bridgeStatus.lowercased()
if bridgeLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if bridgeLower.contains("approval") || bridgeLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
if bridgeLower.contains("reconnecting") || bridgeLower.contains("connecting") {
return StatusPill.Activity(title: "Gateway reconnecting…", systemImage: "arrow.triangle.2.circlepath")
}
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
}
private struct OverlayButton: View {

64
apps/ios/Sources/RootTabs.swift
View File

@@ -26,7 +26,7 @@ struct RootTabs: View {
StatusPill(
bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
activity: nil,
activity: self.statusActivity,
onTap: { self.selectedTab = 2 })
.padding(.leading, 10)
.safeAreaPadding(.top, 10)
@@ -80,4 +80,66 @@ struct RootTabs: View {
return .disconnected
}
private var statusActivity: StatusPill.Activity? {
// Keep the top pill consistent across tabs (camera + voice wake + pairing states).
if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
}
let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let bridgeLower = bridgeStatus.lowercased()
if bridgeLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if bridgeLower.contains("approval") || bridgeLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
if bridgeLower.contains("reconnecting") || bridgeLower.contains("connecting") {
return StatusPill.Activity(title: "Gateway reconnecting…", systemImage: "arrow.triangle.2.circlepath")
}
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText = self.appModel.cameraHUDText,
let cameraHUDKind = self.appModel.cameraHUDKind,
!cameraHUDText.isEmpty
{
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
}

8
apps/macos/Sources/Clawdis/MenuContentView.swift
View File

@@ -14,6 +14,7 @@ struct MenuContent: View {
private let heartbeatStore = HeartbeatStore.shared
private let controlChannel = ControlChannel.shared
private let activityStore = WorkActivityStore.shared
@Bindable private var pairingPrompter = NodePairingApprovalPrompter.shared
@Environment(\.openSettings) private var openSettings
@State private var availableMics: [AudioInputDevice] = []
@State private var loadingMics = false
@@ -32,6 +33,13 @@ struct MenuContent: View {
VStack(alignment: .leading, spacing: 2) {
Text(self.connectionLabel)
self.statusLine(label: self.healthStatus.label, color: self.healthStatus.color)
if self.pairingPrompter.pendingCount > 0 {
let repairCount = self.pairingPrompter.pendingRepairCount
let repairSuffix = repairCount > 0 ? " · \(repairCount) repair" : ""
self.statusLine(
label: "Pairing approval pending (\(self.pairingPrompter.pendingCount))\(repairSuffix)",
color: .orange)
}
}
}
.disabled(self.state.connectionMode == .unconfigured)

16
apps/macos/Sources/Clawdis/NodePairingApprovalPrompter.swift
View File

@@ -2,6 +2,7 @@ import AppKit
import ClawdisIPC
import ClawdisProtocol
import Foundation
import Observation
import OSLog
import UserNotifications
@@ -15,6 +16,7 @@ enum NodePairingReconcilePolicy {
}
@MainActor
@Observable
final class NodePairingApprovalPrompter {
static let shared = NodePairingApprovalPrompter()
@@ -26,6 +28,8 @@ final class NodePairingApprovalPrompter {
private var isStopping = false
private var isPresenting = false
private var queue: [PendingRequest] = []
var pendingCount: Int = 0
var pendingRepairCount: Int = 0
private var activeAlert: NSAlert?
private var activeRequestId: String?
private var alertHostWindow: NSWindow?
@@ -104,6 +108,7 @@ final class NodePairingApprovalPrompter {
self.reconcileOnceTask?.cancel()
self.reconcileOnceTask = nil
self.queue.removeAll(keepingCapacity: false)
self.updatePendingCounts()
self.isPresenting = false
self.activeRequestId = nil
self.alertHostWindow?.orderOut(nil)
@@ -292,6 +297,7 @@ final class NodePairingApprovalPrompter {
private func enqueue(_ req: PendingRequest) {
if self.queue.contains(req) { return }
self.queue.append(req)
self.updatePendingCounts()
self.presentNextIfNeeded()
self.updateReconcileLoop()
}
@@ -362,6 +368,7 @@ final class NodePairingApprovalPrompter {
} else {
self.queue.removeAll { $0 == request }
}
self.updatePendingCounts()
self.isPresenting = false
self.presentNextIfNeeded()
self.updateReconcileLoop()
@@ -501,6 +508,8 @@ final class NodePairingApprovalPrompter {
} else {
self.queue.removeAll { $0 == req }
}
self.updatePendingCounts()
self.isPresenting = false
self.presentNextIfNeeded()
self.updateReconcileLoop()
@@ -599,6 +608,12 @@ final class NodePairingApprovalPrompter {
}
}
private func updatePendingCounts() {
// Keep a cheap observable summary for the menu bar status line.
self.pendingCount = self.queue.count
self.pendingRepairCount = self.queue.filter { $0.isRepair == true }.count
}
private func reconcileOnce(timeoutMs: Double) async {
if self.isStopping { return }
if self.reconcileInFlight { return }
@@ -643,6 +658,7 @@ final class NodePairingApprovalPrompter {
return
}
self.queue.removeAll { $0.requestId == resolved.requestId }
self.updatePendingCounts()
Task { @MainActor in
await self.notify(resolution: resolution, request: request, via: "remote")
}