diff --git a/CHANGELOG.md b/CHANGELOG.md index c17709774..01821b781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ - iOS/Android nodes: bridge auto-connect refreshes stale tokens and settings now show richer bridge/device details. - iOS/Android nodes: status pill now surfaces camera activity instead of overlay toasts. - iOS/Android/macOS nodes: camera snaps recompress to keep base64 payloads under 5 MB. +- iOS/Android nodes: status pill now surfaces pairing, screen recording, voice wake, and foreground-required states. +- macOS menu: top status line now shows pending node pairing approvals (incl. repairs). - CLI: avoid spurious gateway close errors after successful request/response cycles. - Agent runtime: clamp tool-result images to the 5MB Anthropic limit to avoid hard request rejections. - Tests: add Swift Testing coverage for camera errors and Kotest coverage for Android bridge endpoints. diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt index ee1c83c9b..f1fef1640 100644 --- a/apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/MainViewModel.kt @@ -23,9 +23,11 @@ class MainViewModel(app: Application) : AndroidViewModel(app) { val statusText: StateFlow = runtime.statusText val serverName: StateFlow = runtime.serverName val remoteAddress: StateFlow = runtime.remoteAddress + val isForeground: StateFlow = runtime.isForeground val cameraHud: StateFlow = runtime.cameraHud val cameraFlashToken: StateFlow = runtime.cameraFlashToken + val screenRecordActive: StateFlow = runtime.screenRecordActive val instanceId: StateFlow = runtime.instanceId val displayName: StateFlow = runtime.displayName @@ -34,6 +36,7 @@ class MainViewModel(app: Application) : AndroidViewModel(app) { val wakeWords: StateFlow> = runtime.wakeWords val voiceWakeMode: StateFlow = runtime.voiceWakeMode val voiceWakeStatusText: StateFlow = runtime.voiceWakeStatusText + val voiceWakeStatusText: StateFlow = runtime.voiceWakeStatusText val voiceWakeIsListening: StateFlow = runtime.voiceWakeIsListening val talkEnabled: StateFlow = runtime.talkEnabled val talkStatusText: StateFlow = runtime.talkStatusText diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt index 4984f7e0f..21a22a428 100644 --- a/apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt @@ -111,6 +111,9 @@ class NodeRuntime(context: Context) { private val _cameraFlashToken = MutableStateFlow(0L) val cameraFlashToken: StateFlow = _cameraFlashToken.asStateFlow() + private val _screenRecordActive = MutableStateFlow(false) + val screenRecordActive: StateFlow = _screenRecordActive.asStateFlow() + private val _serverName = MutableStateFlow(null) val serverName: StateFlow = _serverName.asStateFlow() @@ -756,14 +759,20 @@ class NodeRuntime(context: Context) { } } ClawdisScreenCommand.Record.rawValue -> { - val res = - try { - screenRecorder.record(paramsJson) - } catch (err: Throwable) { - val (code, message) = invokeErrorFromThrowable(err) - return BridgeSession.InvokeResult.error(code = code, message = message) - } - BridgeSession.InvokeResult.ok(res.payloadJson) + // Status pill mirrors screen recording state so it stays visible without overlay stacking. + _screenRecordActive.value = true + try { + val res = + try { + screenRecorder.record(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + return BridgeSession.InvokeResult.error(code = code, message = message) + } + BridgeSession.InvokeResult.ok(res.payloadJson) + } finally { + _screenRecordActive.value = false + } } else -> BridgeSession.InvokeResult.error( diff --git a/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt b/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt index 2594449b8..86d5a334e 100644 --- a/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt +++ b/apps/android/app/src/main/java/com/steipete/clawdis/node/ui/RootScreen.kt @@ -36,6 +36,10 @@ import androidx.compose.material.icons.filled.CheckCircle import androidx.compose.material.icons.filled.Error import androidx.compose.material.icons.filled.FiberManualRecord import androidx.compose.material.icons.filled.PhotoCamera +import androidx.compose.material.icons.filled.RecordVoiceOver +import androidx.compose.material.icons.filled.Refresh +import androidx.compose.material.icons.filled.Report +import androidx.compose.material.icons.filled.ScreenShare import androidx.compose.material.icons.filled.Settings import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState @@ -65,39 +69,100 @@ fun RootScreen(viewModel: MainViewModel) { val statusText by viewModel.statusText.collectAsState() val cameraHud by viewModel.cameraHud.collectAsState() val cameraFlashToken by viewModel.cameraFlashToken.collectAsState() + val screenRecordActive by viewModel.screenRecordActive.collectAsState() + val isForeground by viewModel.isForeground.collectAsState() + val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState() val activity = - remember(cameraHud) { - // Status pill owns transient capture state so it doesn't overlap the connection indicator. - cameraHud?.let { hud -> - when (hud.kind) { + remember(cameraHud, screenRecordActive, isForeground, statusText, voiceWakeStatusText) { + // Status pill owns transient activity state so it doesn't overlap the connection indicator. + if (!isForeground) { + return@remember StatusActivity( + title = "Foreground required", + icon = Icons.Default.Report, + contentDescription = "Foreground required", + ) + } + + val lowerStatus = statusText.lowercase() + if (lowerStatus.contains("repair")) { + return@remember StatusActivity( + title = "Repairing…", + icon = Icons.Default.Refresh, + contentDescription = "Repairing", + ) + } + if (lowerStatus.contains("pairing") || lowerStatus.contains("approval")) { + return@remember StatusActivity( + title = "Approval pending", + icon = Icons.Default.RecordVoiceOver, + contentDescription = "Approval pending", + ) + } + if (lowerStatus.contains("reconnecting") || lowerStatus.contains("connecting")) { + return@remember StatusActivity( + title = "Gateway reconnecting…", + icon = Icons.Default.Refresh, + contentDescription = "Gateway reconnecting", + ) + } + + if (screenRecordActive) { + return@remember StatusActivity( + title = "Recording screen…", + icon = Icons.Default.ScreenShare, + contentDescription = "Recording screen", + tint = androidx.compose.ui.graphics.Color.Red, + ) + } + + if (cameraHud != null) { + return@remember when (cameraHud.kind) { CameraHudKind.Photo -> StatusActivity( - title = hud.message, + title = cameraHud.message, icon = Icons.Default.PhotoCamera, contentDescription = "Taking photo", ) CameraHudKind.Recording -> StatusActivity( - title = hud.message, + title = cameraHud.message, icon = Icons.Default.FiberManualRecord, contentDescription = "Recording", tint = androidx.compose.ui.graphics.Color.Red, ) CameraHudKind.Success -> StatusActivity( - title = hud.message, + title = cameraHud.message, icon = Icons.Default.CheckCircle, contentDescription = "Capture finished", ) CameraHudKind.Error -> StatusActivity( - title = hud.message, + title = cameraHud.message, icon = Icons.Default.Error, contentDescription = "Capture failed", tint = androidx.compose.ui.graphics.Color.Red, ) } } + + if (voiceWakeStatusText.contains("Microphone permission", ignoreCase = true)) { + return@remember StatusActivity( + title = "Mic permission", + icon = Icons.Default.Error, + contentDescription = "Mic permission required", + ) + } + if (voiceWakeStatusText == "Paused") { + val suffix = if (!isForeground) " (background)" else "" + return@remember StatusActivity( + title = "Voice Wake paused$suffix", + icon = Icons.Default.RecordVoiceOver, + contentDescription = "Voice Wake paused", + ) + } + + null } val bridgeState = diff --git a/apps/ios/Sources/Model/NodeAppModel.swift b/apps/ios/Sources/Model/NodeAppModel.swift index 4c491ea55..8c2935ffc 100644 --- a/apps/ios/Sources/Model/NodeAppModel.swift +++ b/apps/ios/Sources/Model/NodeAppModel.swift @@ -36,6 +36,7 @@ final class NodeAppModel { var cameraHUDText: String? var cameraHUDKind: CameraHUDKind? var cameraFlashNonce: Int = 0 + var screenRecordActive: Bool = false init() { self.voiceWake.configure { [weak self] cmd in @@ -598,6 +599,9 @@ final class NodeAppModel { NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4", ]) } + // Status pill mirrors screen recording state so it stays visible without overlay stacking. + self.screenRecordActive = true + defer { self.screenRecordActive = false } let path = try await self.screenRecorder.record( screenIndex: params.screenIndex, durationMs: params.durationMs, diff --git a/apps/ios/Sources/RootCanvas.swift b/apps/ios/Sources/RootCanvas.swift index c02eceb69..b55f84cc1 100644 --- a/apps/ios/Sources/RootCanvas.swift +++ b/apps/ios/Sources/RootCanvas.swift @@ -119,6 +119,7 @@ struct RootCanvas: View { } private struct CanvasContent: View { + @Environment(NodeAppModel.self) private var appModel var systemColorScheme: ColorScheme var bridgeStatus: StatusPill.BridgeState var voiceWakeEnabled: Bool @@ -173,28 +174,63 @@ private struct CanvasContent: View { } private var statusActivity: StatusPill.Activity? { - // Status pill owns transient capture state so it doesn't overlap the connection indicator. - guard let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind else { return nil } - let systemImage: String - let tint: Color? - switch cameraHUDKind { - case .photo: - systemImage = "camera.fill" - tint = nil - case .recording: - systemImage = "video.fill" - tint = .red - case .success: - systemImage = "checkmark.circle.fill" - tint = .green - case .error: - systemImage = "exclamationmark.triangle.fill" - tint = .red + // Status pill owns transient activity state so it doesn't overlap the connection indicator. + if self.appModel.isBackgrounded { + return StatusPill.Activity( + title: "Foreground required", + systemImage: "exclamationmark.triangle.fill", + tint: .orange) } - return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint) - } + let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines) + let bridgeLower = bridgeStatus.lowercased() + if bridgeLower.contains("repair") { + return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange) + } + if bridgeLower.contains("approval") || bridgeLower.contains("pairing") { + return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock") + } + if bridgeLower.contains("reconnecting") || bridgeLower.contains("connecting") { + return StatusPill.Activity(title: "Gateway reconnecting…", systemImage: "arrow.triangle.2.circlepath") + } + if self.appModel.screenRecordActive { + return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red) + } + + if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind { + let systemImage: String + let tint: Color? + switch cameraHUDKind { + case .photo: + systemImage = "camera.fill" + tint = nil + case .recording: + systemImage = "video.fill" + tint = .red + case .success: + systemImage = "checkmark.circle.fill" + tint = .green + case .error: + systemImage = "exclamationmark.triangle.fill" + tint = .red + } + return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint) + } + + if self.voiceWakeEnabled { + let voiceStatus = self.appModel.voiceWake.statusText + if voiceStatus.localizedCaseInsensitiveContains("microphone permission") { + return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange) + } + if voiceStatus == "Paused" { + let suffix = self.appModel.isBackgrounded ? " (background)" : "" + return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill") + } + } + + return nil + } } private struct OverlayButton: View { diff --git a/apps/ios/Sources/RootTabs.swift b/apps/ios/Sources/RootTabs.swift index 913073d4a..a480fc8ab 100644 --- a/apps/ios/Sources/RootTabs.swift +++ b/apps/ios/Sources/RootTabs.swift @@ -26,7 +26,7 @@ struct RootTabs: View { StatusPill( bridge: self.bridgeStatus, voiceWakeEnabled: self.voiceWakeEnabled, - activity: nil, + activity: self.statusActivity, onTap: { self.selectedTab = 2 }) .padding(.leading, 10) .safeAreaPadding(.top, 10) @@ -80,4 +80,66 @@ struct RootTabs: View { return .disconnected } + + private var statusActivity: StatusPill.Activity? { + // Keep the top pill consistent across tabs (camera + voice wake + pairing states). + if self.appModel.isBackgrounded { + return StatusPill.Activity( + title: "Foreground required", + systemImage: "exclamationmark.triangle.fill", + tint: .orange) + } + + let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines) + let bridgeLower = bridgeStatus.lowercased() + if bridgeLower.contains("repair") { + return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange) + } + if bridgeLower.contains("approval") || bridgeLower.contains("pairing") { + return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock") + } + if bridgeLower.contains("reconnecting") || bridgeLower.contains("connecting") { + return StatusPill.Activity(title: "Gateway reconnecting…", systemImage: "arrow.triangle.2.circlepath") + } + + if self.appModel.screenRecordActive { + return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red) + } + + if let cameraHUDText = self.appModel.cameraHUDText, + let cameraHUDKind = self.appModel.cameraHUDKind, + !cameraHUDText.isEmpty + { + let systemImage: String + let tint: Color? + switch cameraHUDKind { + case .photo: + systemImage = "camera.fill" + tint = nil + case .recording: + systemImage = "video.fill" + tint = .red + case .success: + systemImage = "checkmark.circle.fill" + tint = .green + case .error: + systemImage = "exclamationmark.triangle.fill" + tint = .red + } + return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint) + } + + if self.voiceWakeEnabled { + let voiceStatus = self.appModel.voiceWake.statusText + if voiceStatus.localizedCaseInsensitiveContains("microphone permission") { + return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange) + } + if voiceStatus == "Paused" { + let suffix = self.appModel.isBackgrounded ? " (background)" : "" + return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill") + } + } + + return nil + } } diff --git a/apps/macos/Sources/Clawdis/MenuContentView.swift b/apps/macos/Sources/Clawdis/MenuContentView.swift index 748ce018d..dee70ed5d 100644 --- a/apps/macos/Sources/Clawdis/MenuContentView.swift +++ b/apps/macos/Sources/Clawdis/MenuContentView.swift @@ -14,6 +14,7 @@ struct MenuContent: View { private let heartbeatStore = HeartbeatStore.shared private let controlChannel = ControlChannel.shared private let activityStore = WorkActivityStore.shared + @Bindable private var pairingPrompter = NodePairingApprovalPrompter.shared @Environment(\.openSettings) private var openSettings @State private var availableMics: [AudioInputDevice] = [] @State private var loadingMics = false @@ -32,6 +33,13 @@ struct MenuContent: View { VStack(alignment: .leading, spacing: 2) { Text(self.connectionLabel) self.statusLine(label: self.healthStatus.label, color: self.healthStatus.color) + if self.pairingPrompter.pendingCount > 0 { + let repairCount = self.pairingPrompter.pendingRepairCount + let repairSuffix = repairCount > 0 ? " · \(repairCount) repair" : "" + self.statusLine( + label: "Pairing approval pending (\(self.pairingPrompter.pendingCount))\(repairSuffix)", + color: .orange) + } } } .disabled(self.state.connectionMode == .unconfigured) diff --git a/apps/macos/Sources/Clawdis/NodePairingApprovalPrompter.swift b/apps/macos/Sources/Clawdis/NodePairingApprovalPrompter.swift index 85d8a9f12..932f272f6 100644 --- a/apps/macos/Sources/Clawdis/NodePairingApprovalPrompter.swift +++ b/apps/macos/Sources/Clawdis/NodePairingApprovalPrompter.swift @@ -2,6 +2,7 @@ import AppKit import ClawdisIPC import ClawdisProtocol import Foundation +import Observation import OSLog import UserNotifications @@ -15,6 +16,7 @@ enum NodePairingReconcilePolicy { } @MainActor +@Observable final class NodePairingApprovalPrompter { static let shared = NodePairingApprovalPrompter() @@ -26,6 +28,8 @@ final class NodePairingApprovalPrompter { private var isStopping = false private var isPresenting = false private var queue: [PendingRequest] = [] + var pendingCount: Int = 0 + var pendingRepairCount: Int = 0 private var activeAlert: NSAlert? private var activeRequestId: String? private var alertHostWindow: NSWindow? @@ -104,6 +108,7 @@ final class NodePairingApprovalPrompter { self.reconcileOnceTask?.cancel() self.reconcileOnceTask = nil self.queue.removeAll(keepingCapacity: false) + self.updatePendingCounts() self.isPresenting = false self.activeRequestId = nil self.alertHostWindow?.orderOut(nil) @@ -292,6 +297,7 @@ final class NodePairingApprovalPrompter { private func enqueue(_ req: PendingRequest) { if self.queue.contains(req) { return } self.queue.append(req) + self.updatePendingCounts() self.presentNextIfNeeded() self.updateReconcileLoop() } @@ -362,6 +368,7 @@ final class NodePairingApprovalPrompter { } else { self.queue.removeAll { $0 == request } } + self.updatePendingCounts() self.isPresenting = false self.presentNextIfNeeded() self.updateReconcileLoop() @@ -501,6 +508,8 @@ final class NodePairingApprovalPrompter { } else { self.queue.removeAll { $0 == req } } + + self.updatePendingCounts() self.isPresenting = false self.presentNextIfNeeded() self.updateReconcileLoop() @@ -599,6 +608,12 @@ final class NodePairingApprovalPrompter { } } + private func updatePendingCounts() { + // Keep a cheap observable summary for the menu bar status line. + self.pendingCount = self.queue.count + self.pendingRepairCount = self.queue.filter { $0.isRepair == true }.count + } + private func reconcileOnce(timeoutMs: Double) async { if self.isStopping { return } if self.reconcileInFlight { return } @@ -643,6 +658,7 @@ final class NodePairingApprovalPrompter { return } self.queue.removeAll { $0.requestId == resolved.requestId } + self.updatePendingCounts() Task { @MainActor in await self.notify(resolution: resolution, request: request, via: "remote") }