feat: extend status activity indicators

This commit is contained in:
Peter Steinberger
2025-12-29 23:42:22 +01:00
parent 3c338d1858
commit 303954ae8c
9 changed files with 241 additions and 36 deletions

View File

@@ -14,6 +14,8 @@
- iOS/Android nodes: bridge auto-connect refreshes stale tokens and settings now show richer bridge/device details. - iOS/Android nodes: bridge auto-connect refreshes stale tokens and settings now show richer bridge/device details.
- iOS/Android nodes: status pill now surfaces camera activity instead of overlay toasts. - iOS/Android nodes: status pill now surfaces camera activity instead of overlay toasts.
- iOS/Android/macOS nodes: camera snaps recompress to keep base64 payloads under 5 MB. - iOS/Android/macOS nodes: camera snaps recompress to keep base64 payloads under 5 MB.
- iOS/Android nodes: status pill now surfaces pairing, screen recording, voice wake, and foreground-required states.
- macOS menu: top status line now shows pending node pairing approvals (incl. repairs).
- CLI: avoid spurious gateway close errors after successful request/response cycles. - CLI: avoid spurious gateway close errors after successful request/response cycles.
- Agent runtime: clamp tool-result images to the 5MB Anthropic limit to avoid hard request rejections. - Agent runtime: clamp tool-result images to the 5MB Anthropic limit to avoid hard request rejections.
- Tests: add Swift Testing coverage for camera errors and Kotest coverage for Android bridge endpoints. - Tests: add Swift Testing coverage for camera errors and Kotest coverage for Android bridge endpoints.

View File

@@ -23,9 +23,11 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val statusText: StateFlow<String> = runtime.statusText val statusText: StateFlow<String> = runtime.statusText
val serverName: StateFlow<String?> = runtime.serverName val serverName: StateFlow<String?> = runtime.serverName
val remoteAddress: StateFlow<String?> = runtime.remoteAddress val remoteAddress: StateFlow<String?> = runtime.remoteAddress
val isForeground: StateFlow<Boolean> = runtime.isForeground
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
val instanceId: StateFlow<String> = runtime.instanceId val instanceId: StateFlow<String> = runtime.instanceId
val displayName: StateFlow<String> = runtime.displayName val displayName: StateFlow<String> = runtime.displayName
@@ -34,6 +36,7 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val wakeWords: StateFlow<List<String>> = runtime.wakeWords val wakeWords: StateFlow<List<String>> = runtime.wakeWords
val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening
val talkEnabled: StateFlow<Boolean> = runtime.talkEnabled val talkEnabled: StateFlow<Boolean> = runtime.talkEnabled
val talkStatusText: StateFlow<String> = runtime.talkStatusText val talkStatusText: StateFlow<String> = runtime.talkStatusText

View File

@@ -111,6 +111,9 @@ class NodeRuntime(context: Context) {
private val _cameraFlashToken = MutableStateFlow(0L) private val _cameraFlashToken = MutableStateFlow(0L)
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow() val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
private val _screenRecordActive = MutableStateFlow(false)
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
private val _serverName = MutableStateFlow<String?>(null) private val _serverName = MutableStateFlow<String?>(null)
val serverName: StateFlow<String?> = _serverName.asStateFlow() val serverName: StateFlow<String?> = _serverName.asStateFlow()
@@ -756,6 +759,9 @@ class NodeRuntime(context: Context) {
} }
} }
ClawdisScreenCommand.Record.rawValue -> { ClawdisScreenCommand.Record.rawValue -> {
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
_screenRecordActive.value = true
try {
val res = val res =
try { try {
screenRecorder.record(paramsJson) screenRecorder.record(paramsJson)
@@ -764,6 +770,9 @@ class NodeRuntime(context: Context) {
return BridgeSession.InvokeResult.error(code = code, message = message) return BridgeSession.InvokeResult.error(code = code, message = message)
} }
BridgeSession.InvokeResult.ok(res.payloadJson) BridgeSession.InvokeResult.ok(res.payloadJson)
} finally {
_screenRecordActive.value = false
}
} }
else -> else ->
BridgeSession.InvokeResult.error( BridgeSession.InvokeResult.error(

View File

@@ -36,6 +36,10 @@ import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.Error import androidx.compose.material.icons.filled.Error
import androidx.compose.material.icons.filled.FiberManualRecord import androidx.compose.material.icons.filled.FiberManualRecord
import androidx.compose.material.icons.filled.PhotoCamera import androidx.compose.material.icons.filled.PhotoCamera
import androidx.compose.material.icons.filled.RecordVoiceOver
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Report
import androidx.compose.material.icons.filled.ScreenShare
import androidx.compose.material.icons.filled.Settings import androidx.compose.material.icons.filled.Settings
import androidx.compose.runtime.Composable import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState import androidx.compose.runtime.collectAsState
@@ -65,39 +69,100 @@ fun RootScreen(viewModel: MainViewModel) {
val statusText by viewModel.statusText.collectAsState() val statusText by viewModel.statusText.collectAsState()
val cameraHud by viewModel.cameraHud.collectAsState() val cameraHud by viewModel.cameraHud.collectAsState()
val cameraFlashToken by viewModel.cameraFlashToken.collectAsState() val cameraFlashToken by viewModel.cameraFlashToken.collectAsState()
val screenRecordActive by viewModel.screenRecordActive.collectAsState()
val isForeground by viewModel.isForeground.collectAsState()
val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState()
val activity = val activity =
remember(cameraHud) { remember(cameraHud, screenRecordActive, isForeground, statusText, voiceWakeStatusText) {
// Status pill owns transient capture state so it doesn't overlap the connection indicator. // Status pill owns transient activity state so it doesn't overlap the connection indicator.
cameraHud?.let { hud -> if (!isForeground) {
when (hud.kind) { return@remember StatusActivity(
title = "Foreground required",
icon = Icons.Default.Report,
contentDescription = "Foreground required",
)
}
val lowerStatus = statusText.lowercase()
if (lowerStatus.contains("repair")) {
return@remember StatusActivity(
title = "Repairing…",
icon = Icons.Default.Refresh,
contentDescription = "Repairing",
)
}
if (lowerStatus.contains("pairing") || lowerStatus.contains("approval")) {
return@remember StatusActivity(
title = "Approval pending",
icon = Icons.Default.RecordVoiceOver,
contentDescription = "Approval pending",
)
}
if (lowerStatus.contains("reconnecting") || lowerStatus.contains("connecting")) {
return@remember StatusActivity(
title = "Gateway reconnecting…",
icon = Icons.Default.Refresh,
contentDescription = "Gateway reconnecting",
)
}
if (screenRecordActive) {
return@remember StatusActivity(
title = "Recording screen…",
icon = Icons.Default.ScreenShare,
contentDescription = "Recording screen",
tint = androidx.compose.ui.graphics.Color.Red,
)
}
if (cameraHud != null) {
return@remember when (cameraHud.kind) {
CameraHudKind.Photo -> CameraHudKind.Photo ->
StatusActivity( StatusActivity(
title = hud.message, title = cameraHud.message,
icon = Icons.Default.PhotoCamera, icon = Icons.Default.PhotoCamera,
contentDescription = "Taking photo", contentDescription = "Taking photo",
) )
CameraHudKind.Recording -> CameraHudKind.Recording ->
StatusActivity( StatusActivity(
title = hud.message, title = cameraHud.message,
icon = Icons.Default.FiberManualRecord, icon = Icons.Default.FiberManualRecord,
contentDescription = "Recording", contentDescription = "Recording",
tint = androidx.compose.ui.graphics.Color.Red, tint = androidx.compose.ui.graphics.Color.Red,
) )
CameraHudKind.Success -> CameraHudKind.Success ->
StatusActivity( StatusActivity(
title = hud.message, title = cameraHud.message,
icon = Icons.Default.CheckCircle, icon = Icons.Default.CheckCircle,
contentDescription = "Capture finished", contentDescription = "Capture finished",
) )
CameraHudKind.Error -> CameraHudKind.Error ->
StatusActivity( StatusActivity(
title = hud.message, title = cameraHud.message,
icon = Icons.Default.Error, icon = Icons.Default.Error,
contentDescription = "Capture failed", contentDescription = "Capture failed",
tint = androidx.compose.ui.graphics.Color.Red, tint = androidx.compose.ui.graphics.Color.Red,
) )
} }
} }
if (voiceWakeStatusText.contains("Microphone permission", ignoreCase = true)) {
return@remember StatusActivity(
title = "Mic permission",
icon = Icons.Default.Error,
contentDescription = "Mic permission required",
)
}
if (voiceWakeStatusText == "Paused") {
val suffix = if (!isForeground) " (background)" else ""
return@remember StatusActivity(
title = "Voice Wake paused$suffix",
icon = Icons.Default.RecordVoiceOver,
contentDescription = "Voice Wake paused",
)
}
null
} }
val bridgeState = val bridgeState =

View File

@@ -36,6 +36,7 @@ final class NodeAppModel {
var cameraHUDText: String? var cameraHUDText: String?
var cameraHUDKind: CameraHUDKind? var cameraHUDKind: CameraHUDKind?
var cameraFlashNonce: Int = 0 var cameraFlashNonce: Int = 0
var screenRecordActive: Bool = false
init() { init() {
self.voiceWake.configure { [weak self] cmd in self.voiceWake.configure { [weak self] cmd in
@@ -598,6 +599,9 @@ final class NodeAppModel {
NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4", NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4",
]) ])
} }
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
self.screenRecordActive = true
defer { self.screenRecordActive = false }
let path = try await self.screenRecorder.record( let path = try await self.screenRecorder.record(
screenIndex: params.screenIndex, screenIndex: params.screenIndex,
durationMs: params.durationMs, durationMs: params.durationMs,

View File

@@ -119,6 +119,7 @@ struct RootCanvas: View {
} }
private struct CanvasContent: View { private struct CanvasContent: View {
@Environment(NodeAppModel.self) private var appModel
var systemColorScheme: ColorScheme var systemColorScheme: ColorScheme
var bridgeStatus: StatusPill.BridgeState var bridgeStatus: StatusPill.BridgeState
var voiceWakeEnabled: Bool var voiceWakeEnabled: Bool
@@ -173,8 +174,31 @@ private struct CanvasContent: View {
} }
private var statusActivity: StatusPill.Activity? { private var statusActivity: StatusPill.Activity? {
// Status pill owns transient capture state so it doesn't overlap the connection indicator. // Status pill owns transient activity state so it doesn't overlap the connection indicator.
guard let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind else { return nil } if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
}
let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let bridgeLower = bridgeStatus.lowercased()
if bridgeLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if bridgeLower.contains("approval") || bridgeLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
if bridgeLower.contains("reconnecting") || bridgeLower.contains("connecting") {
return StatusPill.Activity(title: "Gateway reconnecting…", systemImage: "arrow.triangle.2.circlepath")
}
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
let systemImage: String let systemImage: String
let tint: Color? let tint: Color?
switch cameraHUDKind { switch cameraHUDKind {
@@ -191,10 +215,22 @@ private struct CanvasContent: View {
systemImage = "exclamationmark.triangle.fill" systemImage = "exclamationmark.triangle.fill"
tint = .red tint = .red
} }
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint) return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
} }
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
} }
private struct OverlayButton: View { private struct OverlayButton: View {

View File

@@ -26,7 +26,7 @@ struct RootTabs: View {
StatusPill( StatusPill(
bridge: self.bridgeStatus, bridge: self.bridgeStatus,
voiceWakeEnabled: self.voiceWakeEnabled, voiceWakeEnabled: self.voiceWakeEnabled,
activity: nil, activity: self.statusActivity,
onTap: { self.selectedTab = 2 }) onTap: { self.selectedTab = 2 })
.padding(.leading, 10) .padding(.leading, 10)
.safeAreaPadding(.top, 10) .safeAreaPadding(.top, 10)
@@ -80,4 +80,66 @@ struct RootTabs: View {
return .disconnected return .disconnected
} }
private var statusActivity: StatusPill.Activity? {
// Keep the top pill consistent across tabs (camera + voice wake + pairing states).
if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
}
let bridgeStatus = self.appModel.bridgeStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let bridgeLower = bridgeStatus.lowercased()
if bridgeLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if bridgeLower.contains("approval") || bridgeLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
if bridgeLower.contains("reconnecting") || bridgeLower.contains("connecting") {
return StatusPill.Activity(title: "Gateway reconnecting…", systemImage: "arrow.triangle.2.circlepath")
}
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText = self.appModel.cameraHUDText,
let cameraHUDKind = self.appModel.cameraHUDKind,
!cameraHUDText.isEmpty
{
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
} }

View File

@@ -14,6 +14,7 @@ struct MenuContent: View {
private let heartbeatStore = HeartbeatStore.shared private let heartbeatStore = HeartbeatStore.shared
private let controlChannel = ControlChannel.shared private let controlChannel = ControlChannel.shared
private let activityStore = WorkActivityStore.shared private let activityStore = WorkActivityStore.shared
@Bindable private var pairingPrompter = NodePairingApprovalPrompter.shared
@Environment(\.openSettings) private var openSettings @Environment(\.openSettings) private var openSettings
@State private var availableMics: [AudioInputDevice] = [] @State private var availableMics: [AudioInputDevice] = []
@State private var loadingMics = false @State private var loadingMics = false
@@ -32,6 +33,13 @@ struct MenuContent: View {
VStack(alignment: .leading, spacing: 2) { VStack(alignment: .leading, spacing: 2) {
Text(self.connectionLabel) Text(self.connectionLabel)
self.statusLine(label: self.healthStatus.label, color: self.healthStatus.color) self.statusLine(label: self.healthStatus.label, color: self.healthStatus.color)
if self.pairingPrompter.pendingCount > 0 {
let repairCount = self.pairingPrompter.pendingRepairCount
let repairSuffix = repairCount > 0 ? " · \(repairCount) repair" : ""
self.statusLine(
label: "Pairing approval pending (\(self.pairingPrompter.pendingCount))\(repairSuffix)",
color: .orange)
}
} }
} }
.disabled(self.state.connectionMode == .unconfigured) .disabled(self.state.connectionMode == .unconfigured)

View File

@@ -2,6 +2,7 @@ import AppKit
import ClawdisIPC import ClawdisIPC
import ClawdisProtocol import ClawdisProtocol
import Foundation import Foundation
import Observation
import OSLog import OSLog
import UserNotifications import UserNotifications
@@ -15,6 +16,7 @@ enum NodePairingReconcilePolicy {
} }
@MainActor @MainActor
@Observable
final class NodePairingApprovalPrompter { final class NodePairingApprovalPrompter {
static let shared = NodePairingApprovalPrompter() static let shared = NodePairingApprovalPrompter()
@@ -26,6 +28,8 @@ final class NodePairingApprovalPrompter {
private var isStopping = false private var isStopping = false
private var isPresenting = false private var isPresenting = false
private var queue: [PendingRequest] = [] private var queue: [PendingRequest] = []
var pendingCount: Int = 0
var pendingRepairCount: Int = 0
private var activeAlert: NSAlert? private var activeAlert: NSAlert?
private var activeRequestId: String? private var activeRequestId: String?
private var alertHostWindow: NSWindow? private var alertHostWindow: NSWindow?
@@ -104,6 +108,7 @@ final class NodePairingApprovalPrompter {
self.reconcileOnceTask?.cancel() self.reconcileOnceTask?.cancel()
self.reconcileOnceTask = nil self.reconcileOnceTask = nil
self.queue.removeAll(keepingCapacity: false) self.queue.removeAll(keepingCapacity: false)
self.updatePendingCounts()
self.isPresenting = false self.isPresenting = false
self.activeRequestId = nil self.activeRequestId = nil
self.alertHostWindow?.orderOut(nil) self.alertHostWindow?.orderOut(nil)
@@ -292,6 +297,7 @@ final class NodePairingApprovalPrompter {
private func enqueue(_ req: PendingRequest) { private func enqueue(_ req: PendingRequest) {
if self.queue.contains(req) { return } if self.queue.contains(req) { return }
self.queue.append(req) self.queue.append(req)
self.updatePendingCounts()
self.presentNextIfNeeded() self.presentNextIfNeeded()
self.updateReconcileLoop() self.updateReconcileLoop()
} }
@@ -362,6 +368,7 @@ final class NodePairingApprovalPrompter {
} else { } else {
self.queue.removeAll { $0 == request } self.queue.removeAll { $0 == request }
} }
self.updatePendingCounts()
self.isPresenting = false self.isPresenting = false
self.presentNextIfNeeded() self.presentNextIfNeeded()
self.updateReconcileLoop() self.updateReconcileLoop()
@@ -501,6 +508,8 @@ final class NodePairingApprovalPrompter {
} else { } else {
self.queue.removeAll { $0 == req } self.queue.removeAll { $0 == req }
} }
self.updatePendingCounts()
self.isPresenting = false self.isPresenting = false
self.presentNextIfNeeded() self.presentNextIfNeeded()
self.updateReconcileLoop() self.updateReconcileLoop()
@@ -599,6 +608,12 @@ final class NodePairingApprovalPrompter {
} }
} }
private func updatePendingCounts() {
// Keep a cheap observable summary for the menu bar status line.
self.pendingCount = self.queue.count
self.pendingRepairCount = self.queue.filter { $0.isRepair == true }.count
}
private func reconcileOnce(timeoutMs: Double) async { private func reconcileOnce(timeoutMs: Double) async {
if self.isStopping { return } if self.isStopping { return }
if self.reconcileInFlight { return } if self.reconcileInFlight { return }
@@ -643,6 +658,7 @@ final class NodePairingApprovalPrompter {
return return
} }
self.queue.removeAll { $0.requestId == resolved.requestId } self.queue.removeAll { $0.requestId == resolved.requestId }
self.updatePendingCounts()
Task { @MainActor in Task { @MainActor in
await self.notify(resolution: resolution, request: request, via: "remote") await self.notify(resolution: resolution, request: request, via: "remote")
} }