let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(discovery): bonjour beacons + bridge presence

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-13 04:28:12 +00:00
parent 3ee0e041fa
commit 1f37d94f9e
49 changed files with 1182 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
apps/ios/Sources/Bridge/BonjourEscapeDecoder.swift Normal file
View File

@@ -0,0 +1,34 @@
import Foundation
enum BonjourEscapeDecoder {
static func decode(_ input: String) -> String {
// mDNS / DNS-SD commonly escapes bytes in instance names as `\\DDD`
// (decimal-encoded), e.g. spaces are `\\032`.
var out = ""
var i = input.startIndex
while i < input.endIndex {
if input[i] == "\\",
let d0 = input.index(i, offsetBy: 1, limitedBy: input.index(before: input.endIndex)),
let d1 = input.index(i, offsetBy: 2, limitedBy: input.index(before: input.endIndex)),
let d2 = input.index(i, offsetBy: 3, limitedBy: input.index(before: input.endIndex)),
input[d0].isNumber,
input[d1].isNumber,
input[d2].isNumber
{
let digits = String(input[d0...d2])
if let value = Int(digits),
let scalar = UnicodeScalar(value)
{
out.append(Character(scalar))
i = input.index(i, offsetBy: 4)
continue
}
}
out.append(input[i])
i = input.index(after: i)
}
return out
}
}

34
apps/ios/Sources/Bridge/BridgeDiscoveryModel.swift
View File

@@ -50,8 +50,9 @@ final class BridgeDiscoveryModel: ObservableObject {
self.bridges = results.compactMap { result -> DiscoveredBridge? in
switch result.endpoint {
case let .service(name, _, _, _):
let decodedName = BonjourEscapeDecoder.decode(name)
return DiscoveredBridge(
name: name,
name: decodedName,
endpoint: result.endpoint,
debugID: Self.prettyEndpointDebugID(result.endpoint))
default:
@@ -74,35 +75,6 @@ final class BridgeDiscoveryModel: ObservableObject {
}
private static func prettyEndpointDebugID(_ endpoint: NWEndpoint) -> String {
self.decodeBonjourEscapes(String(describing: endpoint))
}
private static func decodeBonjourEscapes(_ input: String) -> String {
// mDNS / DNS-SD commonly escapes spaces as `\\032` (decimal byte value 32). Make this human-friendly for UI.
var out = ""
var i = input.startIndex
while i < input.endIndex {
if input[i] == "\\",
let d0 = input.index(i, offsetBy: 1, limitedBy: input.index(before: input.endIndex)),
let d1 = input.index(i, offsetBy: 2, limitedBy: input.index(before: input.endIndex)),
let d2 = input.index(i, offsetBy: 3, limitedBy: input.index(before: input.endIndex)),
input[d0].isNumber,
input[d1].isNumber,
input[d2].isNumber
{
let digits = String(input[d0...d2])
if let value = Int(digits),
let scalar = UnicodeScalar(value)
{
out.append(Character(scalar))
i = input.index(i, offsetBy: 4)
continue
}
}
out.append(input[i])
i = input.index(after: i)
}
return out
BonjourEscapeDecoder.decode(String(describing: endpoint))
}
}

14
apps/ios/Sources/Bridge/BridgeSession.swift
View File

@@ -19,6 +19,20 @@ actor BridgeSession {
private(set) var state: State = .idle
func currentRemoteAddress() -> String? {
guard let endpoint = self.connection?.currentPath?.remoteEndpoint else { return nil }
return Self.prettyRemoteEndpoint(endpoint)
}
private static func prettyRemoteEndpoint(_ endpoint: NWEndpoint) -> String? {
switch endpoint {
case let .hostPort(host, port):
return "\(host):\(port)".replacingOccurrences(of: "::ffff:", with: "")
default:
return String(describing: endpoint)
}
}
func connect(
endpoint: NWEndpoint,
hello: BridgeHello,

15
apps/ios/Sources/Model/NodeAppModel.swift
View File

@@ -8,6 +8,8 @@ final class NodeAppModel: ObservableObject {
let screen = ScreenController()
@Published var bridgeStatusText: String = "Not connected"
@Published var bridgeServerName: String?
@Published var bridgeRemoteAddress: String?
@Published var connectedBridgeDebugID: String?
private let bridge = BridgeSession()
private var bridgeTask: Task<Void, Never>?
@@ -55,6 +57,8 @@ final class NodeAppModel: ObservableObject {
self.bridgeTask?.cancel()
self.bridgeStatusText = "Connecting…"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeDebugID = BonjourEscapeDecoder.decode(String(describing: endpoint))
self.bridgeTask = Task {
do {
@@ -71,6 +75,11 @@ final class NodeAppModel: ObservableObject {
self?.bridgeStatusText = "Connected"
self?.bridgeServerName = serverName
}
if let addr = await self.bridge.currentRemoteAddress() {
await MainActor.run {
self?.bridgeRemoteAddress = addr
}
}
},
onInvoke: { [weak self] req in
guard let self else {
@@ -85,11 +94,15 @@ final class NodeAppModel: ObservableObject {
await MainActor.run {
self.bridgeStatusText = "Disconnected"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeDebugID = nil
}
} catch {
await MainActor.run {
self.bridgeStatusText = "Bridge error: \(error.localizedDescription)"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeDebugID = nil
}
}
}
@@ -101,6 +114,8 @@ final class NodeAppModel: ObservableObject {
Task { await self.bridge.disconnect() }
self.bridgeStatusText = "Disconnected"
self.bridgeServerName = nil
self.bridgeRemoteAddress = nil
self.connectedBridgeDebugID = nil
}
func sendVoiceTranscript(text: String, sessionKey: String?) async throws {

1
apps/ios/Sources/RootCanvas.swift
View File

@@ -45,5 +45,6 @@ struct RootCanvas: View {
.sheet(isPresented: self.$isShowingSettings) {
SettingsTab()
}
.preferredColorScheme(.dark)
}
}

3
apps/ios/Sources/Screen/ScreenController.swift
View File

@@ -14,6 +14,9 @@ final class ScreenController: ObservableObject {
let config = WKWebViewConfiguration()
config.websiteDataStore = .nonPersistent()
self.webView = WKWebView(frame: .zero, configuration: config)
self.webView.isOpaque = false
self.webView.backgroundColor = .clear
self.webView.scrollView.backgroundColor = .clear
self.webView.scrollView.contentInsetAdjustmentBehavior = .never
self.webView.scrollView.contentInset = .zero
self.webView.scrollView.scrollIndicatorInsets = .zero

56
apps/ios/Sources/Settings/SettingsTab.swift
View File

@@ -10,7 +10,6 @@ struct SettingsTab: View {
@State private var connectStatus: String?
@State private var isConnecting = false
@State private var didAutoConnect = false
@State private var isShowingBridgeList = false
var body: some View {
NavigationStack {
@@ -34,16 +33,17 @@ struct SettingsTab: View {
LabeledContent("Status", value: self.appModel.bridgeStatusText)
if let serverName = self.appModel.bridgeServerName {
LabeledContent("Server", value: serverName)
if let addr = self.appModel.bridgeRemoteAddress {
LabeledContent("Address", value: addr)
}
Button("Disconnect", role: .destructive) {
self.appModel.disconnectBridge()
}
DisclosureGroup("Switch bridge", isExpanded: self.$isShowingBridgeList) {
self.bridgeList(showConnectedRow: true)
}
self.bridgeList(showing: .availableOnly)
} else {
self.bridgeList(showConnectedRow: false)
self.bridgeList(showing: .all)
}
if let connectStatus {
@@ -88,22 +88,32 @@ struct SettingsTab: View {
}
.onChange(of: self.appModel.bridgeServerName) { _, _ in
self.connectStatus = nil
self.isShowingBridgeList = false
}
}
}
@ViewBuilder
private func bridgeList(showConnectedRow: Bool) -> some View {
private func bridgeList(showing: BridgeListMode) -> some View {
if self.discovery.bridges.isEmpty {
Text("No bridges found yet.")
.foregroundStyle(.secondary)
} else {
ForEach(self.discovery.bridges) { bridge in
let isConnected = self.isConnectedBridge(bridge)
if isConnected, !showConnectedRow {
EmptyView()
} else {
let connectedID = self.appModel.connectedBridgeDebugID
let rows = self.discovery.bridges.filter { bridge in
let isConnected = bridge.debugID == connectedID
switch showing {
case .all:
return true
case .availableOnly:
return !isConnected
}
}
if rows.isEmpty, showing == .availableOnly {
Text("No other bridges found.")
.foregroundStyle(.secondary)
} else {
ForEach(rows) { bridge in
HStack {
VStack(alignment: .leading, spacing: 2) {
Text(bridge.name)
@@ -114,29 +124,19 @@ struct SettingsTab: View {
}
Spacer()
if isConnected {
Image(systemName: "checkmark.circle.fill")
.foregroundStyle(.green)
.accessibilityLabel("Connected")
} else {
Button(self.isConnecting ? "…" : "Connect") {
Task { await self.connect(bridge) }
}
.disabled(self.isConnecting)
Button(self.isConnecting ? "…" : "Connect") {
Task { await self.connect(bridge) }
}
.disabled(self.isConnecting)
}
}
}
}
}
private func isConnectedBridge(_ bridge: BridgeDiscoveryModel.DiscoveredBridge) -> Bool {
guard let serverName = self.appModel.bridgeServerName?.trimmingCharacters(in: .whitespacesAndNewlines),
!serverName.isEmpty
else {
return false
}
return bridge.name.localizedCaseInsensitiveContains(serverName)
private enum BridgeListMode: Equatable {
case all
case availableOnly
}
private func keychainAccount() -> String {

116
apps/ios/Sources/Voice/VoiceWakeManager.swift
View File

@@ -2,23 +2,69 @@ import AVFAudio
import Foundation
import Speech
enum SpeechAudioTapFactory {
static func makeAppendTap(requestBox: SpeechRequestBox) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
{ buffer, _ in
requestBox.append(buffer)
}
private final class AudioBufferQueue: @unchecked Sendable {
private let lock = NSLock()
private var buffers: [AVAudioPCMBuffer] = []
func enqueueCopy(of buffer: AVAudioPCMBuffer) {
guard let copy = buffer.deepCopy() else { return }
self.lock.lock()
self.buffers.append(copy)
self.lock.unlock()
}
func drain() -> [AVAudioPCMBuffer] {
self.lock.lock()
let drained = self.buffers
self.buffers.removeAll(keepingCapacity: true)
self.lock.unlock()
return drained
}
func clear() {
self.lock.lock()
self.buffers.removeAll(keepingCapacity: false)
self.lock.unlock()
}
}
final class SpeechRequestBox: @unchecked Sendable {
let request: SFSpeechAudioBufferRecognitionRequest
private extension AVAudioPCMBuffer {
func deepCopy() -> AVAudioPCMBuffer? {
let format = self.format
let frameLength = self.frameLength
guard let copy = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameLength) else {
return nil
}
copy.frameLength = frameLength
init(request: SFSpeechAudioBufferRecognitionRequest) {
self.request = request
}
if let src = self.floatChannelData, let dst = copy.floatChannelData {
let channels = Int(format.channelCount)
let frames = Int(frameLength)
for ch in 0..<channels {
dst[ch].assign(from: src[ch], count: frames)
}
return copy
}
func append(_ buffer: AVAudioPCMBuffer) {
self.request.append(buffer)
if let src = self.int16ChannelData, let dst = copy.int16ChannelData {
let channels = Int(format.channelCount)
let frames = Int(frameLength)
for ch in 0..<channels {
dst[ch].assign(from: src[ch], count: frames)
}
return copy
}
if let src = self.int32ChannelData, let dst = copy.int32ChannelData {
let channels = Int(format.channelCount)
let frames = Int(frameLength)
for ch in 0..<channels {
dst[ch].assign(from: src[ch], count: frames)
}
return copy
}
return nil
}
}
@@ -32,6 +78,8 @@ final class VoiceWakeManager: NSObject, ObservableObject {
private var speechRecognizer: SFSpeechRecognizer?
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
private var recognitionTask: SFSpeechRecognitionTask?
private var tapQueue: AudioBufferQueue?
private var tapDrainTask: Task<Void, Never>?
private var lastDispatched: String?
private var onCommand: (@Sendable (String) async -> Void)?
@@ -92,6 +140,11 @@ final class VoiceWakeManager: NSObject, ObservableObject {
self.isListening = false
self.statusText = "Off"
self.tapDrainTask?.cancel()
self.tapDrainTask = nil
self.tapQueue?.clear()
self.tapQueue = nil
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.recognitionRequest = nil
@@ -107,6 +160,10 @@ final class VoiceWakeManager: NSObject, ObservableObject {
private func startRecognition() throws {
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.tapDrainTask?.cancel()
self.tapDrainTask = nil
self.tapQueue?.clear()
self.tapQueue = nil
let request = SFSpeechAudioBufferRecognitionRequest()
request.shouldReportPartialResults = true
@@ -115,16 +172,33 @@ final class VoiceWakeManager: NSObject, ObservableObject {
let inputNode = self.audioEngine.inputNode
inputNode.removeTap(onBus: 0)
let requestBox = SpeechRequestBox(request: request)
let recordingFormat = inputNode.outputFormat(forBus: 0)
let tap = SpeechAudioTapFactory.makeAppendTap(requestBox: requestBox)
inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat, block: tap)
let queue = AudioBufferQueue()
self.tapQueue = queue
inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { [weak queue] buffer, _ in
// `SFSpeechAudioBufferRecognitionRequest.append` is MainActor-isolated on iOS 26.
// Copy + enqueue in the realtime callback, drain + append from the MainActor.
queue?.enqueueCopy(of: buffer)
}
self.audioEngine.prepare()
try self.audioEngine.start()
let handler = self.makeRecognitionResultHandler()
self.recognitionTask = self.speechRecognizer?.recognitionTask(with: request, resultHandler: handler)
self.tapDrainTask = Task { [weak self] in
guard let self, let queue = self.tapQueue else { return }
while !Task.isCancelled {
try? await Task.sleep(nanoseconds: 40_000_000)
let drained = queue.drain()
if drained.isEmpty { continue }
for buf in drained {
request.append(buf)
}
}
}
}
private nonisolated func makeRecognitionResultHandler() -> @Sendable (SFSpeechRecognitionResult?, Error?) -> Void {
@@ -195,21 +269,17 @@ final class VoiceWakeManager: NSObject, ObservableObject {
}
private nonisolated static func requestMicrophonePermission() async -> Bool {
await withCheckedContinuation { cont in
await withCheckedContinuation(isolation: nil) { cont in
AVAudioApplication.requestRecordPermission { ok in
Task { @MainActor in
cont.resume(returning: ok)
}
cont.resume(returning: ok)
}
}
}
private nonisolated static func requestSpeechPermission() async -> Bool {
await withCheckedContinuation { cont in
await withCheckedContinuation(isolation: nil) { cont in
SFSpeechRecognizer.requestAuthorization { status in
Task { @MainActor in
cont.resume(returning: status == .authorized)
}
cont.resume(returning: status == .authorized)
}
}
}