feat(ios): add ClawdisNode app scaffold
This commit is contained in:
6
apps/ios/.swiftlint.yml
Normal file
6
apps/ios/.swiftlint.yml
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
parent_config: ../../.swiftlint.yml
|
||||||
|
|
||||||
|
included:
|
||||||
|
- Sources
|
||||||
|
- ../shared/ClawdisNodeKit/Sources
|
||||||
|
|
||||||
18
apps/ios/README.md
Normal file
18
apps/ios/README.md
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
# ClawdisNode (iOS)
|
||||||
|
|
||||||
|
Internal-only SwiftUI app scaffold.
|
||||||
|
|
||||||
|
## Lint/format (required)
|
||||||
|
```bash
|
||||||
|
brew install swiftformat swiftlint
|
||||||
|
```
|
||||||
|
|
||||||
|
## Generate the Xcode project
|
||||||
|
```bash
|
||||||
|
cd apps/ios
|
||||||
|
xcodegen generate
|
||||||
|
open ClawdisNode.xcodeproj
|
||||||
|
```
|
||||||
|
|
||||||
|
## Shared packages
|
||||||
|
- `../shared/ClawdisNodeKit` — shared types/constants used by iOS (and later macOS bridge + gateway routing).
|
||||||
121
apps/ios/Sources/Bridge/BridgeClient.swift
Normal file
121
apps/ios/Sources/Bridge/BridgeClient.swift
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
import ClawdisNodeKit
|
||||||
|
import Foundation
|
||||||
|
import Network
|
||||||
|
|
||||||
|
actor BridgeClient {
|
||||||
|
private let encoder = JSONEncoder()
|
||||||
|
private let decoder = JSONDecoder()
|
||||||
|
|
||||||
|
func pairAndHello(
|
||||||
|
endpoint: NWEndpoint,
|
||||||
|
nodeId: String,
|
||||||
|
displayName: String?,
|
||||||
|
platform: String,
|
||||||
|
version: String,
|
||||||
|
existingToken: String?) async throws -> String
|
||||||
|
{
|
||||||
|
let connection = NWConnection(to: endpoint, using: .tcp)
|
||||||
|
let queue = DispatchQueue(label: "com.steipete.clawdis.ios.bridge-client")
|
||||||
|
connection.start(queue: queue)
|
||||||
|
|
||||||
|
let token = existingToken
|
||||||
|
try await self.send(
|
||||||
|
BridgeHello(
|
||||||
|
nodeId: nodeId,
|
||||||
|
displayName: displayName,
|
||||||
|
token: token,
|
||||||
|
platform: platform,
|
||||||
|
version: version),
|
||||||
|
over: connection)
|
||||||
|
|
||||||
|
if let line = try await self.receiveLine(over: connection),
|
||||||
|
let data = line.data(using: .utf8),
|
||||||
|
let base = try? self.decoder.decode(BridgeBaseFrame.self, from: data)
|
||||||
|
{
|
||||||
|
if base.type == "hello-ok" {
|
||||||
|
connection.cancel()
|
||||||
|
return existingToken ?? ""
|
||||||
|
}
|
||||||
|
if base.type == "error" {
|
||||||
|
let err = try self.decoder.decode(BridgeErrorFrame.self, from: data)
|
||||||
|
if err.code == "NOT_PAIRED" || err.code == "UNAUTHORIZED" {
|
||||||
|
try await self.send(
|
||||||
|
BridgePairRequest(
|
||||||
|
nodeId: nodeId,
|
||||||
|
displayName: displayName,
|
||||||
|
platform: platform,
|
||||||
|
version: version),
|
||||||
|
over: connection)
|
||||||
|
|
||||||
|
while let next = try await self.receiveLine(over: connection) {
|
||||||
|
guard let nextData = next.data(using: .utf8) else { continue }
|
||||||
|
let nextBase = try self.decoder.decode(BridgeBaseFrame.self, from: nextData)
|
||||||
|
if nextBase.type == "pair-ok" {
|
||||||
|
let ok = try self.decoder.decode(BridgePairOk.self, from: nextData)
|
||||||
|
connection.cancel()
|
||||||
|
return ok.token
|
||||||
|
}
|
||||||
|
if nextBase.type == "error" {
|
||||||
|
let e = try self.decoder.decode(BridgeErrorFrame.self, from: nextData)
|
||||||
|
connection.cancel()
|
||||||
|
throw NSError(domain: "Bridge", code: 2, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "\(e.code): \(e.message)",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
connection.cancel()
|
||||||
|
throw NSError(domain: "Bridge", code: 1, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "\(err.code): \(err.message)",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
connection.cancel()
|
||||||
|
throw NSError(domain: "Bridge", code: 0, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "Unexpected bridge response",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
private func send(_ obj: some Encodable, over connection: NWConnection) async throws {
|
||||||
|
let data = try self.encoder.encode(obj)
|
||||||
|
var line = Data()
|
||||||
|
line.append(data)
|
||||||
|
line.append(0x0A)
|
||||||
|
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
|
||||||
|
connection.send(content: line, completion: .contentProcessed { err in
|
||||||
|
if let err { cont.resume(throwing: err) } else { cont.resume(returning: ()) }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func receiveLine(over connection: NWConnection) async throws -> String? {
|
||||||
|
var buffer = Data()
|
||||||
|
while true {
|
||||||
|
if let idx = buffer.firstIndex(of: 0x0A) {
|
||||||
|
let lineData = buffer.prefix(upTo: idx)
|
||||||
|
return String(data: lineData, encoding: .utf8)
|
||||||
|
}
|
||||||
|
|
||||||
|
let chunk = try await self.receiveChunk(over: connection)
|
||||||
|
if chunk.isEmpty { return nil }
|
||||||
|
buffer.append(chunk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func receiveChunk(over connection: NWConnection) async throws -> Data {
|
||||||
|
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Data, Error>) in
|
||||||
|
connection.receive(minimumIncompleteLength: 1, maximumLength: 64 * 1024) { data, _, isComplete, error in
|
||||||
|
if let error {
|
||||||
|
cont.resume(throwing: error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if isComplete {
|
||||||
|
cont.resume(returning: Data())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cont.resume(returning: data ?? Data())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
74
apps/ios/Sources/Bridge/BridgeDiscoveryModel.swift
Normal file
74
apps/ios/Sources/Bridge/BridgeDiscoveryModel.swift
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
import ClawdisNodeKit
|
||||||
|
import Foundation
|
||||||
|
import Network
|
||||||
|
|
||||||
|
@MainActor
|
||||||
|
final class BridgeDiscoveryModel: ObservableObject {
|
||||||
|
struct DiscoveredBridge: Identifiable, Equatable {
|
||||||
|
var id: String { self.debugID }
|
||||||
|
var name: String
|
||||||
|
var endpoint: NWEndpoint
|
||||||
|
var debugID: String
|
||||||
|
}
|
||||||
|
|
||||||
|
@Published var bridges: [DiscoveredBridge] = []
|
||||||
|
@Published var statusText: String = "Idle"
|
||||||
|
|
||||||
|
private var browser: NWBrowser?
|
||||||
|
|
||||||
|
func start() {
|
||||||
|
if self.browser != nil { return }
|
||||||
|
let params = NWParameters.tcp
|
||||||
|
let browser = NWBrowser(
|
||||||
|
for: .bonjour(type: ClawdisBonjour.bridgeServiceType, domain: ClawdisBonjour.bridgeServiceDomain),
|
||||||
|
using: params)
|
||||||
|
|
||||||
|
browser.stateUpdateHandler = { [weak self] state in
|
||||||
|
Task { @MainActor in
|
||||||
|
guard let self else { return }
|
||||||
|
switch state {
|
||||||
|
case .setup:
|
||||||
|
self.statusText = "Setup"
|
||||||
|
case .ready:
|
||||||
|
self.statusText = "Searching…"
|
||||||
|
case let .failed(err):
|
||||||
|
self.statusText = "Failed: \(err)"
|
||||||
|
case .cancelled:
|
||||||
|
self.statusText = "Stopped"
|
||||||
|
case let .waiting(err):
|
||||||
|
self.statusText = "Waiting: \(err)"
|
||||||
|
@unknown default:
|
||||||
|
self.statusText = "Unknown"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
browser.browseResultsChangedHandler = { [weak self] results, _ in
|
||||||
|
Task { @MainActor in
|
||||||
|
guard let self else { return }
|
||||||
|
self.bridges = results.compactMap { result -> DiscoveredBridge? in
|
||||||
|
switch result.endpoint {
|
||||||
|
case let .service(name, _, _, _):
|
||||||
|
return DiscoveredBridge(
|
||||||
|
name: name,
|
||||||
|
endpoint: result.endpoint,
|
||||||
|
debugID: String(describing: result.endpoint))
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.browser = browser
|
||||||
|
browser.start(queue: DispatchQueue(label: "com.steipete.clawdis.ios.bridge-discovery"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func stop() {
|
||||||
|
self.browser?.cancel()
|
||||||
|
self.browser = nil
|
||||||
|
self.bridges = []
|
||||||
|
self.statusText = "Stopped"
|
||||||
|
}
|
||||||
|
}
|
||||||
151
apps/ios/Sources/Bridge/BridgeSession.swift
Normal file
151
apps/ios/Sources/Bridge/BridgeSession.swift
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
import ClawdisNodeKit
|
||||||
|
import Foundation
|
||||||
|
import Network
|
||||||
|
|
||||||
|
actor BridgeSession {
|
||||||
|
enum State: Sendable, Equatable {
|
||||||
|
case idle
|
||||||
|
case connecting
|
||||||
|
case connected(serverName: String)
|
||||||
|
case failed(message: String)
|
||||||
|
}
|
||||||
|
|
||||||
|
private let encoder = JSONEncoder()
|
||||||
|
private let decoder = JSONDecoder()
|
||||||
|
|
||||||
|
private var connection: NWConnection?
|
||||||
|
private var queue: DispatchQueue?
|
||||||
|
private var buffer = Data()
|
||||||
|
|
||||||
|
private(set) var state: State = .idle
|
||||||
|
|
||||||
|
func connect(
|
||||||
|
endpoint: NWEndpoint,
|
||||||
|
hello: BridgeHello,
|
||||||
|
onConnected: (@Sendable (String) async -> Void)? = nil,
|
||||||
|
onInvoke: @escaping @Sendable (BridgeInvokeRequest) async -> BridgeInvokeResponse)
|
||||||
|
async throws
|
||||||
|
{
|
||||||
|
await self.disconnect()
|
||||||
|
self.state = .connecting
|
||||||
|
|
||||||
|
let connection = NWConnection(to: endpoint, using: .tcp)
|
||||||
|
let queue = DispatchQueue(label: "com.steipete.clawdis.ios.bridge-session")
|
||||||
|
self.connection = connection
|
||||||
|
self.queue = queue
|
||||||
|
connection.start(queue: queue)
|
||||||
|
|
||||||
|
try await self.send(hello)
|
||||||
|
|
||||||
|
guard let line = try await self.receiveLine(),
|
||||||
|
let data = line.data(using: .utf8),
|
||||||
|
let base = try? self.decoder.decode(BridgeBaseFrame.self, from: data)
|
||||||
|
else {
|
||||||
|
await self.disconnect()
|
||||||
|
throw NSError(domain: "Bridge", code: 1, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "Unexpected bridge response",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
if base.type == "hello-ok" {
|
||||||
|
let ok = try self.decoder.decode(BridgeHelloOk.self, from: data)
|
||||||
|
self.state = .connected(serverName: ok.serverName)
|
||||||
|
await onConnected?(ok.serverName)
|
||||||
|
} else if base.type == "error" {
|
||||||
|
let err = try self.decoder.decode(BridgeErrorFrame.self, from: data)
|
||||||
|
self.state = .failed(message: "\(err.code): \(err.message)")
|
||||||
|
await self.disconnect()
|
||||||
|
throw NSError(domain: "Bridge", code: 2, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "\(err.code): \(err.message)",
|
||||||
|
])
|
||||||
|
} else {
|
||||||
|
self.state = .failed(message: "Unexpected bridge response")
|
||||||
|
await self.disconnect()
|
||||||
|
throw NSError(domain: "Bridge", code: 3, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "Unexpected bridge response",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
while true {
|
||||||
|
guard let next = try await self.receiveLine() else { break }
|
||||||
|
guard let nextData = next.data(using: .utf8) else { continue }
|
||||||
|
guard let nextBase = try? self.decoder.decode(BridgeBaseFrame.self, from: nextData) else { continue }
|
||||||
|
|
||||||
|
switch nextBase.type {
|
||||||
|
case "ping":
|
||||||
|
let ping = try self.decoder.decode(BridgePing.self, from: nextData)
|
||||||
|
try await self.send(BridgePong(type: "pong", id: ping.id))
|
||||||
|
|
||||||
|
case "invoke":
|
||||||
|
let req = try self.decoder.decode(BridgeInvokeRequest.self, from: nextData)
|
||||||
|
let res = await onInvoke(req)
|
||||||
|
try await self.send(res)
|
||||||
|
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await self.disconnect()
|
||||||
|
}
|
||||||
|
|
||||||
|
func sendEvent(event: String, payloadJSON: String?) async throws {
|
||||||
|
try await self.send(BridgeEventFrame(type: "event", event: event, payloadJSON: payloadJSON))
|
||||||
|
}
|
||||||
|
|
||||||
|
func disconnect() async {
|
||||||
|
self.connection?.cancel()
|
||||||
|
self.connection = nil
|
||||||
|
self.queue = nil
|
||||||
|
self.buffer = Data()
|
||||||
|
self.state = .idle
|
||||||
|
}
|
||||||
|
|
||||||
|
private func send(_ obj: some Encodable) async throws {
|
||||||
|
guard let connection = self.connection else {
|
||||||
|
throw NSError(domain: "Bridge", code: 10, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "not connected",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
let data = try self.encoder.encode(obj)
|
||||||
|
var line = Data()
|
||||||
|
line.append(data)
|
||||||
|
line.append(0x0A)
|
||||||
|
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
|
||||||
|
connection.send(content: line, completion: .contentProcessed { err in
|
||||||
|
if let err { cont.resume(throwing: err) } else { cont.resume(returning: ()) }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func receiveLine() async throws -> String? {
|
||||||
|
while true {
|
||||||
|
if let idx = self.buffer.firstIndex(of: 0x0A) {
|
||||||
|
let lineData = self.buffer.prefix(upTo: idx)
|
||||||
|
self.buffer.removeSubrange(...idx)
|
||||||
|
return String(data: lineData, encoding: .utf8)
|
||||||
|
}
|
||||||
|
|
||||||
|
let chunk = try await self.receiveChunk()
|
||||||
|
if chunk.isEmpty { return nil }
|
||||||
|
self.buffer.append(chunk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func receiveChunk() async throws -> Data {
|
||||||
|
guard let connection = self.connection else { return Data() }
|
||||||
|
return try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Data, Error>) in
|
||||||
|
connection.receive(minimumIncompleteLength: 1, maximumLength: 64 * 1024) { data, _, isComplete, error in
|
||||||
|
if let error {
|
||||||
|
cont.resume(throwing: error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if isComplete {
|
||||||
|
cont.resume(returning: Data())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cont.resume(returning: data ?? Data())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
49
apps/ios/Sources/Bridge/KeychainStore.swift
Normal file
49
apps/ios/Sources/Bridge/KeychainStore.swift
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
import Foundation
|
||||||
|
import Security
|
||||||
|
|
||||||
|
enum KeychainStore {
|
||||||
|
static func loadString(service: String, account: String) -> String? {
|
||||||
|
var query: [String: Any] = [
|
||||||
|
kSecClass as String: kSecClassGenericPassword,
|
||||||
|
kSecAttrService as String: service,
|
||||||
|
kSecAttrAccount as String: account,
|
||||||
|
kSecReturnData as String: true,
|
||||||
|
kSecMatchLimit as String: kSecMatchLimitOne,
|
||||||
|
]
|
||||||
|
query[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly
|
||||||
|
|
||||||
|
var item: CFTypeRef?
|
||||||
|
let status = SecItemCopyMatching(query as CFDictionary, &item)
|
||||||
|
guard status == errSecSuccess, let data = item as? Data else { return nil }
|
||||||
|
return String(data: data, encoding: .utf8)
|
||||||
|
}
|
||||||
|
|
||||||
|
static func saveString(_ value: String, service: String, account: String) -> Bool {
|
||||||
|
let data = Data(value.utf8)
|
||||||
|
let base: [String: Any] = [
|
||||||
|
kSecClass as String: kSecClassGenericPassword,
|
||||||
|
kSecAttrService as String: service,
|
||||||
|
kSecAttrAccount as String: account,
|
||||||
|
kSecAttrAccessible as String: kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly,
|
||||||
|
]
|
||||||
|
|
||||||
|
let update: [String: Any] = [kSecValueData as String: data]
|
||||||
|
let status = SecItemUpdate(base as CFDictionary, update as CFDictionary)
|
||||||
|
if status == errSecSuccess { return true }
|
||||||
|
if status != errSecItemNotFound { return false }
|
||||||
|
|
||||||
|
var insert = base
|
||||||
|
insert[kSecValueData as String] = data
|
||||||
|
return SecItemAdd(insert as CFDictionary, nil) == errSecSuccess
|
||||||
|
}
|
||||||
|
|
||||||
|
static func delete(service: String, account: String) -> Bool {
|
||||||
|
let query: [String: Any] = [
|
||||||
|
kSecClass as String: kSecClassGenericPassword,
|
||||||
|
kSecAttrService as String: service,
|
||||||
|
kSecAttrAccount as String: account,
|
||||||
|
]
|
||||||
|
let status = SecItemDelete(query as CFDictionary)
|
||||||
|
return status == errSecSuccess || status == errSecItemNotFound
|
||||||
|
}
|
||||||
|
}
|
||||||
18
apps/ios/Sources/ClawdisNodeApp.swift
Normal file
18
apps/ios/Sources/ClawdisNodeApp.swift
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
import SwiftUI
|
||||||
|
|
||||||
|
@main
|
||||||
|
struct ClawdisNodeApp: App {
|
||||||
|
@StateObject private var appModel = NodeAppModel()
|
||||||
|
@Environment(\.scenePhase) private var scenePhase
|
||||||
|
|
||||||
|
var body: some Scene {
|
||||||
|
WindowGroup {
|
||||||
|
RootTabs()
|
||||||
|
.environmentObject(self.appModel)
|
||||||
|
.environmentObject(self.appModel.voiceWake)
|
||||||
|
.onChange(of: self.scenePhase) { _, newValue in
|
||||||
|
self.appModel.setScenePhase(newValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
45
apps/ios/Sources/Info.plist
Normal file
45
apps/ios/Sources/Info.plist
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||||
|
<plist version="1.0">
|
||||||
|
<dict>
|
||||||
|
<key>CFBundleDevelopmentRegion</key>
|
||||||
|
<string>$(DEVELOPMENT_LANGUAGE)</string>
|
||||||
|
<key>CFBundleDisplayName</key>
|
||||||
|
<string>Clawdis Node</string>
|
||||||
|
<key>CFBundleExecutable</key>
|
||||||
|
<string>$(EXECUTABLE_NAME)</string>
|
||||||
|
<key>CFBundleIdentifier</key>
|
||||||
|
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||||
|
<key>CFBundleInfoDictionaryVersion</key>
|
||||||
|
<string>6.0</string>
|
||||||
|
<key>CFBundleName</key>
|
||||||
|
<string>$(PRODUCT_NAME)</string>
|
||||||
|
<key>CFBundlePackageType</key>
|
||||||
|
<string>APPL</string>
|
||||||
|
<key>CFBundleShortVersionString</key>
|
||||||
|
<string>1.0</string>
|
||||||
|
<key>CFBundleVersion</key>
|
||||||
|
<string>1</string>
|
||||||
|
<key>NSBonjourServices</key>
|
||||||
|
<array>
|
||||||
|
<string>_clawdis-bridge._tcp</string>
|
||||||
|
</array>
|
||||||
|
<key>NSLocalNetworkUsageDescription</key>
|
||||||
|
<string>Clawdis Node discovers and connects to your Clawdis bridge on the local network.</string>
|
||||||
|
<key>NSMicrophoneUsageDescription</key>
|
||||||
|
<string>Clawdis Node needs microphone access for voice wake.</string>
|
||||||
|
<key>NSSpeechRecognitionUsageDescription</key>
|
||||||
|
<string>Clawdis Node uses on-device speech recognition for voice wake.</string>
|
||||||
|
<key>UIApplicationSceneManifest</key>
|
||||||
|
<dict>
|
||||||
|
<key>UIApplicationSupportsMultipleScenes</key>
|
||||||
|
<false/>
|
||||||
|
</dict>
|
||||||
|
<key>UIBackgroundModes</key>
|
||||||
|
<array>
|
||||||
|
<string>audio</string>
|
||||||
|
</array>
|
||||||
|
<key>UILaunchScreen</key>
|
||||||
|
<dict/>
|
||||||
|
</dict>
|
||||||
|
</plist>
|
||||||
185
apps/ios/Sources/Model/NodeAppModel.swift
Normal file
185
apps/ios/Sources/Model/NodeAppModel.swift
Normal file
@@ -0,0 +1,185 @@
|
|||||||
|
import ClawdisNodeKit
|
||||||
|
import Network
|
||||||
|
import SwiftUI
|
||||||
|
|
||||||
|
@MainActor
|
||||||
|
final class NodeAppModel: ObservableObject {
|
||||||
|
@Published var isBackgrounded: Bool = false
|
||||||
|
let screen = ScreenController()
|
||||||
|
@Published var bridgeStatusText: String = "Not connected"
|
||||||
|
@Published var bridgeServerName: String?
|
||||||
|
|
||||||
|
private let bridge = BridgeSession()
|
||||||
|
private var bridgeTask: Task<Void, Never>?
|
||||||
|
let voiceWake = VoiceWakeManager()
|
||||||
|
|
||||||
|
init() {
|
||||||
|
self.voiceWake.configure { [weak self] cmd in
|
||||||
|
guard let self else { return }
|
||||||
|
let nodeId = UserDefaults.standard.string(forKey: "node.instanceId") ?? "ios-node"
|
||||||
|
let sessionKey = "node-\(nodeId)"
|
||||||
|
do {
|
||||||
|
try await self.sendVoiceTranscript(text: cmd, sessionKey: sessionKey)
|
||||||
|
} catch {
|
||||||
|
// Best-effort only.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let enabled = UserDefaults.standard.bool(forKey: "voiceWake.enabled")
|
||||||
|
self.voiceWake.setEnabled(enabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setScenePhase(_ phase: ScenePhase) {
|
||||||
|
switch phase {
|
||||||
|
case .background:
|
||||||
|
self.isBackgrounded = true
|
||||||
|
case .active, .inactive:
|
||||||
|
self.isBackgrounded = false
|
||||||
|
@unknown default:
|
||||||
|
self.isBackgrounded = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setVoiceWakeEnabled(_ enabled: Bool) {
|
||||||
|
self.voiceWake.setEnabled(enabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
func connectToBridge(
|
||||||
|
endpoint: NWEndpoint,
|
||||||
|
token: String,
|
||||||
|
nodeId: String,
|
||||||
|
displayName: String?,
|
||||||
|
platform: String,
|
||||||
|
version: String)
|
||||||
|
{
|
||||||
|
self.bridgeTask?.cancel()
|
||||||
|
self.bridgeStatusText = "Connecting…"
|
||||||
|
self.bridgeServerName = nil
|
||||||
|
|
||||||
|
self.bridgeTask = Task {
|
||||||
|
do {
|
||||||
|
try await self.bridge.connect(
|
||||||
|
endpoint: endpoint,
|
||||||
|
hello: BridgeHello(
|
||||||
|
nodeId: nodeId,
|
||||||
|
displayName: displayName,
|
||||||
|
token: token,
|
||||||
|
platform: platform,
|
||||||
|
version: version),
|
||||||
|
onConnected: { [weak self] serverName in
|
||||||
|
await MainActor.run {
|
||||||
|
self?.bridgeStatusText = "Connected"
|
||||||
|
self?.bridgeServerName = serverName
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onInvoke: { [weak self] req in
|
||||||
|
guard let self else {
|
||||||
|
return BridgeInvokeResponse(
|
||||||
|
id: req.id,
|
||||||
|
ok: false,
|
||||||
|
error: ClawdisNodeError(code: .unavailable, message: "UNAVAILABLE: node not ready"))
|
||||||
|
}
|
||||||
|
return await self.handleInvoke(req)
|
||||||
|
})
|
||||||
|
|
||||||
|
await MainActor.run {
|
||||||
|
self.bridgeStatusText = "Disconnected"
|
||||||
|
self.bridgeServerName = nil
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
await MainActor.run {
|
||||||
|
self.bridgeStatusText = "Bridge error: \(error.localizedDescription)"
|
||||||
|
self.bridgeServerName = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func disconnectBridge() {
|
||||||
|
self.bridgeTask?.cancel()
|
||||||
|
self.bridgeTask = nil
|
||||||
|
Task { await self.bridge.disconnect() }
|
||||||
|
self.bridgeStatusText = "Disconnected"
|
||||||
|
self.bridgeServerName = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sendVoiceTranscript(text: String, sessionKey: String?) async throws {
|
||||||
|
struct Payload: Codable {
|
||||||
|
var text: String
|
||||||
|
var sessionKey: String?
|
||||||
|
}
|
||||||
|
let payload = Payload(text: text, sessionKey: sessionKey)
|
||||||
|
let data = try JSONEncoder().encode(payload)
|
||||||
|
let json = String(decoding: data, as: UTF8.self)
|
||||||
|
try await self.bridge.sendEvent(event: "voice.transcript", payloadJSON: json)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
|
||||||
|
if req.command.hasPrefix("screen."), self.isBackgrounded {
|
||||||
|
return BridgeInvokeResponse(
|
||||||
|
id: req.id,
|
||||||
|
ok: false,
|
||||||
|
error: ClawdisNodeError(
|
||||||
|
code: .backgroundUnavailable,
|
||||||
|
message: "NODE_BACKGROUND_UNAVAILABLE: screen commands require foreground"))
|
||||||
|
}
|
||||||
|
|
||||||
|
do {
|
||||||
|
switch req.command {
|
||||||
|
case ClawdisScreenCommand.show.rawValue:
|
||||||
|
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||||
|
|
||||||
|
case ClawdisScreenCommand.hide.rawValue:
|
||||||
|
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||||
|
|
||||||
|
case ClawdisScreenCommand.setMode.rawValue:
|
||||||
|
let params = try Self.decodeParams(ClawdisScreenSetModeParams.self, from: req.paramsJSON)
|
||||||
|
self.screen.setMode(params.mode)
|
||||||
|
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||||
|
|
||||||
|
case ClawdisScreenCommand.navigate.rawValue:
|
||||||
|
let params = try Self.decodeParams(ClawdisScreenNavigateParams.self, from: req.paramsJSON)
|
||||||
|
self.screen.navigate(to: params.url)
|
||||||
|
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||||
|
|
||||||
|
case ClawdisScreenCommand.evalJS.rawValue:
|
||||||
|
let params = try Self.decodeParams(ClawdisScreenEvalParams.self, from: req.paramsJSON)
|
||||||
|
let result = try await self.screen.eval(javaScript: params.javaScript)
|
||||||
|
let payload = try Self.encodePayload(["result": result])
|
||||||
|
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||||
|
|
||||||
|
case ClawdisScreenCommand.snapshot.rawValue:
|
||||||
|
let params = try? Self.decodeParams(ClawdisScreenSnapshotParams.self, from: req.paramsJSON)
|
||||||
|
let maxWidth = params?.maxWidth.map { CGFloat($0) }
|
||||||
|
let base64 = try await self.screen.snapshotPNGBase64(maxWidth: maxWidth)
|
||||||
|
let payload = try Self.encodePayload(["format": "png", "base64": base64])
|
||||||
|
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||||
|
|
||||||
|
default:
|
||||||
|
return BridgeInvokeResponse(
|
||||||
|
id: req.id,
|
||||||
|
ok: false,
|
||||||
|
error: ClawdisNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return BridgeInvokeResponse(
|
||||||
|
id: req.id,
|
||||||
|
ok: false,
|
||||||
|
error: ClawdisNodeError(code: .unavailable, message: error.localizedDescription))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func decodeParams<T: Decodable>(_ type: T.Type, from json: String?) throws -> T {
|
||||||
|
guard let json, let data = json.data(using: .utf8) else {
|
||||||
|
throw NSError(domain: "Bridge", code: 20, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "INVALID_REQUEST: paramsJSON required",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
return try JSONDecoder().decode(type, from: data)
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func encodePayload(_ obj: some Encodable) throws -> String {
|
||||||
|
let data = try JSONEncoder().encode(obj)
|
||||||
|
return String(decoding: data, as: UTF8.self)
|
||||||
|
}
|
||||||
|
}
|
||||||
16
apps/ios/Sources/RootTabs.swift
Normal file
16
apps/ios/Sources/RootTabs.swift
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
import SwiftUI
|
||||||
|
|
||||||
|
struct RootTabs: View {
|
||||||
|
var body: some View {
|
||||||
|
TabView {
|
||||||
|
ScreenTab()
|
||||||
|
.tabItem { Label("Screen", systemImage: "rectangle.and.hand.point.up.left") }
|
||||||
|
|
||||||
|
VoiceTab()
|
||||||
|
.tabItem { Label("Voice", systemImage: "mic") }
|
||||||
|
|
||||||
|
SettingsTab()
|
||||||
|
.tabItem { Label("Settings", systemImage: "gearshape") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
120
apps/ios/Sources/Screen/ScreenController.swift
Normal file
120
apps/ios/Sources/Screen/ScreenController.swift
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
import ClawdisNodeKit
|
||||||
|
import SwiftUI
|
||||||
|
import WebKit
|
||||||
|
|
||||||
|
@MainActor
|
||||||
|
final class ScreenController: ObservableObject {
|
||||||
|
let webView: WKWebView
|
||||||
|
|
||||||
|
@Published var mode: ClawdisScreenMode = .web
|
||||||
|
@Published var urlString: String = "https://example.com"
|
||||||
|
@Published var errorText: String?
|
||||||
|
|
||||||
|
init() {
|
||||||
|
let config = WKWebViewConfiguration()
|
||||||
|
config.websiteDataStore = .nonPersistent()
|
||||||
|
self.webView = WKWebView(frame: .zero, configuration: config)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setMode(_ mode: ClawdisScreenMode) {
|
||||||
|
self.mode = mode
|
||||||
|
self.reload()
|
||||||
|
}
|
||||||
|
|
||||||
|
func navigate(to urlString: String) {
|
||||||
|
self.urlString = urlString
|
||||||
|
self.reload()
|
||||||
|
}
|
||||||
|
|
||||||
|
func reload() {
|
||||||
|
switch self.mode {
|
||||||
|
case .web:
|
||||||
|
guard let url = URL(string: self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)) else { return }
|
||||||
|
self.webView.load(URLRequest(url: url))
|
||||||
|
case .canvas:
|
||||||
|
self.webView.loadHTMLString(Self.canvasScaffoldHTML, baseURL: nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func eval(javaScript: String) async throws -> String {
|
||||||
|
try await withCheckedThrowingContinuation { cont in
|
||||||
|
self.webView.evaluateJavaScript(javaScript) { result, error in
|
||||||
|
if let error {
|
||||||
|
cont.resume(throwing: error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if let result {
|
||||||
|
cont.resume(returning: String(describing: result))
|
||||||
|
} else {
|
||||||
|
cont.resume(returning: "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func snapshotPNGBase64(maxWidth: CGFloat? = nil) async throws -> String {
|
||||||
|
let config = WKSnapshotConfiguration()
|
||||||
|
if let maxWidth {
|
||||||
|
config.snapshotWidth = NSNumber(value: Double(maxWidth))
|
||||||
|
}
|
||||||
|
let image: UIImage = try await withCheckedThrowingContinuation { (cont: CheckedContinuation<UIImage, Error>) in
|
||||||
|
self.webView.takeSnapshot(with: config) { image, error in
|
||||||
|
if let error {
|
||||||
|
cont.resume(throwing: error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
guard let image else {
|
||||||
|
cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "snapshot failed",
|
||||||
|
]))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cont.resume(returning: image)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
guard let data = image.pngData() else {
|
||||||
|
throw NSError(domain: "Screen", code: 1, userInfo: [
|
||||||
|
NSLocalizedDescriptionKey: "snapshot encode failed",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
return data.base64EncodedString()
|
||||||
|
}
|
||||||
|
|
||||||
|
private static let canvasScaffoldHTML = """
|
||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>Canvas</title>
|
||||||
|
<style>
|
||||||
|
:root { color-scheme: dark; }
|
||||||
|
html,body { height:100%; margin:0; }
|
||||||
|
body {
|
||||||
|
font: 13px -apple-system, system-ui;
|
||||||
|
display:flex;
|
||||||
|
align-items:center;
|
||||||
|
justify-content:center;
|
||||||
|
background:#0b1020;
|
||||||
|
color:#e5e7eb;
|
||||||
|
}
|
||||||
|
.card {
|
||||||
|
max-width: 520px;
|
||||||
|
padding: 18px;
|
||||||
|
border-radius: 14px;
|
||||||
|
border: 1px solid rgba(255,255,255,.10);
|
||||||
|
background: rgba(255,255,255,.06);
|
||||||
|
box-shadow: 0 18px 60px rgba(0,0,0,.35);
|
||||||
|
}
|
||||||
|
.muted { color: rgba(229,231,235,.75); margin-top: 8px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="card">
|
||||||
|
<div style="font-weight:600; font-size:14px;">Canvas scaffold</div>
|
||||||
|
<div class="muted">Next: agent-driven on-disk workspace.</div>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
}
|
||||||
74
apps/ios/Sources/Screen/ScreenTab.swift
Normal file
74
apps/ios/Sources/Screen/ScreenTab.swift
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
import ClawdisNodeKit
|
||||||
|
import SwiftUI
|
||||||
|
|
||||||
|
struct ScreenTab: View {
|
||||||
|
@EnvironmentObject private var appModel: NodeAppModel
|
||||||
|
|
||||||
|
var body: some View {
|
||||||
|
NavigationStack {
|
||||||
|
VStack(spacing: 0) {
|
||||||
|
ScreenWebView(controller: self.appModel.screen)
|
||||||
|
.overlay(alignment: .top) {
|
||||||
|
if let errorText = self.appModel.screen.errorText {
|
||||||
|
Text(errorText)
|
||||||
|
.font(.footnote)
|
||||||
|
.padding(10)
|
||||||
|
.background(.thinMaterial)
|
||||||
|
.clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous))
|
||||||
|
.padding()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Divider()
|
||||||
|
|
||||||
|
VStack(spacing: 10) {
|
||||||
|
Picker(
|
||||||
|
"Mode",
|
||||||
|
selection: Binding(
|
||||||
|
get: { self.appModel.screen.mode },
|
||||||
|
set: { self.appModel.screen.setMode($0) }))
|
||||||
|
{
|
||||||
|
Text("Web").tag(ClawdisScreenMode.web)
|
||||||
|
Text("Canvas").tag(ClawdisScreenMode.canvas)
|
||||||
|
}
|
||||||
|
.pickerStyle(.segmented)
|
||||||
|
|
||||||
|
HStack(spacing: 10) {
|
||||||
|
TextField(
|
||||||
|
"URL",
|
||||||
|
text: Binding(
|
||||||
|
get: { self.appModel.screen.urlString },
|
||||||
|
set: { self.appModel.screen.urlString = $0 }))
|
||||||
|
.textInputAutocapitalization(.never)
|
||||||
|
.autocorrectionDisabled()
|
||||||
|
.keyboardType(.URL)
|
||||||
|
.textFieldStyle(.roundedBorder)
|
||||||
|
Button("Go") { self.navigate() }
|
||||||
|
.buttonStyle(.borderedProminent)
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.appModel.isBackgrounded {
|
||||||
|
Text("Screen commands unavailable while backgrounded.")
|
||||||
|
.font(.footnote)
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
.frame(maxWidth: .infinity, alignment: .leading)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.padding()
|
||||||
|
}
|
||||||
|
.navigationTitle("Screen")
|
||||||
|
.navigationBarTitleDisplayMode(.inline)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func navigate() {
|
||||||
|
if self.appModel.isBackgrounded {
|
||||||
|
self.appModel.screen.errorText = ClawdisNodeError(
|
||||||
|
code: .backgroundUnavailable,
|
||||||
|
message: "NODE_BACKGROUND_UNAVAILABLE: screen commands require foreground").message
|
||||||
|
return
|
||||||
|
}
|
||||||
|
self.appModel.screen.errorText = nil
|
||||||
|
self.appModel.screen.reload()
|
||||||
|
}
|
||||||
|
}
|
||||||
15
apps/ios/Sources/Screen/ScreenWebView.swift
Normal file
15
apps/ios/Sources/Screen/ScreenWebView.swift
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
import ClawdisNodeKit
|
||||||
|
import SwiftUI
|
||||||
|
import WebKit
|
||||||
|
|
||||||
|
struct ScreenWebView: UIViewRepresentable {
|
||||||
|
@ObservedObject var controller: ScreenController
|
||||||
|
|
||||||
|
func makeUIView(context: Context) -> WKWebView {
|
||||||
|
self.controller.webView
|
||||||
|
}
|
||||||
|
|
||||||
|
func updateUIView(_ webView: WKWebView, context: Context) {
|
||||||
|
// State changes are driven by ScreenController.
|
||||||
|
}
|
||||||
|
}
|
||||||
30
apps/ios/Sources/Voice/VoiceTab.swift
Normal file
30
apps/ios/Sources/Voice/VoiceTab.swift
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
import SwiftUI
|
||||||
|
|
||||||
|
struct VoiceTab: View {
|
||||||
|
@EnvironmentObject private var appModel: NodeAppModel
|
||||||
|
@EnvironmentObject private var voiceWake: VoiceWakeManager
|
||||||
|
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
|
||||||
|
|
||||||
|
var body: some View {
|
||||||
|
NavigationStack {
|
||||||
|
List {
|
||||||
|
Section("Status") {
|
||||||
|
LabeledContent("Voice Wake", value: self.voiceWakeEnabled ? "Enabled" : "Disabled")
|
||||||
|
LabeledContent("Listener", value: self.voiceWake.isListening ? "Listening" : "Idle")
|
||||||
|
Text(self.voiceWake.statusText)
|
||||||
|
.font(.footnote)
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
}
|
||||||
|
|
||||||
|
Section("Notes") {
|
||||||
|
Text("Say “clawdis …” to trigger.")
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.navigationTitle("Voice")
|
||||||
|
.onChange(of: self.voiceWakeEnabled) { _, newValue in
|
||||||
|
self.appModel.setVoiceWakeEnabled(newValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
174
apps/ios/Sources/Voice/VoiceWakeManager.swift
Normal file
174
apps/ios/Sources/Voice/VoiceWakeManager.swift
Normal file
@@ -0,0 +1,174 @@
|
|||||||
|
import AVFAudio
|
||||||
|
import Foundation
|
||||||
|
import Speech
|
||||||
|
|
||||||
|
@MainActor
|
||||||
|
final class VoiceWakeManager: NSObject, ObservableObject {
|
||||||
|
@Published var isEnabled: Bool = false
|
||||||
|
@Published var isListening: Bool = false
|
||||||
|
@Published var statusText: String = "Off"
|
||||||
|
|
||||||
|
private let audioEngine = AVAudioEngine()
|
||||||
|
private var speechRecognizer: SFSpeechRecognizer?
|
||||||
|
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
|
||||||
|
private var recognitionTask: SFSpeechRecognitionTask?
|
||||||
|
|
||||||
|
private var lastDispatched: String?
|
||||||
|
private var onCommand: (@Sendable (String) async -> Void)?
|
||||||
|
|
||||||
|
func configure(onCommand: @escaping @Sendable (String) async -> Void) {
|
||||||
|
self.onCommand = onCommand
|
||||||
|
}
|
||||||
|
|
||||||
|
func setEnabled(_ enabled: Bool) {
|
||||||
|
self.isEnabled = enabled
|
||||||
|
if enabled {
|
||||||
|
Task { await self.start() }
|
||||||
|
} else {
|
||||||
|
self.stop()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func start() async {
|
||||||
|
guard self.isEnabled else { return }
|
||||||
|
if self.isListening { return }
|
||||||
|
|
||||||
|
self.statusText = "Requesting permissions…"
|
||||||
|
|
||||||
|
let micOk = await Self.requestMicrophonePermission()
|
||||||
|
guard micOk else {
|
||||||
|
self.statusText = "Microphone permission denied"
|
||||||
|
self.isListening = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let speechOk = await Self.requestSpeechPermission()
|
||||||
|
guard speechOk else {
|
||||||
|
self.statusText = "Speech recognition permission denied"
|
||||||
|
self.isListening = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
self.speechRecognizer = SFSpeechRecognizer()
|
||||||
|
guard self.speechRecognizer != nil else {
|
||||||
|
self.statusText = "Speech recognizer unavailable"
|
||||||
|
self.isListening = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
do {
|
||||||
|
try Self.configureAudioSession()
|
||||||
|
try self.startRecognition()
|
||||||
|
self.isListening = true
|
||||||
|
self.statusText = "Listening"
|
||||||
|
} catch {
|
||||||
|
self.isListening = false
|
||||||
|
self.statusText = "Start failed: \(error.localizedDescription)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func stop() {
|
||||||
|
self.isEnabled = false
|
||||||
|
self.isListening = false
|
||||||
|
self.statusText = "Off"
|
||||||
|
|
||||||
|
self.recognitionTask?.cancel()
|
||||||
|
self.recognitionTask = nil
|
||||||
|
self.recognitionRequest = nil
|
||||||
|
|
||||||
|
if self.audioEngine.isRunning {
|
||||||
|
self.audioEngine.stop()
|
||||||
|
self.audioEngine.inputNode.removeTap(onBus: 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func startRecognition() throws {
|
||||||
|
self.recognitionTask?.cancel()
|
||||||
|
self.recognitionTask = nil
|
||||||
|
|
||||||
|
let request = SFSpeechAudioBufferRecognitionRequest()
|
||||||
|
request.shouldReportPartialResults = true
|
||||||
|
self.recognitionRequest = request
|
||||||
|
|
||||||
|
let inputNode = self.audioEngine.inputNode
|
||||||
|
inputNode.removeTap(onBus: 0)
|
||||||
|
|
||||||
|
let recordingFormat = inputNode.outputFormat(forBus: 0)
|
||||||
|
inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { [weak self] buffer, _ in
|
||||||
|
guard let self else { return }
|
||||||
|
self.recognitionRequest?.append(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
self.audioEngine.prepare()
|
||||||
|
try self.audioEngine.start()
|
||||||
|
|
||||||
|
self.recognitionTask = self.speechRecognizer?.recognitionTask(with: request) { [weak self] result, error in
|
||||||
|
guard let self else { return }
|
||||||
|
if let error {
|
||||||
|
self.statusText = "Recognizer error: \(error.localizedDescription)"
|
||||||
|
self.isListening = false
|
||||||
|
if self.isEnabled {
|
||||||
|
Task {
|
||||||
|
try? await Task.sleep(nanoseconds: 700_000_000)
|
||||||
|
await self.start()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
guard let result else { return }
|
||||||
|
|
||||||
|
let transcript = result.bestTranscription.formattedString
|
||||||
|
if let cmd = self.extractCommand(from: transcript) {
|
||||||
|
if cmd != self.lastDispatched {
|
||||||
|
self.lastDispatched = cmd
|
||||||
|
self.statusText = "Triggered"
|
||||||
|
Task { [weak self] in
|
||||||
|
guard let self else { return }
|
||||||
|
await self.onCommand?(cmd)
|
||||||
|
if self.isEnabled {
|
||||||
|
await self.start()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func extractCommand(from transcript: String) -> String? {
|
||||||
|
let lower = transcript.lowercased()
|
||||||
|
guard let range = lower.range(of: "clawdis", options: .backwards) else { return nil }
|
||||||
|
let after = lower[range.upperBound...]
|
||||||
|
let trimmed = after.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
if trimmed.isEmpty { return nil }
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func configureAudioSession() throws {
|
||||||
|
let session = AVAudioSession.sharedInstance()
|
||||||
|
try session.setCategory(.playAndRecord, mode: .measurement, options: [
|
||||||
|
.duckOthers,
|
||||||
|
.mixWithOthers,
|
||||||
|
.allowBluetooth,
|
||||||
|
.defaultToSpeaker,
|
||||||
|
])
|
||||||
|
try session.setActive(true, options: [])
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func requestMicrophonePermission() async -> Bool {
|
||||||
|
await withCheckedContinuation { cont in
|
||||||
|
AVAudioSession.sharedInstance().requestRecordPermission { ok in
|
||||||
|
cont.resume(returning: ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func requestSpeechPermission() async -> Bool {
|
||||||
|
await withCheckedContinuation { cont in
|
||||||
|
SFSpeechRecognizer.requestAuthorization { status in
|
||||||
|
cont.resume(returning: status == .authorized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
56
apps/ios/project.yml
Normal file
56
apps/ios/project.yml
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
name: ClawdisNode
|
||||||
|
options:
|
||||||
|
bundleIdPrefix: com.steipete.clawdis
|
||||||
|
deploymentTarget:
|
||||||
|
iOS: "17.0"
|
||||||
|
xcodeVersion: "16.0"
|
||||||
|
|
||||||
|
packages:
|
||||||
|
ClawdisNodeKit:
|
||||||
|
path: ../shared/ClawdisNodeKit
|
||||||
|
|
||||||
|
targets:
|
||||||
|
ClawdisNode:
|
||||||
|
type: application
|
||||||
|
platform: iOS
|
||||||
|
sources:
|
||||||
|
- path: Sources
|
||||||
|
dependencies:
|
||||||
|
- package: ClawdisNodeKit
|
||||||
|
preBuildScripts:
|
||||||
|
- name: SwiftFormat (lint)
|
||||||
|
script: |
|
||||||
|
set -euo pipefail
|
||||||
|
if ! command -v swiftformat >/dev/null 2>&1; then
|
||||||
|
echo "error: swiftformat not found (brew install swiftformat)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
swiftformat --lint --config "$SRCROOT/../../.swiftformat" \
|
||||||
|
"$SRCROOT/Sources" \
|
||||||
|
"$SRCROOT/../shared/ClawdisNodeKit/Sources"
|
||||||
|
- name: SwiftLint
|
||||||
|
script: |
|
||||||
|
set -euo pipefail
|
||||||
|
if ! command -v swiftlint >/dev/null 2>&1; then
|
||||||
|
echo "error: swiftlint not found (brew install swiftlint)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
swiftlint lint --config "$SRCROOT/.swiftlint.yml"
|
||||||
|
settings:
|
||||||
|
base:
|
||||||
|
PRODUCT_BUNDLE_IDENTIFIER: com.steipete.clawdis.node
|
||||||
|
SWIFT_VERSION: "6.0"
|
||||||
|
info:
|
||||||
|
path: Sources/Info.plist
|
||||||
|
properties:
|
||||||
|
CFBundleDisplayName: Clawdis Node
|
||||||
|
UILaunchScreen: {}
|
||||||
|
UIApplicationSceneManifest:
|
||||||
|
UIApplicationSupportsMultipleScenes: false
|
||||||
|
UIBackgroundModes:
|
||||||
|
- audio
|
||||||
|
NSLocalNetworkUsageDescription: Clawdis Node discovers and connects to your Clawdis bridge on the local network.
|
||||||
|
NSBonjourServices:
|
||||||
|
- _clawdis-bridge._tcp
|
||||||
|
NSMicrophoneUsageDescription: Clawdis Node needs microphone access for voice wake.
|
||||||
|
NSSpeechRecognitionUsageDescription: Clawdis Node uses on-device speech recognition for voice wake.
|
||||||
196
docs/ios/spec.md
Normal file
196
docs/ios/spec.md
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
---
|
||||||
|
summary: "Plan for an iOS voice + screen (Canvas) node that connects via a secure Bonjour-discovered macOS bridge"
|
||||||
|
read_when:
|
||||||
|
- Designing iOS node + gateway integration
|
||||||
|
- Extending the Gateway protocol for node/screen commands
|
||||||
|
- Implementing Bonjour pairing or transport security
|
||||||
|
---
|
||||||
|
# iOS Node (internal) — Voice Trigger + Screen/Canvas
|
||||||
|
|
||||||
|
Status: design plan (internal/TestFlight) · Date: 2025-12-12
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
- Build an **iOS app** that acts as a **remote node** for Clawdis:
|
||||||
|
- **Voice trigger** (wake-word / always-listening intent) that forwards transcripts to the Gateway `agent` method.
|
||||||
|
- **Screen/Canvas** surface that the agent can control: navigate, draw/render, evaluate JS, snapshot.
|
||||||
|
- **Dead-simple setup**:
|
||||||
|
- Auto-discover the host on the local network via **Bonjour**.
|
||||||
|
- One-tap pairing with an approval prompt on the Mac.
|
||||||
|
- iOS is **never** a local gateway; it is always a remote node.
|
||||||
|
- Operational clarity:
|
||||||
|
- When iOS is backgrounded, voice may still run; **screen/canvas commands must fail fast** with a structured error.
|
||||||
|
- Provide **settings**: node display name, enable/disable voice wake, pairing status.
|
||||||
|
|
||||||
|
Non-goals (v1):
|
||||||
|
- Exposing the Node Gateway directly on the LAN.
|
||||||
|
- Supporting arbitrary third-party “plugins” on iOS.
|
||||||
|
- Perfect App Store compliance; this is **internal-only** initially.
|
||||||
|
|
||||||
|
## Current repo reality (constraints we respect)
|
||||||
|
- The Gateway WebSocket server binds to `127.0.0.1:18789` (`src/gateway/server.ts`) with an optional `CLAWDIS_GATEWAY_TOKEN`.
|
||||||
|
- macOS “Canvas” exists today, but is **mac-only** and controlled via mac app IPC (`clawdis-mac canvas ...`) rather than the Gateway protocol (`docs/mac/canvas.md`).
|
||||||
|
- Voice wake forwards via `GatewayChannel` to Gateway `agent` (mac app: `VoiceWakeForwarder` → `AgentRPC`).
|
||||||
|
|
||||||
|
## Recommended topology (B): macOS Bridge + loopback Gateway
|
||||||
|
Keep the Node gateway loopback-only; expose a dedicated **macOS bridge** to the LAN.
|
||||||
|
|
||||||
|
**iOS App** ⇄ (TLS + pairing) ⇄ **macOS Bridge** ⇄ (loopback) ⇄ **Gateway WS** (`ws://127.0.0.1:18789`)
|
||||||
|
|
||||||
|
Why:
|
||||||
|
- Preserves current threat model: Gateway remains local-only.
|
||||||
|
- Centralizes auth, rate limiting, and allowlisting in the bridge.
|
||||||
|
- Lets us unify “screen node” semantics across mac + iOS without exposing raw gateway methods.
|
||||||
|
|
||||||
|
## Security plan (internal, but still robust)
|
||||||
|
### Transport
|
||||||
|
- Bridge listens on LAN and uses **TLS**.
|
||||||
|
- Prefer **mutual authentication** (mTLS-like) or explicit public key pinning after pairing.
|
||||||
|
|
||||||
|
### Pairing
|
||||||
|
- Bonjour discovery shows a candidate “Clawdis Bridge” on the LAN.
|
||||||
|
- First connection:
|
||||||
|
1) iOS generates a keypair (Secure Enclave if available).
|
||||||
|
2) iOS connects to the bridge and requests pairing.
|
||||||
|
3) macOS app shows “Approve node” with node name + device metadata.
|
||||||
|
4) On approve, mac stores the node public key + permissions; iOS stores bridge identity + trust anchor in Keychain.
|
||||||
|
- Subsequent connections:
|
||||||
|
- The bridge requires the paired identity. Unpaired clients get a structured “not paired” error and no access.
|
||||||
|
|
||||||
|
### Authorization / scope control (bridge-side ACL)
|
||||||
|
The bridge must not be a raw proxy to every gateway method.
|
||||||
|
|
||||||
|
- Allow by default:
|
||||||
|
- `agent` (with guardrails; idempotency required)
|
||||||
|
- minimal `system-event` beacons (presence updates for the node)
|
||||||
|
- node/screen methods defined below (new protocol surface)
|
||||||
|
- Deny by default:
|
||||||
|
- anything that widens control without explicit intent (future “shell”, “files”, etc.)
|
||||||
|
- Rate limit:
|
||||||
|
- handshake attempts
|
||||||
|
- voice forwards per minute
|
||||||
|
- snapshot frequency / payload size
|
||||||
|
|
||||||
|
## Protocol unification: add “node/screen” to Gateway protocol
|
||||||
|
### Principle
|
||||||
|
Unify mac Canvas + iOS Canvas under a single conceptual surface:
|
||||||
|
- The agent talks to the Gateway using a stable method set (typed protocol).
|
||||||
|
- The Gateway routes node-targeted requests to:
|
||||||
|
- local mac Canvas implementation, or
|
||||||
|
- remote iOS node via the bridge
|
||||||
|
|
||||||
|
### Minimal protocol additions (v1)
|
||||||
|
Add to `src/gateway/protocol/schema.ts` (and regenerate Swift models):
|
||||||
|
|
||||||
|
**Identity**
|
||||||
|
- Node identity comes from `hello.client.instanceId` (stable), and `hello.client.mode = "node"` (or `"ios-node"`).
|
||||||
|
|
||||||
|
**Methods**
|
||||||
|
- `node.list` → list paired/connected nodes + capabilities
|
||||||
|
- `node.invoke` → send a command to a specific node
|
||||||
|
- Params: `{ nodeId, command, params, idempotencyKey }`
|
||||||
|
|
||||||
|
**Events**
|
||||||
|
- `node.event` → async node status/errors
|
||||||
|
- e.g. background/foreground transitions, voice availability, screen availability
|
||||||
|
|
||||||
|
### Node command set (screen-focused)
|
||||||
|
These are values for `node.invoke.command`:
|
||||||
|
- `screen.show` / `screen.hide`
|
||||||
|
- `screen.navigate` with `{ url }` (Canvas URL or https URL)
|
||||||
|
- `screen.eval` with `{ javaScript }`
|
||||||
|
- `screen.snapshot` with `{ maxWidth?, quality?, format? }`
|
||||||
|
- `screen.setMode` with `{ mode: "canvas" | "web" }`
|
||||||
|
|
||||||
|
Result pattern:
|
||||||
|
- Request is a standard `req/res` with `ok` / `error`.
|
||||||
|
- Long operations (loads, streaming drawing, etc.) may also emit `node.event` progress.
|
||||||
|
|
||||||
|
### Background behavior requirement
|
||||||
|
When iOS is backgrounded:
|
||||||
|
- Voice may still be active (subject to iOS suspension).
|
||||||
|
- **All `screen.*` commands must fail** with a stable error code, e.g.:
|
||||||
|
- `NODE_BACKGROUND_UNAVAILABLE`
|
||||||
|
- Include `retryable: true` and `retryAfterMs` if we want the agent to wait.
|
||||||
|
|
||||||
|
## iOS app architecture (SwiftUI)
|
||||||
|
### App structure
|
||||||
|
- Tab bar:
|
||||||
|
- **Canvas/Screen** (WKWebView + overlay chrome)
|
||||||
|
- **Voice** (status + last transcript + test)
|
||||||
|
- **Settings** (node name, voice wake toggle, pairing state, debug)
|
||||||
|
|
||||||
|
### Components
|
||||||
|
- `BridgeDiscovery`: Bonjour browse + resolve (Network.framework `NWBrowser`)
|
||||||
|
- `BridgeConnection`: TLS session + pairing handshake + reconnect
|
||||||
|
- `NodeRuntime`:
|
||||||
|
- Voice pipeline (wake-word + capture + forward)
|
||||||
|
- Screen pipeline (WKWebView controller + snapshot + eval)
|
||||||
|
- Background state tracking; enforces “screen unavailable in background”
|
||||||
|
|
||||||
|
### Voice in background (internal)
|
||||||
|
- Enable background audio mode (and required session configuration) so the mic pipeline can keep running when the user switches apps.
|
||||||
|
- If iOS suspends the app anyway, surface a clear node status (`node.event`) so operators can see voice is unavailable.
|
||||||
|
|
||||||
|
## Code sharing (macOS + iOS)
|
||||||
|
Create/expand SwiftPM targets so both apps share:
|
||||||
|
- `ClawdisProtocol` (generated models; platform-neutral)
|
||||||
|
- `ClawdisGatewayClient` (shared WS framing + hello/req/res + seq-gap handling)
|
||||||
|
- `ClawdisNodeKit` (node.invoke command types + error codes)
|
||||||
|
|
||||||
|
macOS continues to own:
|
||||||
|
- local Canvas implementation details (custom scheme handler serving on-disk HTML, window/panel presentation)
|
||||||
|
|
||||||
|
iOS owns:
|
||||||
|
- iOS-specific audio/speech + WKWebView presentation and lifecycle
|
||||||
|
|
||||||
|
## Repo layout
|
||||||
|
- iOS app: `apps/ios/` (XcodeGen `project.yml`)
|
||||||
|
- Shared Swift packages: `apps/shared/`
|
||||||
|
- Lint/format: iOS target runs `swiftformat --lint` + `swiftlint lint` using repo configs (`.swiftformat`, `.swiftlint.yml`).
|
||||||
|
|
||||||
|
Generate the Xcode project:
|
||||||
|
```bash
|
||||||
|
cd apps/ios
|
||||||
|
xcodegen generate
|
||||||
|
open ClawdisNode.xcodeproj
|
||||||
|
```
|
||||||
|
|
||||||
|
## Storage plan (private by default)
|
||||||
|
### iOS
|
||||||
|
- Canvas/workspace files (persistent, private):
|
||||||
|
- `Application Support/Clawdis/canvas/<sessionKey>/...`
|
||||||
|
- Snapshots / temp exports (evictable):
|
||||||
|
- `Library/Caches/Clawdis/canvas-snapshots/<sessionKey>/...`
|
||||||
|
- Credentials:
|
||||||
|
- Keychain (paired identity + bridge trust anchor)
|
||||||
|
|
||||||
|
### macOS
|
||||||
|
- Keep current Canvas root (already implemented):
|
||||||
|
- `~/Library/Application Support/Clawdis/canvas/<session>/...`
|
||||||
|
- Bridge state:
|
||||||
|
- `~/Library/Application Support/Clawdis/bridge/paired-nodes.json`
|
||||||
|
- `~/Library/Application Support/Clawdis/bridge/keys/...`
|
||||||
|
|
||||||
|
## Rollout plan (phased)
|
||||||
|
1) **Bridge discovery + pairing (mac + iOS)**
|
||||||
|
- Bonjour browse + resolve
|
||||||
|
- Approve prompt on mac
|
||||||
|
- Persist pairing in Keychain/App Support
|
||||||
|
2) **Voice-only node**
|
||||||
|
- iOS voice wake toggle
|
||||||
|
- Forward transcript to Gateway `agent` via bridge
|
||||||
|
- Presence beacons via `system-event` (or node.event)
|
||||||
|
3) **Protocol additions for nodes**
|
||||||
|
- Add `node.list` / `node.invoke` / `node.event` to Gateway
|
||||||
|
- Implement bridge routing + ACLs
|
||||||
|
4) **iOS screen/canvas**
|
||||||
|
- WKWebView screen surface
|
||||||
|
- `screen.navigate/eval/snapshot`
|
||||||
|
- Background fast-fail for `screen.*`
|
||||||
|
5) **Unify mac Canvas under the same node.invoke**
|
||||||
|
- Keep existing implementation, but expose it through the unified protocol path so the agent uses one API.
|
||||||
|
|
||||||
|
## Open questions
|
||||||
|
- Should `hello.client.mode` be `"node"` with `platform="ios ..."` or a distinct mode `"ios-node"`? (Presence filtering currently excludes `"cli"` only.)
|
||||||
|
- Do we want a “permissions” model per node (voice only vs voice+screen) at pairing time?
|
||||||
|
- Should “website mode” allow arbitrary https, or enforce an allowlist to reduce risk?
|
||||||
Reference in New Issue
Block a user