refactor: streamline node invoke handling

This commit is contained in:
Peter Steinberger
2026-01-04 16:23:46 +01:00
parent c0b248f291
commit fd95ededaa
8 changed files with 810 additions and 666 deletions

View File

@@ -67,7 +67,7 @@ final class LocationService: NSObject, CLLocationManagerDelegate {
} }
self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy) self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy)
let timeout = max(0, timeoutMs ?? 10_000) let timeout = max(0, timeoutMs ?? 10000)
return try await self.withTimeout(timeoutMs: timeout) { return try await self.withTimeout(timeoutMs: timeout) {
try await self.requestLocation() try await self.requestLocation()
} }
@@ -109,11 +109,11 @@ final class LocationService: NSObject, CLLocationManagerDelegate {
private static func accuracyValue(_ accuracy: ClawdbotLocationAccuracy) -> CLLocationAccuracy { private static func accuracyValue(_ accuracy: ClawdbotLocationAccuracy) -> CLLocationAccuracy {
switch accuracy { switch accuracy {
case .coarse: case .coarse:
return kCLLocationAccuracyKilometer kCLLocationAccuracyKilometer
case .balanced: case .balanced:
return kCLLocationAccuracyHundredMeters kCLLocationAccuracyHundredMeters
case .precise: case .precise:
return kCLLocationAccuracyBest kCLLocationAccuracyBest
} }
} }

View File

@@ -250,7 +250,9 @@ final class NodeAppModel {
return BridgeInvokeResponse( return BridgeInvokeResponse(
id: req.id, id: req.id,
ok: false, ok: false,
error: ClawdbotNodeError(code: .unavailable, message: "UNAVAILABLE: node not ready")) error: ClawdbotNodeError(
code: .unavailable,
message: "UNAVAILABLE: node not ready"))
} }
return await self.handleInvoke(req) return await self.handleInvoke(req)
}) })
@@ -454,13 +456,10 @@ final class NodeAppModel {
return false return false
} }
// swiftlint:disable:next function_body_length cyclomatic_complexity
private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse { private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
let command = req.command let command = req.command
if command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen."), if self.isBackgrounded, self.isBackgroundRestricted(command) {
self.isBackgrounded
{
return BridgeInvokeResponse( return BridgeInvokeResponse(
id: req.id, id: req.id,
ok: false, ok: false,
@@ -481,6 +480,46 @@ final class NodeAppModel {
do { do {
switch command { switch command {
case ClawdbotLocationCommand.get.rawValue: case ClawdbotLocationCommand.get.rawValue:
return try await self.handleLocationInvoke(req)
case ClawdbotCanvasCommand.present.rawValue,
ClawdbotCanvasCommand.hide.rawValue,
ClawdbotCanvasCommand.navigate.rawValue,
ClawdbotCanvasCommand.evalJS.rawValue,
ClawdbotCanvasCommand.snapshot.rawValue:
return try await self.handleCanvasInvoke(req)
case ClawdbotCanvasA2UICommand.reset.rawValue,
ClawdbotCanvasA2UICommand.push.rawValue,
ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
return try await self.handleCanvasA2UIInvoke(req)
case ClawdbotCameraCommand.list.rawValue,
ClawdbotCameraCommand.snap.rawValue,
ClawdbotCameraCommand.clip.rawValue:
return try await self.handleCameraInvoke(req)
case ClawdbotScreenCommand.record.rawValue:
return try await self.handleScreenRecordInvoke(req)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
} catch {
if command.hasPrefix("camera.") {
let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2)
}
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(code: .unavailable, message: error.localizedDescription))
}
}
private func isBackgroundRestricted(_ command: String) -> Bool {
command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen.")
}
private func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let mode = self.locationMode() let mode = self.locationMode()
guard mode != .off else { guard mode != .off else {
return BridgeInvokeResponse( return BridgeInvokeResponse(
@@ -503,7 +542,7 @@ final class NodeAppModel {
let desired = params.desiredAccuracy ?? let desired = params.desiredAccuracy ??
(self.isLocationPreciseEnabled() ? .precise : .balanced) (self.isLocationPreciseEnabled() ? .precise : .balanced)
let status = self.locationService.authorizationStatus() let status = self.locationService.authorizationStatus()
if status != .authorizedAlways && status != .authorizedWhenInUse { if status != .authorizedAlways, status != .authorizedWhenInUse {
return BridgeInvokeResponse( return BridgeInvokeResponse(
id: req.id, id: req.id,
ok: false, ok: false,
@@ -511,7 +550,7 @@ final class NodeAppModel {
code: .unavailable, code: .unavailable,
message: "LOCATION_PERMISSION_REQUIRED: grant Location permission")) message: "LOCATION_PERMISSION_REQUIRED: grant Location permission"))
} }
if self.isBackgrounded && status != .authorizedAlways { if self.isBackgrounded, status != .authorizedAlways {
return BridgeInvokeResponse( return BridgeInvokeResponse(
id: req.id, id: req.id,
ok: false, ok: false,
@@ -537,7 +576,10 @@ final class NodeAppModel {
source: nil) source: nil)
let json = try Self.encodePayload(payload) let json = try Self.encodePayload(payload)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
}
private func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
switch req.command {
case ClawdbotCanvasCommand.present.rawValue: case ClawdbotCanvasCommand.present.rawValue:
let params = (try? Self.decodeParams(ClawdbotCanvasPresentParams.self, from: req.paramsJSON)) ?? let params = (try? Self.decodeParams(ClawdbotCanvasPresentParams.self, from: req.paramsJSON)) ??
ClawdbotCanvasPresentParams() ClawdbotCanvasPresentParams()
@@ -548,21 +590,17 @@ final class NodeAppModel {
self.screen.navigate(to: url) self.screen.navigate(to: url)
} }
return BridgeInvokeResponse(id: req.id, ok: true) return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdbotCanvasCommand.hide.rawValue: case ClawdbotCanvasCommand.hide.rawValue:
return BridgeInvokeResponse(id: req.id, ok: true) return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdbotCanvasCommand.navigate.rawValue: case ClawdbotCanvasCommand.navigate.rawValue:
let params = try Self.decodeParams(ClawdbotCanvasNavigateParams.self, from: req.paramsJSON) let params = try Self.decodeParams(ClawdbotCanvasNavigateParams.self, from: req.paramsJSON)
self.screen.navigate(to: params.url) self.screen.navigate(to: params.url)
return BridgeInvokeResponse(id: req.id, ok: true) return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdbotCanvasCommand.evalJS.rawValue: case ClawdbotCanvasCommand.evalJS.rawValue:
let params = try Self.decodeParams(ClawdbotCanvasEvalParams.self, from: req.paramsJSON) let params = try Self.decodeParams(ClawdbotCanvasEvalParams.self, from: req.paramsJSON)
let result = try await self.screen.eval(javaScript: params.javaScript) let result = try await self.screen.eval(javaScript: params.javaScript)
let payload = try Self.encodePayload(["result": result]) let payload = try Self.encodePayload(["result": result])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdbotCanvasCommand.snapshot.rawValue: case ClawdbotCanvasCommand.snapshot.rawValue:
let params = try? Self.decodeParams(ClawdbotCanvasSnapshotParams.self, from: req.paramsJSON) let params = try? Self.decodeParams(ClawdbotCanvasSnapshotParams.self, from: req.paramsJSON)
let format = params?.format ?? .jpeg let format = params?.format ?? .jpeg
@@ -584,7 +622,17 @@ final class NodeAppModel {
"base64": base64, "base64": base64,
]) ])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
}
private func handleCanvasA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let command = req.command
switch command {
case ClawdbotCanvasA2UICommand.reset.rawValue: case ClawdbotCanvasA2UICommand.reset.rawValue:
guard let a2uiUrl = await self.resolveA2UIHostURL() else { guard let a2uiUrl = await self.resolveA2UIHostURL() else {
return BridgeInvokeResponse( return BridgeInvokeResponse(
@@ -611,7 +659,6 @@ final class NodeAppModel {
})() })()
""") """)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
case ClawdbotCanvasA2UICommand.push.rawValue, ClawdbotCanvasA2UICommand.pushJSONL.rawValue: case ClawdbotCanvasA2UICommand.push.rawValue, ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
let messages: [AnyCodable] let messages: [AnyCodable]
if command == ClawdbotCanvasA2UICommand.pushJSONL.rawValue { if command == ClawdbotCanvasA2UICommand.pushJSONL.rawValue {
@@ -660,7 +707,16 @@ final class NodeAppModel {
""" """
let resultJSON = try await self.screen.eval(javaScript: js) let resultJSON = try await self.screen.eval(javaScript: js)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
}
private func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
switch req.command {
case ClawdbotCameraCommand.list.rawValue: case ClawdbotCameraCommand.list.rawValue:
let devices = await self.camera.listDevices() let devices = await self.camera.listDevices()
struct Payload: Codable { struct Payload: Codable {
@@ -668,7 +724,6 @@ final class NodeAppModel {
} }
let payload = try Self.encodePayload(Payload(devices: devices)) let payload = try Self.encodePayload(Payload(devices: devices))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdbotCameraCommand.snap.rawValue: case ClawdbotCameraCommand.snap.rawValue:
self.showCameraHUD(text: "Taking photo…", kind: .photo) self.showCameraHUD(text: "Taking photo…", kind: .photo)
self.triggerCameraFlash() self.triggerCameraFlash()
@@ -689,7 +744,6 @@ final class NodeAppModel {
height: res.height)) height: res.height))
self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6) self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdbotCameraCommand.clip.rawValue: case ClawdbotCameraCommand.clip.rawValue:
let params = (try? Self.decodeParams(ClawdbotCameraClipParams.self, from: req.paramsJSON)) ?? let params = (try? Self.decodeParams(ClawdbotCameraClipParams.self, from: req.paramsJSON)) ??
ClawdbotCameraClipParams() ClawdbotCameraClipParams()
@@ -713,8 +767,15 @@ final class NodeAppModel {
hasAudio: res.hasAudio)) hasAudio: res.hasAudio))
self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8) self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
}
case ClawdbotScreenCommand.record.rawValue: private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let params = (try? Self.decodeParams(ClawdbotScreenRecordParams.self, from: req.paramsJSON)) ?? let params = (try? Self.decodeParams(ClawdbotScreenRecordParams.self, from: req.paramsJSON)) ??
ClawdbotScreenRecordParams() ClawdbotScreenRecordParams()
if let format = params.format, format.lowercased() != "mp4" { if let format = params.format, format.lowercased() != "mp4" {
@@ -749,23 +810,6 @@ final class NodeAppModel {
screenIndex: params.screenIndex, screenIndex: params.screenIndex,
hasAudio: params.includeAudio ?? true)) hasAudio: params.includeAudio ?? true))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
} catch {
if command.hasPrefix("camera.") {
let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2)
}
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(code: .unavailable, message: error.localizedDescription))
}
} }
private func locationMode() -> ClawdbotLocationMode { private func locationMode() -> ClawdbotLocationMode {

View File

@@ -40,7 +40,6 @@ final class ScreenRecordService: @unchecked Sendable {
} }
} }
// swiftlint:disable:next cyclomatic_complexity
func record( func record(
screenIndex: Int?, screenIndex: Int?,
durationMs: Int?, durationMs: Int?,
@@ -48,30 +47,96 @@ final class ScreenRecordService: @unchecked Sendable {
includeAudio: Bool?, includeAudio: Bool?,
outPath: String?) async throws -> String outPath: String?) async throws -> String
{ {
let config = try self.makeRecordConfig(
screenIndex: screenIndex,
durationMs: durationMs,
fps: fps,
includeAudio: includeAudio,
outPath: outPath)
let state = CaptureState()
let recordQueue = DispatchQueue(label: "com.clawdis.screenrecord")
try await self.startCapture(state: state, config: config, recordQueue: recordQueue)
try await Task.sleep(nanoseconds: UInt64(config.durationMs) * 1_000_000)
try await self.stopCapture()
try self.finalizeCapture(state: state)
try await self.finishWriting(state: state)
return config.outURL.path
}
private struct RecordConfig {
let durationMs: Int
let fpsValue: Double
let includeAudio: Bool
let outURL: URL
}
private func makeRecordConfig(
screenIndex: Int?,
durationMs: Int?,
fps: Double?,
includeAudio: Bool?,
outPath: String?) throws -> RecordConfig
{
if let idx = screenIndex, idx != 0 {
throw ScreenRecordError.invalidScreenIndex(idx)
}
let durationMs = Self.clampDurationMs(durationMs) let durationMs = Self.clampDurationMs(durationMs)
let fps = Self.clampFps(fps) let fps = Self.clampFps(fps)
let fpsInt = Int32(fps.rounded()) let fpsInt = Int32(fps.rounded())
let fpsValue = Double(fpsInt) let fpsValue = Double(fpsInt)
let includeAudio = includeAudio ?? true let includeAudio = includeAudio ?? true
if let idx = screenIndex, idx != 0 { let outURL = self.makeOutputURL(outPath: outPath)
throw ScreenRecordError.invalidScreenIndex(idx) try? FileManager.default.removeItem(at: outURL)
return RecordConfig(
durationMs: durationMs,
fpsValue: fpsValue,
includeAudio: includeAudio,
outURL: outURL)
} }
let outURL: URL = { private func makeOutputURL(outPath: String?) -> URL {
if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return URL(fileURLWithPath: outPath) return URL(fileURLWithPath: outPath)
} }
return FileManager.default.temporaryDirectory return FileManager.default.temporaryDirectory
.appendingPathComponent("clawdbot-screen-record-\(UUID().uuidString).mp4") .appendingPathComponent("clawdbot-screen-record-\(UUID().uuidString).mp4")
}() }
try? FileManager.default.removeItem(at: outURL)
let state = CaptureState()
let recordQueue = DispatchQueue(label: "com.clawdbot.screenrecord")
private func startCapture(
state: CaptureState,
config: RecordConfig,
recordQueue: DispatchQueue) async throws
{
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
let handler: @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void = { sample, type, error in let handler = self.makeCaptureHandler(
state: state,
config: config,
recordQueue: recordQueue)
let completion: @Sendable (Error?) -> Void = { error in
if let error { cont.resume(throwing: error) } else { cont.resume() }
}
Task { @MainActor in
startReplayKitCapture(
includeAudio: config.includeAudio,
handler: handler,
completion: completion)
}
}
}
private func makeCaptureHandler(
state: CaptureState,
config: RecordConfig,
recordQueue: DispatchQueue) -> @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void
{
{ sample, type, error in
// ReplayKit can call the capture handler on a background queue. // ReplayKit can call the capture handler on a background queue.
// Serialize writes to avoid queue asserts. // Serialize writes to avoid queue asserts.
recordQueue.async { recordQueue.async {
@@ -85,68 +150,33 @@ final class ScreenRecordService: @unchecked Sendable {
switch type { switch type {
case .video: case .video:
self.handleVideoSample(sample, state: state, config: config)
case .audioApp, .audioMic:
self.handleAudioSample(sample, state: state, includeAudio: config.includeAudio)
@unknown default:
break
}
}
}
}
private func handleVideoSample(
_ sample: CMSampleBuffer,
state: CaptureState,
config: RecordConfig)
{
let pts = CMSampleBufferGetPresentationTimeStamp(sample) let pts = CMSampleBufferGetPresentationTimeStamp(sample)
let shouldSkip = state.withLock { state in let shouldSkip = state.withLock { state in
if let lastVideoTime = state.lastVideoTime { if let lastVideoTime = state.lastVideoTime {
let delta = CMTimeSubtract(pts, lastVideoTime) let delta = CMTimeSubtract(pts, lastVideoTime)
return delta.seconds < (1.0 / fpsValue) return delta.seconds < (1.0 / config.fpsValue)
} }
return false return false
} }
if shouldSkip { return } if shouldSkip { return }
if state.withLock({ $0.writer == nil }) { if state.withLock({ $0.writer == nil }) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else { self.prepareWriter(sample: sample, state: state, config: config, pts: pts)
state.withLock { state in
if state.handlerError == nil {
state.handlerError = ScreenRecordError.captureFailed("Missing image buffer")
}
}
return
}
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
do {
let w = try AVAssetWriter(outputURL: outURL, fileType: .mp4)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
vInput.expectsMediaDataInRealTime = true
guard w.canAdd(vInput) else {
throw ScreenRecordError.writeFailed("Cannot add video input")
}
w.add(vInput)
if includeAudio {
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
aInput.expectsMediaDataInRealTime = true
if w.canAdd(aInput) {
w.add(aInput)
state.withLock { state in
state.audioInput = aInput
}
}
}
guard w.startWriting() else {
throw ScreenRecordError
.writeFailed(w.error?.localizedDescription ?? "Failed to start writer")
}
w.startSession(atSourceTime: pts)
state.withLock { state in
state.writer = w
state.videoInput = vInput
state.started = true
}
} catch {
state.withLock { state in
if state.handlerError == nil { state.handlerError = error }
}
return
}
} }
let vInput = state.withLock { $0.videoInput } let vInput = state.withLock { $0.videoInput }
@@ -169,44 +199,92 @@ final class ScreenRecordService: @unchecked Sendable {
} }
} }
} }
}
case .audioApp, .audioMic: private func prepareWriter(
sample: CMSampleBuffer,
state: CaptureState,
config: RecordConfig,
pts: CMTime)
{
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
state.withLock { state in
if state.handlerError == nil {
state.handlerError = ScreenRecordError.captureFailed("Missing image buffer")
}
}
return
}
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
do {
let writer = try AVAssetWriter(outputURL: config.outURL, fileType: .mp4)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
vInput.expectsMediaDataInRealTime = true
guard writer.canAdd(vInput) else {
throw ScreenRecordError.writeFailed("Cannot add video input")
}
writer.add(vInput)
if config.includeAudio {
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
aInput.expectsMediaDataInRealTime = true
if writer.canAdd(aInput) {
writer.add(aInput)
state.withLock { state in
state.audioInput = aInput
}
}
}
guard writer.startWriting() else {
throw ScreenRecordError.writeFailed(
writer.error?.localizedDescription ?? "Failed to start writer")
}
writer.startSession(atSourceTime: pts)
state.withLock { state in
state.writer = writer
state.videoInput = vInput
state.started = true
}
} catch {
state.withLock { state in
if state.handlerError == nil { state.handlerError = error }
}
}
}
private func handleAudioSample(
_ sample: CMSampleBuffer,
state: CaptureState,
includeAudio: Bool)
{
let aInput = state.withLock { $0.audioInput } let aInput = state.withLock { $0.audioInput }
let isStarted = state.withLock { $0.started } let isStarted = state.withLock { $0.started }
guard includeAudio, let aInput, isStarted else { return } guard includeAudio, let aInput, isStarted else { return }
if aInput.isReadyForMoreMediaData { if aInput.isReadyForMoreMediaData {
_ = aInput.append(sample) _ = aInput.append(sample)
} }
@unknown default:
break
}
}
} }
let completion: @Sendable (Error?) -> Void = { error in private func stopCapture() async throws {
if let error { cont.resume(throwing: error) } else { cont.resume() }
}
Task { @MainActor in
startReplayKitCapture(
includeAudio: includeAudio,
handler: handler,
completion: completion)
}
}
try await Task.sleep(nanoseconds: UInt64(durationMs) * 1_000_000)
let stopError = await withCheckedContinuation { cont in let stopError = await withCheckedContinuation { cont in
Task { @MainActor in Task { @MainActor in
stopReplayKitCapture { error in cont.resume(returning: error) } stopReplayKitCapture { error in cont.resume(returning: error) }
} }
} }
if let stopError { throw stopError } if let stopError { throw stopError }
}
let handlerErrorSnapshot = state.withLock { $0.handlerError } private func finalizeCapture(state: CaptureState) throws {
if let handlerErrorSnapshot { throw handlerErrorSnapshot } if let handlerErrorSnapshot = state.withLock({ $0.handlerError }) {
throw handlerErrorSnapshot
}
let writerSnapshot = state.withLock { $0.writer } let writerSnapshot = state.withLock { $0.writer }
let videoInputSnapshot = state.withLock { $0.videoInput } let videoInputSnapshot = state.withLock { $0.videoInput }
let audioInputSnapshot = state.withLock { $0.audioInput } let audioInputSnapshot = state.withLock { $0.audioInput }
@@ -217,7 +295,13 @@ final class ScreenRecordService: @unchecked Sendable {
videoInputSnapshot.markAsFinished() videoInputSnapshot.markAsFinished()
audioInputSnapshot?.markAsFinished() audioInputSnapshot?.markAsFinished()
_ = writerSnapshot
}
private func finishWriting(state: CaptureState) async throws {
guard let writerSnapshot = state.withLock({ $0.writer }) else {
throw ScreenRecordError.captureFailed("Missing writer")
}
let writerBox = UncheckedSendableBox(value: writerSnapshot) let writerBox = UncheckedSendableBox(value: writerSnapshot)
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
writerBox.value.finishWriting { writerBox.value.finishWriting {
@@ -231,8 +315,6 @@ final class ScreenRecordService: @unchecked Sendable {
} }
} }
} }
return outURL.path
} }
private nonisolated static func clampDurationMs(_ ms: Int?) -> Int { private nonisolated static func clampDurationMs(_ ms: Int?) -> Int {

View File

@@ -288,9 +288,9 @@ final class TalkModeManager: NSObject {
self.chatSubscribedSessionKeys.insert(key) self.chatSubscribedSessionKeys.insert(key)
self.logger.info("chat.subscribe ok sessionKey=\(key, privacy: .public)") self.logger.info("chat.subscribe ok sessionKey=\(key, privacy: .public)")
} catch { } catch {
self.logger self.logger.warning(
.warning( "chat.subscribe failed sessionKey=\(key, privacy: .public) " +
"chat.subscribe failed sessionKey=\(key, privacy: .public) err=\(error.localizedDescription, privacy: .public)") "err=\(error.localizedDescription, privacy: .public)")
} }
} }
@@ -340,7 +340,12 @@ final class TalkModeManager: NSObject {
"idempotencyKey": UUID().uuidString, "idempotencyKey": UUID().uuidString,
] ]
let data = try JSONSerialization.data(withJSONObject: payload) let data = try JSONSerialization.data(withJSONObject: payload)
let json = String(decoding: data, as: UTF8.self) guard let json = String(bytes: data, encoding: .utf8) else {
throw NSError(
domain: "TalkModeManager",
code: 1,
userInfo: [NSLocalizedDescriptionKey: "Failed to encode chat payload"])
}
let res = try await bridge.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 30) let res = try await bridge.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 30)
let decoded = try JSONDecoder().decode(SendResponse.self, from: res) let decoded = try JSONDecoder().decode(SendResponse.self, from: res)
return decoded.runId return decoded.runId
@@ -523,9 +528,9 @@ final class TalkModeManager: NSObject {
self.lastPlaybackWasPCM = false self.lastPlaybackWasPCM = false
result = await self.mp3Player.play(stream: stream) result = await self.mp3Player.play(stream: stream)
} }
self.logger self.logger.info(
.info( "elevenlabs stream finished=\(result.finished, privacy: .public) " +
"elevenlabs stream finished=\(result.finished, privacy: .public) dur=\(Date().timeIntervalSince(started), privacy: .public)s") "dur=\(Date().timeIntervalSince(started), privacy: .public)s")
if !result.finished, let interruptedAt = result.interruptedAt { if !result.finished, let interruptedAt = result.interruptedAt {
self.lastInterruptedAtSeconds = interruptedAt self.lastInterruptedAtSeconds = interruptedAt
} }

View File

@@ -192,6 +192,4 @@ actor MacNodeBridgePairingClient {
} }
} }
} }
} }

View File

@@ -325,6 +325,4 @@ actor MacNodeBridgeSession {
]) ])
}) })
} }
} }

View File

@@ -47,7 +47,7 @@ final class MacNodeLocationService: NSObject, CLLocationManagerDelegate {
} }
self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy) self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy)
let timeout = max(0, timeoutMs ?? 10_000) let timeout = max(0, timeoutMs ?? 10000)
return try await self.withTimeout(timeoutMs: timeout) { return try await self.withTimeout(timeoutMs: timeout) {
try await self.requestLocation() try await self.requestLocation()
} }
@@ -83,11 +83,11 @@ final class MacNodeLocationService: NSObject, CLLocationManagerDelegate {
private static func accuracyValue(_ accuracy: ClawdbotLocationAccuracy) -> CLLocationAccuracy { private static func accuracyValue(_ accuracy: ClawdbotLocationAccuracy) -> CLLocationAccuracy {
switch accuracy { switch accuracy {
case .coarse: case .coarse:
return kCLLocationAccuracyKilometer kCLLocationAccuracyKilometer
case .balanced: case .balanced:
return kCLLocationAccuracyHundredMeters kCLLocationAccuracyHundredMeters
case .precise: case .precise:
return kCLLocationAccuracyBest kCLLocationAccuracyBest
} }
} }

View File

@@ -8,10 +8,9 @@ actor MacNodeRuntime {
@MainActor private let screenRecorder = ScreenRecordService() @MainActor private let screenRecorder = ScreenRecordService()
@MainActor private let locationService = MacNodeLocationService() @MainActor private let locationService = MacNodeLocationService()
// swiftlint:disable:next function_body_length cyclomatic_complexity
func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse { func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
let command = req.command let command = req.command
if command.hasPrefix("canvas.") || command.hasPrefix("canvas.a2ui."), !Self.canvasEnabled() { if self.isCanvasCommand(command), !Self.canvasEnabled() {
return BridgeInvokeResponse( return BridgeInvokeResponse(
id: req.id, id: req.id,
ok: false, ok: false,
@@ -21,6 +20,42 @@ actor MacNodeRuntime {
} }
do { do {
switch command { switch command {
case ClawdbotCanvasCommand.present.rawValue,
ClawdbotCanvasCommand.hide.rawValue,
ClawdbotCanvasCommand.navigate.rawValue,
ClawdbotCanvasCommand.evalJS.rawValue,
ClawdbotCanvasCommand.snapshot.rawValue:
return try await self.handleCanvasInvoke(req)
case ClawdbotCanvasA2UICommand.reset.rawValue,
ClawdbotCanvasA2UICommand.push.rawValue,
ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
return try await self.handleA2UIInvoke(req)
case ClawdbotCameraCommand.snap.rawValue,
ClawdbotCameraCommand.clip.rawValue,
ClawdbotCameraCommand.list.rawValue:
return try await self.handleCameraInvoke(req)
case ClawdbotLocationCommand.get.rawValue:
return try await self.handleLocationInvoke(req)
case MacNodeScreenCommand.record.rawValue:
return try await self.handleScreenRecordInvoke(req)
case ClawdbotSystemCommand.run.rawValue:
return try await self.handleSystemRun(req)
case ClawdbotSystemCommand.notify.rawValue:
return try await self.handleSystemNotify(req)
default:
return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
}
} catch {
return Self.errorResponse(req, code: .unavailable, message: error.localizedDescription)
}
}
private func isCanvasCommand(_ command: String) -> Bool {
command.hasPrefix("canvas.") || command.hasPrefix("canvas.a2ui.")
}
private func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
switch req.command {
case ClawdbotCanvasCommand.present.rawValue: case ClawdbotCanvasCommand.present.rawValue:
let params = (try? Self.decodeParams(ClawdbotCanvasPresentParams.self, from: req.paramsJSON)) ?? let params = (try? Self.decodeParams(ClawdbotCanvasPresentParams.self, from: req.paramsJSON)) ??
ClawdbotCanvasPresentParams() ClawdbotCanvasPresentParams()
@@ -36,20 +71,17 @@ actor MacNodeRuntime {
placement: placement) placement: placement)
} }
return BridgeInvokeResponse(id: req.id, ok: true) return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdbotCanvasCommand.hide.rawValue: case ClawdbotCanvasCommand.hide.rawValue:
await MainActor.run { await MainActor.run {
CanvasManager.shared.hide(sessionKey: "main") CanvasManager.shared.hide(sessionKey: "main")
} }
return BridgeInvokeResponse(id: req.id, ok: true) return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdbotCanvasCommand.navigate.rawValue: case ClawdbotCanvasCommand.navigate.rawValue:
let params = try Self.decodeParams(ClawdbotCanvasNavigateParams.self, from: req.paramsJSON) let params = try Self.decodeParams(ClawdbotCanvasNavigateParams.self, from: req.paramsJSON)
try await MainActor.run { try await MainActor.run {
_ = try CanvasManager.shared.show(sessionKey: "main", path: params.url) _ = try CanvasManager.shared.show(sessionKey: "main", path: params.url)
} }
return BridgeInvokeResponse(id: req.id, ok: true) return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdbotCanvasCommand.evalJS.rawValue: case ClawdbotCanvasCommand.evalJS.rawValue:
let params = try Self.decodeParams(ClawdbotCanvasEvalParams.self, from: req.paramsJSON) let params = try Self.decodeParams(ClawdbotCanvasEvalParams.self, from: req.paramsJSON)
let result = try await CanvasManager.shared.eval( let result = try await CanvasManager.shared.eval(
@@ -57,7 +89,6 @@ actor MacNodeRuntime {
javaScript: params.javaScript) javaScript: params.javaScript)
let payload = try Self.encodePayload(["result": result] as [String: String]) let payload = try Self.encodePayload(["result": result] as [String: String])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdbotCanvasCommand.snapshot.rawValue: case ClawdbotCanvasCommand.snapshot.rawValue:
let params = try? Self.decodeParams(ClawdbotCanvasSnapshotParams.self, from: req.paramsJSON) let params = try? Self.decodeParams(ClawdbotCanvasSnapshotParams.self, from: req.paramsJSON)
let format = params?.format ?? .jpeg let format = params?.format ?? .jpeg
@@ -86,14 +117,24 @@ actor MacNodeRuntime {
"base64": encoded.base64EncodedString(), "base64": encoded.base64EncodedString(),
]) ])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
}
}
private func handleA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
switch req.command {
case ClawdbotCanvasA2UICommand.reset.rawValue: case ClawdbotCanvasA2UICommand.reset.rawValue:
return try await self.handleA2UIReset(req) try await self.handleA2UIReset(req)
case ClawdbotCanvasA2UICommand.push.rawValue,
ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
try await self.handleA2UIPush(req)
default:
Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
}
}
case ClawdbotCanvasA2UICommand.push.rawValue, ClawdbotCanvasA2UICommand.pushJSONL.rawValue: private func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
return try await self.handleA2UIPush(req)
case ClawdbotCameraCommand.snap.rawValue:
guard Self.cameraEnabled() else { guard Self.cameraEnabled() else {
return BridgeInvokeResponse( return BridgeInvokeResponse(
id: req.id, id: req.id,
@@ -102,9 +143,11 @@ actor MacNodeRuntime {
code: .unavailable, code: .unavailable,
message: "CAMERA_DISABLED: enable Camera in Settings")) message: "CAMERA_DISABLED: enable Camera in Settings"))
} }
switch req.command {
case ClawdbotCameraCommand.snap.rawValue:
let params = (try? Self.decodeParams(ClawdbotCameraSnapParams.self, from: req.paramsJSON)) ?? let params = (try? Self.decodeParams(ClawdbotCameraSnapParams.self, from: req.paramsJSON)) ??
ClawdbotCameraSnapParams() ClawdbotCameraSnapParams()
let delayMs = min(10_000, max(0, params.delayMs ?? 2000)) let delayMs = min(10000, max(0, params.delayMs ?? 2000))
let res = try await self.cameraCapture.snap( let res = try await self.cameraCapture.snap(
facing: CameraFacing(rawValue: params.facing?.rawValue ?? "") ?? .front, facing: CameraFacing(rawValue: params.facing?.rawValue ?? "") ?? .front,
maxWidth: params.maxWidth, maxWidth: params.maxWidth,
@@ -123,16 +166,7 @@ actor MacNodeRuntime {
width: Int(res.size.width), width: Int(res.size.width),
height: Int(res.size.height))) height: Int(res.size.height)))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdbotCameraCommand.clip.rawValue: case ClawdbotCameraCommand.clip.rawValue:
guard Self.cameraEnabled() else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(
code: .unavailable,
message: "CAMERA_DISABLED: enable Camera in Settings"))
}
let params = (try? Self.decodeParams(ClawdbotCameraClipParams.self, from: req.paramsJSON)) ?? let params = (try? Self.decodeParams(ClawdbotCameraClipParams.self, from: req.paramsJSON)) ??
ClawdbotCameraClipParams() ClawdbotCameraClipParams()
let res = try await self.cameraCapture.clip( let res = try await self.cameraCapture.clip(
@@ -155,21 +189,16 @@ actor MacNodeRuntime {
durationMs: res.durationMs, durationMs: res.durationMs,
hasAudio: res.hasAudio)) hasAudio: res.hasAudio))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdbotCameraCommand.list.rawValue: case ClawdbotCameraCommand.list.rawValue:
guard Self.cameraEnabled() else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdbotNodeError(
code: .unavailable,
message: "CAMERA_DISABLED: enable Camera in Settings"))
}
let devices = await self.cameraCapture.listDevices() let devices = await self.cameraCapture.listDevices()
let payload = try Self.encodePayload(["devices": devices]) let payload = try Self.encodePayload(["devices": devices])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
}
}
case ClawdbotLocationCommand.get.rawValue: private func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let mode = Self.locationMode() let mode = Self.locationMode()
guard mode != .off else { guard mode != .off else {
return BridgeInvokeResponse( return BridgeInvokeResponse(
@@ -184,7 +213,7 @@ actor MacNodeRuntime {
let desired = params.desiredAccuracy ?? let desired = params.desiredAccuracy ??
(Self.locationPreciseEnabled() ? .precise : .balanced) (Self.locationPreciseEnabled() ? .precise : .balanced)
let status = await self.locationService.authorizationStatus() let status = await self.locationService.authorizationStatus()
if status != .authorizedAlways && status != .authorizedWhenInUse { if status != .authorizedAlways {
return BridgeInvokeResponse( return BridgeInvokeResponse(
id: req.id, id: req.id,
ok: false, ok: false,
@@ -225,8 +254,9 @@ actor MacNodeRuntime {
code: .unavailable, code: .unavailable,
message: "LOCATION_UNAVAILABLE: \(error.localizedDescription)")) message: "LOCATION_UNAVAILABLE: \(error.localizedDescription)"))
} }
}
case MacNodeScreenCommand.record.rawValue: private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let params = (try? Self.decodeParams(MacNodeScreenRecordParams.self, from: req.paramsJSON)) ?? let params = (try? Self.decodeParams(MacNodeScreenRecordParams.self, from: req.paramsJSON)) ??
MacNodeScreenRecordParams() MacNodeScreenRecordParams()
if let format = params.format?.lowercased(), !format.isEmpty, format != "mp4" { if let format = params.format?.lowercased(), !format.isEmpty, format != "mp4" {
@@ -259,19 +289,6 @@ actor MacNodeRuntime {
screenIndex: params.screenIndex, screenIndex: params.screenIndex,
hasAudio: res.hasAudio)) hasAudio: res.hasAudio))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdbotSystemCommand.run.rawValue:
return try await self.handleSystemRun(req)
case ClawdbotSystemCommand.notify.rawValue:
return try await self.handleSystemNotify(req)
default:
return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
}
} catch {
return Self.errorResponse(req, code: .unavailable, message: error.localizedDescription)
}
} }
private func handleA2UIReset(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { private func handleA2UIReset(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {