refactor: streamline node invoke handling
This commit is contained in:
@@ -67,7 +67,7 @@ final class LocationService: NSObject, CLLocationManagerDelegate {
|
||||
}
|
||||
|
||||
self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy)
|
||||
let timeout = max(0, timeoutMs ?? 10_000)
|
||||
let timeout = max(0, timeoutMs ?? 10000)
|
||||
return try await self.withTimeout(timeoutMs: timeout) {
|
||||
try await self.requestLocation()
|
||||
}
|
||||
@@ -109,11 +109,11 @@ final class LocationService: NSObject, CLLocationManagerDelegate {
|
||||
private static func accuracyValue(_ accuracy: ClawdbotLocationAccuracy) -> CLLocationAccuracy {
|
||||
switch accuracy {
|
||||
case .coarse:
|
||||
return kCLLocationAccuracyKilometer
|
||||
kCLLocationAccuracyKilometer
|
||||
case .balanced:
|
||||
return kCLLocationAccuracyHundredMeters
|
||||
kCLLocationAccuracyHundredMeters
|
||||
case .precise:
|
||||
return kCLLocationAccuracyBest
|
||||
kCLLocationAccuracyBest
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -250,7 +250,9 @@ final class NodeAppModel {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(code: .unavailable, message: "UNAVAILABLE: node not ready"))
|
||||
error: ClawdbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "UNAVAILABLE: node not ready"))
|
||||
}
|
||||
return await self.handleInvoke(req)
|
||||
})
|
||||
@@ -454,13 +456,10 @@ final class NodeAppModel {
|
||||
return false
|
||||
}
|
||||
|
||||
// swiftlint:disable:next function_body_length cyclomatic_complexity
|
||||
private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
|
||||
let command = req.command
|
||||
|
||||
if command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen."),
|
||||
self.isBackgrounded
|
||||
{
|
||||
if self.isBackgrounded, self.isBackgroundRestricted(command) {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
@@ -481,6 +480,46 @@ final class NodeAppModel {
|
||||
do {
|
||||
switch command {
|
||||
case ClawdbotLocationCommand.get.rawValue:
|
||||
return try await self.handleLocationInvoke(req)
|
||||
case ClawdbotCanvasCommand.present.rawValue,
|
||||
ClawdbotCanvasCommand.hide.rawValue,
|
||||
ClawdbotCanvasCommand.navigate.rawValue,
|
||||
ClawdbotCanvasCommand.evalJS.rawValue,
|
||||
ClawdbotCanvasCommand.snapshot.rawValue:
|
||||
return try await self.handleCanvasInvoke(req)
|
||||
case ClawdbotCanvasA2UICommand.reset.rawValue,
|
||||
ClawdbotCanvasA2UICommand.push.rawValue,
|
||||
ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
|
||||
return try await self.handleCanvasA2UIInvoke(req)
|
||||
case ClawdbotCameraCommand.list.rawValue,
|
||||
ClawdbotCameraCommand.snap.rawValue,
|
||||
ClawdbotCameraCommand.clip.rawValue:
|
||||
return try await self.handleCameraInvoke(req)
|
||||
case ClawdbotScreenCommand.record.rawValue:
|
||||
return try await self.handleScreenRecordInvoke(req)
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
} catch {
|
||||
if command.hasPrefix("camera.") {
|
||||
let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
|
||||
self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2)
|
||||
}
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(code: .unavailable, message: error.localizedDescription))
|
||||
}
|
||||
}
|
||||
|
||||
private func isBackgroundRestricted(_ command: String) -> Bool {
|
||||
command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen.")
|
||||
}
|
||||
|
||||
private func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let mode = self.locationMode()
|
||||
guard mode != .off else {
|
||||
return BridgeInvokeResponse(
|
||||
@@ -503,7 +542,7 @@ final class NodeAppModel {
|
||||
let desired = params.desiredAccuracy ??
|
||||
(self.isLocationPreciseEnabled() ? .precise : .balanced)
|
||||
let status = self.locationService.authorizationStatus()
|
||||
if status != .authorizedAlways && status != .authorizedWhenInUse {
|
||||
if status != .authorizedAlways, status != .authorizedWhenInUse {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
@@ -511,7 +550,7 @@ final class NodeAppModel {
|
||||
code: .unavailable,
|
||||
message: "LOCATION_PERMISSION_REQUIRED: grant Location permission"))
|
||||
}
|
||||
if self.isBackgrounded && status != .authorizedAlways {
|
||||
if self.isBackgrounded, status != .authorizedAlways {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
@@ -537,7 +576,10 @@ final class NodeAppModel {
|
||||
source: nil)
|
||||
let json = try Self.encodePayload(payload)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
|
||||
}
|
||||
|
||||
private func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
switch req.command {
|
||||
case ClawdbotCanvasCommand.present.rawValue:
|
||||
let params = (try? Self.decodeParams(ClawdbotCanvasPresentParams.self, from: req.paramsJSON)) ??
|
||||
ClawdbotCanvasPresentParams()
|
||||
@@ -548,21 +590,17 @@ final class NodeAppModel {
|
||||
self.screen.navigate(to: url)
|
||||
}
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
|
||||
case ClawdbotCanvasCommand.hide.rawValue:
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
|
||||
case ClawdbotCanvasCommand.navigate.rawValue:
|
||||
let params = try Self.decodeParams(ClawdbotCanvasNavigateParams.self, from: req.paramsJSON)
|
||||
self.screen.navigate(to: params.url)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
|
||||
case ClawdbotCanvasCommand.evalJS.rawValue:
|
||||
let params = try Self.decodeParams(ClawdbotCanvasEvalParams.self, from: req.paramsJSON)
|
||||
let result = try await self.screen.eval(javaScript: params.javaScript)
|
||||
let payload = try Self.encodePayload(["result": result])
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
|
||||
case ClawdbotCanvasCommand.snapshot.rawValue:
|
||||
let params = try? Self.decodeParams(ClawdbotCanvasSnapshotParams.self, from: req.paramsJSON)
|
||||
let format = params?.format ?? .jpeg
|
||||
@@ -584,7 +622,17 @@ final class NodeAppModel {
|
||||
"base64": base64,
|
||||
])
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
}
|
||||
|
||||
private func handleCanvasA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let command = req.command
|
||||
switch command {
|
||||
case ClawdbotCanvasA2UICommand.reset.rawValue:
|
||||
guard let a2uiUrl = await self.resolveA2UIHostURL() else {
|
||||
return BridgeInvokeResponse(
|
||||
@@ -611,7 +659,6 @@ final class NodeAppModel {
|
||||
})()
|
||||
""")
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
|
||||
|
||||
case ClawdbotCanvasA2UICommand.push.rawValue, ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
|
||||
let messages: [AnyCodable]
|
||||
if command == ClawdbotCanvasA2UICommand.pushJSONL.rawValue {
|
||||
@@ -660,7 +707,16 @@ final class NodeAppModel {
|
||||
"""
|
||||
let resultJSON = try await self.screen.eval(javaScript: js)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON)
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
}
|
||||
|
||||
private func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
switch req.command {
|
||||
case ClawdbotCameraCommand.list.rawValue:
|
||||
let devices = await self.camera.listDevices()
|
||||
struct Payload: Codable {
|
||||
@@ -668,7 +724,6 @@ final class NodeAppModel {
|
||||
}
|
||||
let payload = try Self.encodePayload(Payload(devices: devices))
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
|
||||
case ClawdbotCameraCommand.snap.rawValue:
|
||||
self.showCameraHUD(text: "Taking photo…", kind: .photo)
|
||||
self.triggerCameraFlash()
|
||||
@@ -689,7 +744,6 @@ final class NodeAppModel {
|
||||
height: res.height))
|
||||
self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
|
||||
case ClawdbotCameraCommand.clip.rawValue:
|
||||
let params = (try? Self.decodeParams(ClawdbotCameraClipParams.self, from: req.paramsJSON)) ??
|
||||
ClawdbotCameraClipParams()
|
||||
@@ -713,8 +767,15 @@ final class NodeAppModel {
|
||||
hasAudio: res.hasAudio))
|
||||
self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
}
|
||||
|
||||
case ClawdbotScreenCommand.record.rawValue:
|
||||
private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let params = (try? Self.decodeParams(ClawdbotScreenRecordParams.self, from: req.paramsJSON)) ??
|
||||
ClawdbotScreenRecordParams()
|
||||
if let format = params.format, format.lowercased() != "mp4" {
|
||||
@@ -749,23 +810,6 @@ final class NodeAppModel {
|
||||
screenIndex: params.screenIndex,
|
||||
hasAudio: params.includeAudio ?? true))
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
} catch {
|
||||
if command.hasPrefix("camera.") {
|
||||
let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
|
||||
self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2)
|
||||
}
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(code: .unavailable, message: error.localizedDescription))
|
||||
}
|
||||
}
|
||||
|
||||
private func locationMode() -> ClawdbotLocationMode {
|
||||
|
||||
@@ -40,7 +40,6 @@ final class ScreenRecordService: @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
// swiftlint:disable:next cyclomatic_complexity
|
||||
func record(
|
||||
screenIndex: Int?,
|
||||
durationMs: Int?,
|
||||
@@ -48,30 +47,96 @@ final class ScreenRecordService: @unchecked Sendable {
|
||||
includeAudio: Bool?,
|
||||
outPath: String?) async throws -> String
|
||||
{
|
||||
let config = try self.makeRecordConfig(
|
||||
screenIndex: screenIndex,
|
||||
durationMs: durationMs,
|
||||
fps: fps,
|
||||
includeAudio: includeAudio,
|
||||
outPath: outPath)
|
||||
|
||||
let state = CaptureState()
|
||||
let recordQueue = DispatchQueue(label: "com.clawdis.screenrecord")
|
||||
|
||||
try await self.startCapture(state: state, config: config, recordQueue: recordQueue)
|
||||
try await Task.sleep(nanoseconds: UInt64(config.durationMs) * 1_000_000)
|
||||
try await self.stopCapture()
|
||||
try self.finalizeCapture(state: state)
|
||||
try await self.finishWriting(state: state)
|
||||
|
||||
return config.outURL.path
|
||||
}
|
||||
|
||||
private struct RecordConfig {
|
||||
let durationMs: Int
|
||||
let fpsValue: Double
|
||||
let includeAudio: Bool
|
||||
let outURL: URL
|
||||
}
|
||||
|
||||
private func makeRecordConfig(
|
||||
screenIndex: Int?,
|
||||
durationMs: Int?,
|
||||
fps: Double?,
|
||||
includeAudio: Bool?,
|
||||
outPath: String?) throws -> RecordConfig
|
||||
{
|
||||
if let idx = screenIndex, idx != 0 {
|
||||
throw ScreenRecordError.invalidScreenIndex(idx)
|
||||
}
|
||||
|
||||
let durationMs = Self.clampDurationMs(durationMs)
|
||||
let fps = Self.clampFps(fps)
|
||||
let fpsInt = Int32(fps.rounded())
|
||||
let fpsValue = Double(fpsInt)
|
||||
let includeAudio = includeAudio ?? true
|
||||
|
||||
if let idx = screenIndex, idx != 0 {
|
||||
throw ScreenRecordError.invalidScreenIndex(idx)
|
||||
let outURL = self.makeOutputURL(outPath: outPath)
|
||||
try? FileManager.default.removeItem(at: outURL)
|
||||
|
||||
return RecordConfig(
|
||||
durationMs: durationMs,
|
||||
fpsValue: fpsValue,
|
||||
includeAudio: includeAudio,
|
||||
outURL: outURL)
|
||||
}
|
||||
|
||||
let outURL: URL = {
|
||||
private func makeOutputURL(outPath: String?) -> URL {
|
||||
if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
||||
return URL(fileURLWithPath: outPath)
|
||||
}
|
||||
return FileManager.default.temporaryDirectory
|
||||
.appendingPathComponent("clawdbot-screen-record-\(UUID().uuidString).mp4")
|
||||
}()
|
||||
try? FileManager.default.removeItem(at: outURL)
|
||||
|
||||
let state = CaptureState()
|
||||
let recordQueue = DispatchQueue(label: "com.clawdbot.screenrecord")
|
||||
}
|
||||
|
||||
private func startCapture(
|
||||
state: CaptureState,
|
||||
config: RecordConfig,
|
||||
recordQueue: DispatchQueue) async throws
|
||||
{
|
||||
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
|
||||
let handler: @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void = { sample, type, error in
|
||||
let handler = self.makeCaptureHandler(
|
||||
state: state,
|
||||
config: config,
|
||||
recordQueue: recordQueue)
|
||||
let completion: @Sendable (Error?) -> Void = { error in
|
||||
if let error { cont.resume(throwing: error) } else { cont.resume() }
|
||||
}
|
||||
|
||||
Task { @MainActor in
|
||||
startReplayKitCapture(
|
||||
includeAudio: config.includeAudio,
|
||||
handler: handler,
|
||||
completion: completion)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func makeCaptureHandler(
|
||||
state: CaptureState,
|
||||
config: RecordConfig,
|
||||
recordQueue: DispatchQueue) -> @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void
|
||||
{
|
||||
{ sample, type, error in
|
||||
// ReplayKit can call the capture handler on a background queue.
|
||||
// Serialize writes to avoid queue asserts.
|
||||
recordQueue.async {
|
||||
@@ -85,68 +150,33 @@ final class ScreenRecordService: @unchecked Sendable {
|
||||
|
||||
switch type {
|
||||
case .video:
|
||||
self.handleVideoSample(sample, state: state, config: config)
|
||||
case .audioApp, .audioMic:
|
||||
self.handleAudioSample(sample, state: state, includeAudio: config.includeAudio)
|
||||
@unknown default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleVideoSample(
|
||||
_ sample: CMSampleBuffer,
|
||||
state: CaptureState,
|
||||
config: RecordConfig)
|
||||
{
|
||||
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
|
||||
let shouldSkip = state.withLock { state in
|
||||
if let lastVideoTime = state.lastVideoTime {
|
||||
let delta = CMTimeSubtract(pts, lastVideoTime)
|
||||
return delta.seconds < (1.0 / fpsValue)
|
||||
return delta.seconds < (1.0 / config.fpsValue)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if shouldSkip { return }
|
||||
|
||||
if state.withLock({ $0.writer == nil }) {
|
||||
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
|
||||
state.withLock { state in
|
||||
if state.handlerError == nil {
|
||||
state.handlerError = ScreenRecordError.captureFailed("Missing image buffer")
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
let width = CVPixelBufferGetWidth(imageBuffer)
|
||||
let height = CVPixelBufferGetHeight(imageBuffer)
|
||||
do {
|
||||
let w = try AVAssetWriter(outputURL: outURL, fileType: .mp4)
|
||||
let settings: [String: Any] = [
|
||||
AVVideoCodecKey: AVVideoCodecType.h264,
|
||||
AVVideoWidthKey: width,
|
||||
AVVideoHeightKey: height,
|
||||
]
|
||||
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
|
||||
vInput.expectsMediaDataInRealTime = true
|
||||
guard w.canAdd(vInput) else {
|
||||
throw ScreenRecordError.writeFailed("Cannot add video input")
|
||||
}
|
||||
w.add(vInput)
|
||||
|
||||
if includeAudio {
|
||||
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
|
||||
aInput.expectsMediaDataInRealTime = true
|
||||
if w.canAdd(aInput) {
|
||||
w.add(aInput)
|
||||
state.withLock { state in
|
||||
state.audioInput = aInput
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
guard w.startWriting() else {
|
||||
throw ScreenRecordError
|
||||
.writeFailed(w.error?.localizedDescription ?? "Failed to start writer")
|
||||
}
|
||||
w.startSession(atSourceTime: pts)
|
||||
state.withLock { state in
|
||||
state.writer = w
|
||||
state.videoInput = vInput
|
||||
state.started = true
|
||||
}
|
||||
} catch {
|
||||
state.withLock { state in
|
||||
if state.handlerError == nil { state.handlerError = error }
|
||||
}
|
||||
return
|
||||
}
|
||||
self.prepareWriter(sample: sample, state: state, config: config, pts: pts)
|
||||
}
|
||||
|
||||
let vInput = state.withLock { $0.videoInput }
|
||||
@@ -169,44 +199,92 @@ final class ScreenRecordService: @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case .audioApp, .audioMic:
|
||||
private func prepareWriter(
|
||||
sample: CMSampleBuffer,
|
||||
state: CaptureState,
|
||||
config: RecordConfig,
|
||||
pts: CMTime)
|
||||
{
|
||||
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
|
||||
state.withLock { state in
|
||||
if state.handlerError == nil {
|
||||
state.handlerError = ScreenRecordError.captureFailed("Missing image buffer")
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
let width = CVPixelBufferGetWidth(imageBuffer)
|
||||
let height = CVPixelBufferGetHeight(imageBuffer)
|
||||
do {
|
||||
let writer = try AVAssetWriter(outputURL: config.outURL, fileType: .mp4)
|
||||
let settings: [String: Any] = [
|
||||
AVVideoCodecKey: AVVideoCodecType.h264,
|
||||
AVVideoWidthKey: width,
|
||||
AVVideoHeightKey: height,
|
||||
]
|
||||
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
|
||||
vInput.expectsMediaDataInRealTime = true
|
||||
guard writer.canAdd(vInput) else {
|
||||
throw ScreenRecordError.writeFailed("Cannot add video input")
|
||||
}
|
||||
writer.add(vInput)
|
||||
|
||||
if config.includeAudio {
|
||||
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
|
||||
aInput.expectsMediaDataInRealTime = true
|
||||
if writer.canAdd(aInput) {
|
||||
writer.add(aInput)
|
||||
state.withLock { state in
|
||||
state.audioInput = aInput
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
guard writer.startWriting() else {
|
||||
throw ScreenRecordError.writeFailed(
|
||||
writer.error?.localizedDescription ?? "Failed to start writer")
|
||||
}
|
||||
writer.startSession(atSourceTime: pts)
|
||||
state.withLock { state in
|
||||
state.writer = writer
|
||||
state.videoInput = vInput
|
||||
state.started = true
|
||||
}
|
||||
} catch {
|
||||
state.withLock { state in
|
||||
if state.handlerError == nil { state.handlerError = error }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleAudioSample(
|
||||
_ sample: CMSampleBuffer,
|
||||
state: CaptureState,
|
||||
includeAudio: Bool)
|
||||
{
|
||||
let aInput = state.withLock { $0.audioInput }
|
||||
let isStarted = state.withLock { $0.started }
|
||||
guard includeAudio, let aInput, isStarted else { return }
|
||||
if aInput.isReadyForMoreMediaData {
|
||||
_ = aInput.append(sample)
|
||||
}
|
||||
|
||||
@unknown default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let completion: @Sendable (Error?) -> Void = { error in
|
||||
if let error { cont.resume(throwing: error) } else { cont.resume() }
|
||||
}
|
||||
|
||||
Task { @MainActor in
|
||||
startReplayKitCapture(
|
||||
includeAudio: includeAudio,
|
||||
handler: handler,
|
||||
completion: completion)
|
||||
}
|
||||
}
|
||||
|
||||
try await Task.sleep(nanoseconds: UInt64(durationMs) * 1_000_000)
|
||||
|
||||
private func stopCapture() async throws {
|
||||
let stopError = await withCheckedContinuation { cont in
|
||||
Task { @MainActor in
|
||||
stopReplayKitCapture { error in cont.resume(returning: error) }
|
||||
}
|
||||
}
|
||||
if let stopError { throw stopError }
|
||||
}
|
||||
|
||||
let handlerErrorSnapshot = state.withLock { $0.handlerError }
|
||||
if let handlerErrorSnapshot { throw handlerErrorSnapshot }
|
||||
private func finalizeCapture(state: CaptureState) throws {
|
||||
if let handlerErrorSnapshot = state.withLock({ $0.handlerError }) {
|
||||
throw handlerErrorSnapshot
|
||||
}
|
||||
let writerSnapshot = state.withLock { $0.writer }
|
||||
let videoInputSnapshot = state.withLock { $0.videoInput }
|
||||
let audioInputSnapshot = state.withLock { $0.audioInput }
|
||||
@@ -217,7 +295,13 @@ final class ScreenRecordService: @unchecked Sendable {
|
||||
|
||||
videoInputSnapshot.markAsFinished()
|
||||
audioInputSnapshot?.markAsFinished()
|
||||
_ = writerSnapshot
|
||||
}
|
||||
|
||||
private func finishWriting(state: CaptureState) async throws {
|
||||
guard let writerSnapshot = state.withLock({ $0.writer }) else {
|
||||
throw ScreenRecordError.captureFailed("Missing writer")
|
||||
}
|
||||
let writerBox = UncheckedSendableBox(value: writerSnapshot)
|
||||
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
|
||||
writerBox.value.finishWriting {
|
||||
@@ -231,8 +315,6 @@ final class ScreenRecordService: @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return outURL.path
|
||||
}
|
||||
|
||||
private nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
|
||||
|
||||
@@ -288,9 +288,9 @@ final class TalkModeManager: NSObject {
|
||||
self.chatSubscribedSessionKeys.insert(key)
|
||||
self.logger.info("chat.subscribe ok sessionKey=\(key, privacy: .public)")
|
||||
} catch {
|
||||
self.logger
|
||||
.warning(
|
||||
"chat.subscribe failed sessionKey=\(key, privacy: .public) err=\(error.localizedDescription, privacy: .public)")
|
||||
self.logger.warning(
|
||||
"chat.subscribe failed sessionKey=\(key, privacy: .public) " +
|
||||
"err=\(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -340,7 +340,12 @@ final class TalkModeManager: NSObject {
|
||||
"idempotencyKey": UUID().uuidString,
|
||||
]
|
||||
let data = try JSONSerialization.data(withJSONObject: payload)
|
||||
let json = String(decoding: data, as: UTF8.self)
|
||||
guard let json = String(bytes: data, encoding: .utf8) else {
|
||||
throw NSError(
|
||||
domain: "TalkModeManager",
|
||||
code: 1,
|
||||
userInfo: [NSLocalizedDescriptionKey: "Failed to encode chat payload"])
|
||||
}
|
||||
let res = try await bridge.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 30)
|
||||
let decoded = try JSONDecoder().decode(SendResponse.self, from: res)
|
||||
return decoded.runId
|
||||
@@ -523,9 +528,9 @@ final class TalkModeManager: NSObject {
|
||||
self.lastPlaybackWasPCM = false
|
||||
result = await self.mp3Player.play(stream: stream)
|
||||
}
|
||||
self.logger
|
||||
.info(
|
||||
"elevenlabs stream finished=\(result.finished, privacy: .public) dur=\(Date().timeIntervalSince(started), privacy: .public)s")
|
||||
self.logger.info(
|
||||
"elevenlabs stream finished=\(result.finished, privacy: .public) " +
|
||||
"dur=\(Date().timeIntervalSince(started), privacy: .public)s")
|
||||
if !result.finished, let interruptedAt = result.interruptedAt {
|
||||
self.lastInterruptedAtSeconds = interruptedAt
|
||||
}
|
||||
|
||||
@@ -192,6 +192,4 @@ actor MacNodeBridgePairingClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -325,6 +325,4 @@ actor MacNodeBridgeSession {
|
||||
])
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ final class MacNodeLocationService: NSObject, CLLocationManagerDelegate {
|
||||
}
|
||||
|
||||
self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy)
|
||||
let timeout = max(0, timeoutMs ?? 10_000)
|
||||
let timeout = max(0, timeoutMs ?? 10000)
|
||||
return try await self.withTimeout(timeoutMs: timeout) {
|
||||
try await self.requestLocation()
|
||||
}
|
||||
@@ -83,11 +83,11 @@ final class MacNodeLocationService: NSObject, CLLocationManagerDelegate {
|
||||
private static func accuracyValue(_ accuracy: ClawdbotLocationAccuracy) -> CLLocationAccuracy {
|
||||
switch accuracy {
|
||||
case .coarse:
|
||||
return kCLLocationAccuracyKilometer
|
||||
kCLLocationAccuracyKilometer
|
||||
case .balanced:
|
||||
return kCLLocationAccuracyHundredMeters
|
||||
kCLLocationAccuracyHundredMeters
|
||||
case .precise:
|
||||
return kCLLocationAccuracyBest
|
||||
kCLLocationAccuracyBest
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,10 +8,9 @@ actor MacNodeRuntime {
|
||||
@MainActor private let screenRecorder = ScreenRecordService()
|
||||
@MainActor private let locationService = MacNodeLocationService()
|
||||
|
||||
// swiftlint:disable:next function_body_length cyclomatic_complexity
|
||||
func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
|
||||
let command = req.command
|
||||
if command.hasPrefix("canvas.") || command.hasPrefix("canvas.a2ui."), !Self.canvasEnabled() {
|
||||
if self.isCanvasCommand(command), !Self.canvasEnabled() {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
@@ -21,6 +20,42 @@ actor MacNodeRuntime {
|
||||
}
|
||||
do {
|
||||
switch command {
|
||||
case ClawdbotCanvasCommand.present.rawValue,
|
||||
ClawdbotCanvasCommand.hide.rawValue,
|
||||
ClawdbotCanvasCommand.navigate.rawValue,
|
||||
ClawdbotCanvasCommand.evalJS.rawValue,
|
||||
ClawdbotCanvasCommand.snapshot.rawValue:
|
||||
return try await self.handleCanvasInvoke(req)
|
||||
case ClawdbotCanvasA2UICommand.reset.rawValue,
|
||||
ClawdbotCanvasA2UICommand.push.rawValue,
|
||||
ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
|
||||
return try await self.handleA2UIInvoke(req)
|
||||
case ClawdbotCameraCommand.snap.rawValue,
|
||||
ClawdbotCameraCommand.clip.rawValue,
|
||||
ClawdbotCameraCommand.list.rawValue:
|
||||
return try await self.handleCameraInvoke(req)
|
||||
case ClawdbotLocationCommand.get.rawValue:
|
||||
return try await self.handleLocationInvoke(req)
|
||||
case MacNodeScreenCommand.record.rawValue:
|
||||
return try await self.handleScreenRecordInvoke(req)
|
||||
case ClawdbotSystemCommand.run.rawValue:
|
||||
return try await self.handleSystemRun(req)
|
||||
case ClawdbotSystemCommand.notify.rawValue:
|
||||
return try await self.handleSystemNotify(req)
|
||||
default:
|
||||
return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
|
||||
}
|
||||
} catch {
|
||||
return Self.errorResponse(req, code: .unavailable, message: error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func isCanvasCommand(_ command: String) -> Bool {
|
||||
command.hasPrefix("canvas.") || command.hasPrefix("canvas.a2ui.")
|
||||
}
|
||||
|
||||
private func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
switch req.command {
|
||||
case ClawdbotCanvasCommand.present.rawValue:
|
||||
let params = (try? Self.decodeParams(ClawdbotCanvasPresentParams.self, from: req.paramsJSON)) ??
|
||||
ClawdbotCanvasPresentParams()
|
||||
@@ -36,20 +71,17 @@ actor MacNodeRuntime {
|
||||
placement: placement)
|
||||
}
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
|
||||
case ClawdbotCanvasCommand.hide.rawValue:
|
||||
await MainActor.run {
|
||||
CanvasManager.shared.hide(sessionKey: "main")
|
||||
}
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
|
||||
case ClawdbotCanvasCommand.navigate.rawValue:
|
||||
let params = try Self.decodeParams(ClawdbotCanvasNavigateParams.self, from: req.paramsJSON)
|
||||
try await MainActor.run {
|
||||
_ = try CanvasManager.shared.show(sessionKey: "main", path: params.url)
|
||||
}
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
|
||||
case ClawdbotCanvasCommand.evalJS.rawValue:
|
||||
let params = try Self.decodeParams(ClawdbotCanvasEvalParams.self, from: req.paramsJSON)
|
||||
let result = try await CanvasManager.shared.eval(
|
||||
@@ -57,7 +89,6 @@ actor MacNodeRuntime {
|
||||
javaScript: params.javaScript)
|
||||
let payload = try Self.encodePayload(["result": result] as [String: String])
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
|
||||
case ClawdbotCanvasCommand.snapshot.rawValue:
|
||||
let params = try? Self.decodeParams(ClawdbotCanvasSnapshotParams.self, from: req.paramsJSON)
|
||||
let format = params?.format ?? .jpeg
|
||||
@@ -86,14 +117,24 @@ actor MacNodeRuntime {
|
||||
"base64": encoded.base64EncodedString(),
|
||||
])
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
default:
|
||||
return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
|
||||
}
|
||||
}
|
||||
|
||||
private func handleA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
switch req.command {
|
||||
case ClawdbotCanvasA2UICommand.reset.rawValue:
|
||||
return try await self.handleA2UIReset(req)
|
||||
try await self.handleA2UIReset(req)
|
||||
case ClawdbotCanvasA2UICommand.push.rawValue,
|
||||
ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
|
||||
try await self.handleA2UIPush(req)
|
||||
default:
|
||||
Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
|
||||
}
|
||||
}
|
||||
|
||||
case ClawdbotCanvasA2UICommand.push.rawValue, ClawdbotCanvasA2UICommand.pushJSONL.rawValue:
|
||||
return try await self.handleA2UIPush(req)
|
||||
|
||||
case ClawdbotCameraCommand.snap.rawValue:
|
||||
private func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
guard Self.cameraEnabled() else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
@@ -102,9 +143,11 @@ actor MacNodeRuntime {
|
||||
code: .unavailable,
|
||||
message: "CAMERA_DISABLED: enable Camera in Settings"))
|
||||
}
|
||||
switch req.command {
|
||||
case ClawdbotCameraCommand.snap.rawValue:
|
||||
let params = (try? Self.decodeParams(ClawdbotCameraSnapParams.self, from: req.paramsJSON)) ??
|
||||
ClawdbotCameraSnapParams()
|
||||
let delayMs = min(10_000, max(0, params.delayMs ?? 2000))
|
||||
let delayMs = min(10000, max(0, params.delayMs ?? 2000))
|
||||
let res = try await self.cameraCapture.snap(
|
||||
facing: CameraFacing(rawValue: params.facing?.rawValue ?? "") ?? .front,
|
||||
maxWidth: params.maxWidth,
|
||||
@@ -123,16 +166,7 @@ actor MacNodeRuntime {
|
||||
width: Int(res.size.width),
|
||||
height: Int(res.size.height)))
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
|
||||
case ClawdbotCameraCommand.clip.rawValue:
|
||||
guard Self.cameraEnabled() else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "CAMERA_DISABLED: enable Camera in Settings"))
|
||||
}
|
||||
let params = (try? Self.decodeParams(ClawdbotCameraClipParams.self, from: req.paramsJSON)) ??
|
||||
ClawdbotCameraClipParams()
|
||||
let res = try await self.cameraCapture.clip(
|
||||
@@ -155,21 +189,16 @@ actor MacNodeRuntime {
|
||||
durationMs: res.durationMs,
|
||||
hasAudio: res.hasAudio))
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
|
||||
case ClawdbotCameraCommand.list.rawValue:
|
||||
guard Self.cameraEnabled() else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: ClawdbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "CAMERA_DISABLED: enable Camera in Settings"))
|
||||
}
|
||||
let devices = await self.cameraCapture.listDevices()
|
||||
let payload = try Self.encodePayload(["devices": devices])
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
default:
|
||||
return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
|
||||
}
|
||||
}
|
||||
|
||||
case ClawdbotLocationCommand.get.rawValue:
|
||||
private func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let mode = Self.locationMode()
|
||||
guard mode != .off else {
|
||||
return BridgeInvokeResponse(
|
||||
@@ -184,7 +213,7 @@ actor MacNodeRuntime {
|
||||
let desired = params.desiredAccuracy ??
|
||||
(Self.locationPreciseEnabled() ? .precise : .balanced)
|
||||
let status = await self.locationService.authorizationStatus()
|
||||
if status != .authorizedAlways && status != .authorizedWhenInUse {
|
||||
if status != .authorizedAlways {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
@@ -225,8 +254,9 @@ actor MacNodeRuntime {
|
||||
code: .unavailable,
|
||||
message: "LOCATION_UNAVAILABLE: \(error.localizedDescription)"))
|
||||
}
|
||||
}
|
||||
|
||||
case MacNodeScreenCommand.record.rawValue:
|
||||
private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let params = (try? Self.decodeParams(MacNodeScreenRecordParams.self, from: req.paramsJSON)) ??
|
||||
MacNodeScreenRecordParams()
|
||||
if let format = params.format?.lowercased(), !format.isEmpty, format != "mp4" {
|
||||
@@ -259,19 +289,6 @@ actor MacNodeRuntime {
|
||||
screenIndex: params.screenIndex,
|
||||
hasAudio: res.hasAudio))
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
|
||||
case ClawdbotSystemCommand.run.rawValue:
|
||||
return try await self.handleSystemRun(req)
|
||||
|
||||
case ClawdbotSystemCommand.notify.rawValue:
|
||||
return try await self.handleSystemNotify(req)
|
||||
|
||||
default:
|
||||
return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command")
|
||||
}
|
||||
} catch {
|
||||
return Self.errorResponse(req, code: .unavailable, message: error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func handleA2UIReset(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
|
||||
Reference in New Issue
Block a user