let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Nodes: advertise canvas invoke commands

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-18 02:05:06 +00:00
parent 54830e8401
commit efed2ae30f
15 changed files with 212 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
apps/android/app/src/main/java/com/steipete/clawdis/node/NodeRuntime.kt
View File

@@ -266,6 +266,20 @@ class NodeRuntime(context: Context) {
.joinToString(" ")
.trim()
.ifEmpty { null }
val invokeCommands =
buildList {
add("canvas.show")
add("canvas.hide")
add("canvas.setMode")
add("canvas.navigate")
add("canvas.eval")
add("canvas.snapshot")
if (cameraEnabled.value) {
add("camera.snap")
add("camera.clip")
}
}
val resolved =
if (storedToken.isNullOrBlank()) {
_statusText.value = "Pairing…"
@@ -288,6 +302,7 @@ class NodeRuntime(context: Context) {
deviceFamily = "Android",
modelIdentifier = modelIdentifier,
caps = caps,
commands = invokeCommands,
),
)
} else {
@@ -311,19 +326,20 @@ class NodeRuntime(context: Context) {
platform = "Android",
version = "dev",
deviceFamily = "Android",
modelIdentifier = modelIdentifier,
caps =
buildList {
add(ClawdisCapability.Canvas.rawValue)
if (cameraEnabled.value) add(ClawdisCapability.Camera.rawValue)
if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(ClawdisCapability.VoiceWake.rawValue)
}
},
),
)
}
}
modelIdentifier = modelIdentifier,
caps =
buildList {
add(ClawdisCapability.Canvas.rawValue)
if (cameraEnabled.value) add(ClawdisCapability.Camera.rawValue)
if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(ClawdisCapability.VoiceWake.rawValue)
}
},
commands = invokeCommands,
),
)
}
}
private fun hasRecordAudioPermission(): Boolean {
return (

3
apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgePairingClient.kt
View File

@@ -28,6 +28,7 @@ class BridgePairingClient {
val deviceFamily: String?,
val modelIdentifier: String?,
val caps: List<String>?,
val commands: List<String>?,
)
data class PairResult(val ok: Boolean, val token: String?, val error: String? = null)
@@ -62,6 +63,7 @@ class BridgePairingClient {
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)
@@ -86,6 +88,7 @@ class BridgePairingClient {
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)

2
apps/android/app/src/main/java/com/steipete/clawdis/node/bridge/BridgeSession.kt
View File

@@ -43,6 +43,7 @@ class BridgeSession(
val deviceFamily: String?,
val modelIdentifier: String?,
val caps: List<String>?,
val commands: List<String>?,
)
data class InvokeRequest(val id: String, val command: String, val paramsJson: String?)
@@ -198,6 +199,7 @@ class BridgeSession(
hello.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
hello.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
hello.caps?.let { put("caps", JsonArray(it.map(::JsonPrimitive))) }
hello.commands?.let { put("commands", JsonArray(it.map(::JsonPrimitive))) }
},
)

2
apps/android/app/src/test/java/com/steipete/clawdis/node/bridge/BridgePairingClientTest.kt
View File

@@ -49,6 +49,7 @@ class BridgePairingClientTest {
deviceFamily = "Android",
modelIdentifier = "SM-X000",
caps = null,
commands = null,
),
)
assertTrue(res.ok)
@@ -97,6 +98,7 @@ class BridgePairingClientTest {
deviceFamily = "Android",
modelIdentifier = "SM-X000",
caps = null,
commands = null,
),
)
assertTrue(res.ok)

4
apps/android/app/src/test/java/com/steipete/clawdis/node/bridge/BridgeSessionTest.kt
View File

@@ -72,6 +72,7 @@ class BridgeSessionTest {
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
@@ -137,6 +138,7 @@ class BridgeSessionTest {
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
connected.await()
@@ -207,6 +209,7 @@ class BridgeSessionTest {
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)
connected.await()
@@ -279,6 +282,7 @@ class BridgeSessionTest {
deviceFamily = null,
modelIdentifier = null,
caps = null,
commands = null,
),
)

21
apps/ios/Sources/Bridge/BridgeClient.swift
View File

@@ -46,16 +46,17 @@ actor BridgeClient {
}
onStatus?("Requesting approval…")
try await self.send(
BridgePairRequest(
nodeId: hello.nodeId,
displayName: hello.displayName,
platform: hello.platform,
version: hello.version,
deviceFamily: hello.deviceFamily,
modelIdentifier: hello.modelIdentifier,
caps: hello.caps),
over: connection)
try await self.send(
BridgePairRequest(
nodeId: hello.nodeId,
displayName: hello.displayName,
platform: hello.platform,
version: hello.version,
deviceFamily: hello.deviceFamily,
modelIdentifier: hello.modelIdentifier,
caps: hello.caps,
commands: hello.commands),
over: connection)
onStatus?("Waiting for approval…")
let ok = try await self.withTimeout(seconds: 60, purpose: "pairing approval") {

22
apps/ios/Sources/Bridge/BridgeConnectionController.swift
View File

@@ -136,7 +136,8 @@ final class BridgeConnectionController {
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps())
caps: self.currentCaps(),
commands: self.currentCommands())
}
private func resolvedDisplayName(defaults: UserDefaults) -> String {
@@ -170,6 +171,25 @@ final class BridgeConnectionController {
return caps
}
private func currentCommands() -> [String] {
var commands: [String] = [
ClawdisCanvasCommand.show.rawValue,
ClawdisCanvasCommand.hide.rawValue,
ClawdisCanvasCommand.setMode.rawValue,
ClawdisCanvasCommand.navigate.rawValue,
ClawdisCanvasCommand.evalJS.rawValue,
ClawdisCanvasCommand.snapshot.rawValue,
]
let caps = Set(self.currentCaps())
if caps.contains(ClawdisCapability.camera.rawValue) {
commands.append(ClawdisCameraCommand.snap.rawValue)
commands.append(ClawdisCameraCommand.clip.rawValue)
}
return commands
}
private func platformString() -> String {
let v = ProcessInfo.processInfo.operatingSystemVersion
let name = switch UIDevice.current.userInterfaceIdiom {

20
apps/ios/Sources/Model/NodeAppModel.swift
View File

@@ -287,30 +287,30 @@ final class NodeAppModel {
do {
switch command {
case ClawdisScreenCommand.show.rawValue:
case ClawdisCanvasCommand.show.rawValue:
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisScreenCommand.hide.rawValue:
case ClawdisCanvasCommand.hide.rawValue:
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisScreenCommand.setMode.rawValue:
let params = try Self.decodeParams(ClawdisScreenSetModeParams.self, from: req.paramsJSON)
case ClawdisCanvasCommand.setMode.rawValue:
let params = try Self.decodeParams(ClawdisCanvasSetModeParams.self, from: req.paramsJSON)
self.screen.setMode(params.mode)
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisScreenCommand.navigate.rawValue:
let params = try Self.decodeParams(ClawdisScreenNavigateParams.self, from: req.paramsJSON)
case ClawdisCanvasCommand.navigate.rawValue:
let params = try Self.decodeParams(ClawdisCanvasNavigateParams.self, from: req.paramsJSON)
self.screen.navigate(to: params.url)
return BridgeInvokeResponse(id: req.id, ok: true)
case ClawdisScreenCommand.evalJS.rawValue:
let params = try Self.decodeParams(ClawdisScreenEvalParams.self, from: req.paramsJSON)
case ClawdisCanvasCommand.evalJS.rawValue:
let params = try Self.decodeParams(ClawdisCanvasEvalParams.self, from: req.paramsJSON)
let result = try await self.screen.eval(javaScript: params.javaScript)
let payload = try Self.encodePayload(["result": result])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisScreenCommand.snapshot.rawValue:
let params = try? Self.decodeParams(ClawdisScreenSnapshotParams.self, from: req.paramsJSON)
case ClawdisCanvasCommand.snapshot.rawValue:
let params = try? Self.decodeParams(ClawdisCanvasSnapshotParams.self, from: req.paramsJSON)
let maxWidth = params?.maxWidth.map { CGFloat($0) }
let base64 = try await self.screen.snapshotPNGBase64(maxWidth: maxWidth)
let payload = try Self.encodePayload(["format": "png", "base64": base64])

4
apps/ios/Sources/Screen/ScreenController.swift
View File

@@ -9,7 +9,7 @@ final class ScreenController {
let webView: WKWebView
private let navigationDelegate: ScreenNavigationDelegate
var mode: ClawdisScreenMode = .canvas
var mode: ClawdisCanvasMode = .canvas
var urlString: String = ""
var errorText: String?
@@ -36,7 +36,7 @@ final class ScreenController {
self.reload()
}
func setMode(_ mode: ClawdisScreenMode) {
func setMode(_ mode: ClawdisCanvasMode) {
self.mode = mode
self.reload()
}

78
apps/ios/Sources/Settings/SettingsTab.swift
View File

@@ -262,6 +262,60 @@ struct SettingsTab: View {
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
}
private func deviceFamily() -> String {
switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPad"
case .phone:
"iPhone"
default:
"iOS"
}
}
private func modelIdentifier() -> String {
var systemInfo = utsname()
uname(&systemInfo)
let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in
String(decoding: ptr.prefix { $0 != 0 }, as: UTF8.self)
}
return machine.isEmpty ? "unknown" : machine
}
private func currentCaps() -> [String] {
var caps = [ClawdisCapability.canvas.rawValue]
let cameraEnabled =
UserDefaults.standard.object(forKey: "camera.enabled") == nil
? true
: UserDefaults.standard.bool(forKey: "camera.enabled")
if cameraEnabled { caps.append(ClawdisCapability.camera.rawValue) }
let voiceWakeEnabled = UserDefaults.standard.bool(forKey: VoiceWakePreferences.enabledKey)
if voiceWakeEnabled { caps.append(ClawdisCapability.voiceWake.rawValue) }
return caps
}
private func currentCommands() -> [String] {
var commands: [String] = [
ClawdisCanvasCommand.show.rawValue,
ClawdisCanvasCommand.hide.rawValue,
ClawdisCanvasCommand.setMode.rawValue,
ClawdisCanvasCommand.navigate.rawValue,
ClawdisCanvasCommand.evalJS.rawValue,
ClawdisCanvasCommand.snapshot.rawValue,
]
let caps = Set(self.currentCaps())
if caps.contains(ClawdisCapability.camera.rawValue) {
commands.append(ClawdisCameraCommand.snap.rawValue)
commands.append(ClawdisCameraCommand.clip.rawValue)
}
return commands
}
private func connect(_ bridge: BridgeDiscoveryModel.DiscoveredBridge) async {
self.connectingBridgeID = bridge.id
self.manualBridgeEnabled = false
@@ -285,7 +339,11 @@ struct SettingsTab: View {
displayName: self.displayName,
token: existingToken,
platform: self.platformString(),
version: self.appVersion())
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands())
let token = try await BridgeClient().pairAndHello(
endpoint: bridge.endpoint,
hello: hello,
@@ -309,7 +367,11 @@ struct SettingsTab: View {
displayName: self.displayName,
token: token,
platform: self.platformString(),
version: self.appVersion()))
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands()))
} catch {
self.connectStatus.text = "Failed: \(error.localizedDescription)"
@@ -351,7 +413,11 @@ struct SettingsTab: View {
displayName: self.displayName,
token: existingToken,
platform: self.platformString(),
version: self.appVersion())
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands())
let token = try await BridgeClient().pairAndHello(
endpoint: endpoint,
hello: hello,
@@ -375,7 +441,11 @@ struct SettingsTab: View {
displayName: self.displayName,
token: token,
platform: self.platformString(),
version: self.appVersion()))
version: self.appVersion(),
deviceFamily: self.deviceFamily(),
modelIdentifier: self.modelIdentifier(),
caps: self.currentCaps(),
commands: self.currentCommands()))
} catch {
self.connectStatus.text = "Failed: \(error.localizedDescription)"

38
apps/ios/Tests/CanvasCommandAliasTests.swift
View File

@@ -1,38 +0,0 @@
import ClawdisKit
import Testing
@Suite struct CanvasCommandAliasTests {
@Test func mapsKnownCanvasCommandsToScreen() {
let mappings: [(ClawdisCanvasCommand, ClawdisScreenCommand)] = [
(.show, .show),
(.hide, .hide),
(.setMode, .setMode),
(.navigate, .navigate),
(.evalJS, .evalJS),
(.snapshot, .snapshot),
]
for (canvas, screen) in mappings {
#expect(
ClawdisInvokeCommandAliases.canonicalizeCanvasToScreen(canvas.rawValue) ==
screen.rawValue)
}
}
@Test func mapsUnknownCanvasNamespaceToScreen() {
#expect(ClawdisInvokeCommandAliases.canonicalizeCanvasToScreen("canvas.foo") == "screen.foo")
}
@Test func leavesNonCanvasCommandsUnchanged() {
#expect(
ClawdisInvokeCommandAliases.canonicalizeCanvasToScreen(ClawdisCameraCommand.snap.rawValue) ==
ClawdisCameraCommand.snap.rawValue)
}
@Test func capabilitiesUseStableStrings() {
#expect(ClawdisCapability.canvas.rawValue == "canvas")
#expect(ClawdisCapability.camera.rawValue == "camera")
#expect(ClawdisCapability.voiceWake.rawValue == "voiceWake")
}
}

8
apps/shared/ClawdisKit/Sources/ClawdisKit/BridgeFrames.swift
View File

@@ -66,6 +66,7 @@ public struct BridgeHello: Codable, Sendable {
public let deviceFamily: String?
public let modelIdentifier: String?
public let caps: [String]?
public let commands: [String]?
public init(
type: String = "hello",
@@ -76,7 +77,8 @@ public struct BridgeHello: Codable, Sendable {
version: String?,
deviceFamily: String? = nil,
modelIdentifier: String? = nil,
caps: [String]? = nil)
caps: [String]? = nil,
commands: [String]? = nil)
{
self.type = type
self.nodeId = nodeId
@@ -87,6 +89,7 @@ public struct BridgeHello: Codable, Sendable {
self.deviceFamily = deviceFamily
self.modelIdentifier = modelIdentifier
self.caps = caps
self.commands = commands
}
}
@@ -109,6 +112,7 @@ public struct BridgePairRequest: Codable, Sendable {
public let deviceFamily: String?
public let modelIdentifier: String?
public let caps: [String]?
public let commands: [String]?
public let remoteAddress: String?
public init(
@@ -120,6 +124,7 @@ public struct BridgePairRequest: Codable, Sendable {
deviceFamily: String? = nil,
modelIdentifier: String? = nil,
caps: [String]? = nil,
commands: [String]? = nil,
remoteAddress: String? = nil)
{
self.type = type
@@ -130,6 +135,7 @@ public struct BridgePairRequest: Codable, Sendable {
self.deviceFamily = deviceFamily
self.modelIdentifier = modelIdentifier
self.caps = caps
self.commands = commands
self.remoteAddress = remoteAddress
}
}

47
apps/shared/ClawdisKit/Sources/ClawdisKit/CanvasCommandParams.swift Normal file
View File

@@ -0,0 +1,47 @@
import Foundation
public enum ClawdisCanvasMode: String, Codable, Sendable {
case canvas
case web
}
public struct ClawdisCanvasNavigateParams: Codable, Sendable, Equatable {
public var url: String
public init(url: String) {
self.url = url
}
}
public struct ClawdisCanvasSetModeParams: Codable, Sendable, Equatable {
public var mode: ClawdisCanvasMode
public init(mode: ClawdisCanvasMode) {
self.mode = mode
}
}
public struct ClawdisCanvasEvalParams: Codable, Sendable, Equatable {
public var javaScript: String
public init(javaScript: String) {
self.javaScript = javaScript
}
}
public enum ClawdisCanvasSnapshotFormat: String, Codable, Sendable {
case png
case jpeg
}
public struct ClawdisCanvasSnapshotParams: Codable, Sendable, Equatable {
public var maxWidth: Int?
public var quality: Double?
public var format: ClawdisCanvasSnapshotFormat?
public init(maxWidth: Int? = nil, quality: Double? = nil, format: ClawdisCanvasSnapshotFormat? = nil) {
self.maxWidth = maxWidth
self.quality = quality
self.format = format
}
}

18
apps/shared/ClawdisKit/Sources/ClawdisKit/CanvasCommands.swift
View File

@@ -8,21 +8,3 @@ public enum ClawdisCanvasCommand: String, Codable, Sendable {
case evalJS = "canvas.eval"
case snapshot = "canvas.snapshot"
}
public enum ClawdisInvokeCommandAliases {
public static func canonicalizeCanvasToScreen(_ command: String) -> String {
if command.hasPrefix(ClawdisCanvasCommand.namespacePrefix) {
return ClawdisScreenCommand.namespacePrefix +
command.dropFirst(ClawdisCanvasCommand.namespacePrefix.count)
}
return command
}
}
extension ClawdisCanvasCommand {
public static var namespacePrefix: String { "canvas." }
}
extension ClawdisScreenCommand {
public static var namespacePrefix: String { "screen." }
}

56
apps/shared/ClawdisKit/Sources/ClawdisKit/ScreenCommands.swift
View File

@@ -1,56 +0,0 @@
import Foundation
public enum ClawdisScreenMode: String, Codable, Sendable {
case canvas
case web
}
public enum ClawdisScreenCommand: String, Codable, Sendable {
case show = "canvas.show"
case hide = "canvas.hide"
case setMode = "canvas.setMode"
case navigate = "canvas.navigate"
case evalJS = "canvas.eval"
case snapshot = "canvas.snapshot"
}
public struct ClawdisScreenNavigateParams: Codable, Sendable, Equatable {
public var url: String
public init(url: String) {
self.url = url
}
}
public struct ClawdisScreenSetModeParams: Codable, Sendable, Equatable {
public var mode: ClawdisScreenMode
public init(mode: ClawdisScreenMode) {
self.mode = mode
}
}
public struct ClawdisScreenEvalParams: Codable, Sendable, Equatable {
public var javaScript: String
public init(javaScript: String) {
self.javaScript = javaScript
}
}
public enum ClawdisSnapshotFormat: String, Codable, Sendable {
case png
case jpeg
}
public struct ClawdisScreenSnapshotParams: Codable, Sendable, Equatable {
public var maxWidth: Int?
public var quality: Double?
public var format: ClawdisSnapshotFormat?
public init(maxWidth: Int? = nil, quality: Double? = nil, format: ClawdisSnapshotFormat? = nil) {
self.maxWidth = maxWidth
self.quality = quality
self.format = format
}
}