chore: rename project to clawdbot

This commit is contained in:
Peter Steinberger
2026-01-04 14:32:47 +00:00
parent d48dc71fa4
commit 246adaa119
841 changed files with 4590 additions and 4328 deletions

View File

@@ -0,0 +1,93 @@
import Foundation
/// Lightweight `Codable` wrapper that round-trips heterogeneous JSON payloads.
///
/// Marked `@unchecked Sendable` because it can hold reference types.
public struct AnyCodable: Codable, @unchecked Sendable, Hashable {
public let value: Any
public init(_ value: Any) { self.value = value }
public init(from decoder: Decoder) throws {
let container = try decoder.singleValueContainer()
if let intVal = try? container.decode(Int.self) { self.value = intVal; return }
if let doubleVal = try? container.decode(Double.self) { self.value = doubleVal; return }
if let boolVal = try? container.decode(Bool.self) { self.value = boolVal; return }
if let stringVal = try? container.decode(String.self) { self.value = stringVal; return }
if container.decodeNil() { self.value = NSNull(); return }
if let dict = try? container.decode([String: AnyCodable].self) { self.value = dict; return }
if let array = try? container.decode([AnyCodable].self) { self.value = array; return }
throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported type")
}
public func encode(to encoder: Encoder) throws {
var container = encoder.singleValueContainer()
switch self.value {
case let intVal as Int: try container.encode(intVal)
case let doubleVal as Double: try container.encode(doubleVal)
case let boolVal as Bool: try container.encode(boolVal)
case let stringVal as String: try container.encode(stringVal)
case is NSNull: try container.encodeNil()
case let dict as [String: AnyCodable]: try container.encode(dict)
case let array as [AnyCodable]: try container.encode(array)
case let dict as [String: Any]:
try container.encode(dict.mapValues { AnyCodable($0) })
case let array as [Any]:
try container.encode(array.map { AnyCodable($0) })
case let dict as NSDictionary:
var converted: [String: AnyCodable] = [:]
for (k, v) in dict {
guard let key = k as? String else { continue }
converted[key] = AnyCodable(v)
}
try container.encode(converted)
case let array as NSArray:
try container.encode(array.map { AnyCodable($0) })
default:
let context = EncodingError.Context(codingPath: encoder.codingPath, debugDescription: "Unsupported type")
throw EncodingError.invalidValue(self.value, context)
}
}
public static func == (lhs: AnyCodable, rhs: AnyCodable) -> Bool {
switch (lhs.value, rhs.value) {
case let (l as Int, r as Int): l == r
case let (l as Double, r as Double): l == r
case let (l as Bool, r as Bool): l == r
case let (l as String, r as String): l == r
case (_ as NSNull, _ as NSNull): true
case let (l as [String: AnyCodable], r as [String: AnyCodable]): l == r
case let (l as [AnyCodable], r as [AnyCodable]): l == r
default:
false
}
}
public func hash(into hasher: inout Hasher) {
switch self.value {
case let v as Int:
hasher.combine(0); hasher.combine(v)
case let v as Double:
hasher.combine(1); hasher.combine(v)
case let v as Bool:
hasher.combine(2); hasher.combine(v)
case let v as String:
hasher.combine(3); hasher.combine(v)
case _ as NSNull:
hasher.combine(4)
case let v as [String: AnyCodable]:
hasher.combine(5)
for (k, val) in v.sorted(by: { $0.key < $1.key }) {
hasher.combine(k)
hasher.combine(val)
}
case let v as [AnyCodable]:
hasher.combine(6)
for item in v {
hasher.combine(item)
}
default:
hasher.combine(999)
}
}
}

View File

@@ -0,0 +1,16 @@
import Foundation
@MainActor
public protocol StreamingAudioPlaying {
func play(stream: AsyncThrowingStream<Data, Error>) async -> StreamingPlaybackResult
func stop() -> Double?
}
@MainActor
public protocol PCMStreamingAudioPlaying {
func play(stream: AsyncThrowingStream<Data, Error>, sampleRate: Double) async -> StreamingPlaybackResult
func stop() -> Double?
}
extension StreamingAudioPlayer: StreamingAudioPlaying {}
extension PCMStreamingAudioPlayer: PCMStreamingAudioPlaying {}

View File

@@ -0,0 +1,33 @@
import Foundation
public enum BonjourEscapes {
/// mDNS / DNS-SD commonly escapes bytes in instance names as `\DDD` (decimal-encoded),
/// e.g. spaces are `\032`.
public static func decode(_ input: String) -> String {
var out = ""
var i = input.startIndex
while i < input.endIndex {
if input[i] == "\\",
let d0 = input.index(i, offsetBy: 1, limitedBy: input.index(before: input.endIndex)),
let d1 = input.index(i, offsetBy: 2, limitedBy: input.index(before: input.endIndex)),
let d2 = input.index(i, offsetBy: 3, limitedBy: input.index(before: input.endIndex)),
input[d0].isNumber,
input[d1].isNumber,
input[d2].isNumber
{
let digits = String(input[d0...d2])
if let value = Int(digits),
let scalar = UnicodeScalar(value)
{
out.append(Character(scalar))
i = input.index(i, offsetBy: 4)
continue
}
}
out.append(input[i])
i = input.index(after: i)
}
return out
}
}

View File

@@ -0,0 +1,27 @@
import Foundation
public enum ClawdbotBonjour {
// v0: internal-only, subject to rename.
public static let bridgeServiceType = "_clawdbot-bridge._tcp"
public static let bridgeServiceDomain = "local."
public static let wideAreaBridgeServiceDomain = "clawdbot.internal."
public static let bridgeServiceDomains = [
bridgeServiceDomain,
wideAreaBridgeServiceDomain,
]
public static func normalizeServiceDomain(_ raw: String?) -> String {
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
return self.bridgeServiceDomain
}
let lower = trimmed.lowercased()
if lower == "local" || lower == "local." {
return self.bridgeServiceDomain
}
return lower.hasSuffix(".") ? lower : (lower + ".")
}
}

View File

@@ -0,0 +1,242 @@
import Foundation
public struct BridgeBaseFrame: Codable, Sendable {
public let type: String
public init(type: String) {
self.type = type
}
}
public struct BridgeInvokeRequest: Codable, Sendable {
public let type: String
public let id: String
public let command: String
public let paramsJSON: String?
public init(type: String = "invoke", id: String, command: String, paramsJSON: String? = nil) {
self.type = type
self.id = id
self.command = command
self.paramsJSON = paramsJSON
}
}
public struct BridgeInvokeResponse: Codable, Sendable {
public let type: String
public let id: String
public let ok: Bool
public let payloadJSON: String?
public let error: ClawdbotNodeError?
public init(
type: String = "invoke-res",
id: String,
ok: Bool,
payloadJSON: String? = nil,
error: ClawdbotNodeError? = nil)
{
self.type = type
self.id = id
self.ok = ok
self.payloadJSON = payloadJSON
self.error = error
}
}
public struct BridgeEventFrame: Codable, Sendable {
public let type: String
public let event: String
public let payloadJSON: String?
public init(type: String = "event", event: String, payloadJSON: String? = nil) {
self.type = type
self.event = event
self.payloadJSON = payloadJSON
}
}
public struct BridgeHello: Codable, Sendable {
public let type: String
public let nodeId: String
public let displayName: String?
public let token: String?
public let platform: String?
public let version: String?
public let deviceFamily: String?
public let modelIdentifier: String?
public let caps: [String]?
public let commands: [String]?
public let permissions: [String: Bool]?
public init(
type: String = "hello",
nodeId: String,
displayName: String?,
token: String?,
platform: String?,
version: String?,
deviceFamily: String? = nil,
modelIdentifier: String? = nil,
caps: [String]? = nil,
commands: [String]? = nil,
permissions: [String: Bool]? = nil)
{
self.type = type
self.nodeId = nodeId
self.displayName = displayName
self.token = token
self.platform = platform
self.version = version
self.deviceFamily = deviceFamily
self.modelIdentifier = modelIdentifier
self.caps = caps
self.commands = commands
self.permissions = permissions
}
}
public struct BridgeHelloOk: Codable, Sendable {
public let type: String
public let serverName: String
public let canvasHostUrl: String?
public init(type: String = "hello-ok", serverName: String, canvasHostUrl: String? = nil) {
self.type = type
self.serverName = serverName
self.canvasHostUrl = canvasHostUrl
}
}
public struct BridgePairRequest: Codable, Sendable {
public let type: String
public let nodeId: String
public let displayName: String?
public let platform: String?
public let version: String?
public let deviceFamily: String?
public let modelIdentifier: String?
public let caps: [String]?
public let commands: [String]?
public let permissions: [String: Bool]?
public let remoteAddress: String?
public let silent: Bool?
public init(
type: String = "pair-request",
nodeId: String,
displayName: String?,
platform: String?,
version: String?,
deviceFamily: String? = nil,
modelIdentifier: String? = nil,
caps: [String]? = nil,
commands: [String]? = nil,
permissions: [String: Bool]? = nil,
remoteAddress: String? = nil,
silent: Bool? = nil)
{
self.type = type
self.nodeId = nodeId
self.displayName = displayName
self.platform = platform
self.version = version
self.deviceFamily = deviceFamily
self.modelIdentifier = modelIdentifier
self.caps = caps
self.commands = commands
self.permissions = permissions
self.remoteAddress = remoteAddress
self.silent = silent
}
}
public struct BridgePairOk: Codable, Sendable {
public let type: String
public let token: String
public init(type: String = "pair-ok", token: String) {
self.type = type
self.token = token
}
}
public struct BridgePing: Codable, Sendable {
public let type: String
public let id: String
public init(type: String = "ping", id: String) {
self.type = type
self.id = id
}
}
public struct BridgePong: Codable, Sendable {
public let type: String
public let id: String
public init(type: String = "pong", id: String) {
self.type = type
self.id = id
}
}
public struct BridgeErrorFrame: Codable, Sendable {
public let type: String
public let code: String
public let message: String
public init(type: String = "error", code: String, message: String) {
self.type = type
self.code = code
self.message = message
}
}
// MARK: - Optional RPC (node -> bridge)
public struct BridgeRPCRequest: Codable, Sendable {
public let type: String
public let id: String
public let method: String
public let paramsJSON: String?
public init(type: String = "req", id: String, method: String, paramsJSON: String? = nil) {
self.type = type
self.id = id
self.method = method
self.paramsJSON = paramsJSON
}
}
public struct BridgeRPCError: Codable, Sendable, Equatable {
public let code: String
public let message: String
public init(code: String, message: String) {
self.code = code
self.message = message
}
}
public struct BridgeRPCResponse: Codable, Sendable {
public let type: String
public let id: String
public let ok: Bool
public let payloadJSON: String?
public let error: BridgeRPCError?
public init(
type: String = "res",
id: String,
ok: Bool,
payloadJSON: String? = nil,
error: BridgeRPCError? = nil)
{
self.type = type
self.id = id
self.ok = ok
self.payloadJSON = payloadJSON
self.error = error
}
}

View File

@@ -0,0 +1,68 @@
import Foundation
public enum ClawdbotCameraCommand: String, Codable, Sendable {
case list = "camera.list"
case snap = "camera.snap"
case clip = "camera.clip"
}
public enum ClawdbotCameraFacing: String, Codable, Sendable {
case back
case front
}
public enum ClawdbotCameraImageFormat: String, Codable, Sendable {
case jpg
case jpeg
}
public enum ClawdbotCameraVideoFormat: String, Codable, Sendable {
case mp4
}
public struct ClawdbotCameraSnapParams: Codable, Sendable, Equatable {
public var facing: ClawdbotCameraFacing?
public var maxWidth: Int?
public var quality: Double?
public var format: ClawdbotCameraImageFormat?
public var deviceId: String?
public var delayMs: Int?
public init(
facing: ClawdbotCameraFacing? = nil,
maxWidth: Int? = nil,
quality: Double? = nil,
format: ClawdbotCameraImageFormat? = nil,
deviceId: String? = nil,
delayMs: Int? = nil)
{
self.facing = facing
self.maxWidth = maxWidth
self.quality = quality
self.format = format
self.deviceId = deviceId
self.delayMs = delayMs
}
}
public struct ClawdbotCameraClipParams: Codable, Sendable, Equatable {
public var facing: ClawdbotCameraFacing?
public var durationMs: Int?
public var includeAudio: Bool?
public var format: ClawdbotCameraVideoFormat?
public var deviceId: String?
public init(
facing: ClawdbotCameraFacing? = nil,
durationMs: Int? = nil,
includeAudio: Bool? = nil,
format: ClawdbotCameraVideoFormat? = nil,
deviceId: String? = nil)
{
self.facing = facing
self.durationMs = durationMs
self.includeAudio = includeAudio
self.format = format
self.deviceId = deviceId
}
}

View File

@@ -0,0 +1,99 @@
import Foundation
public enum ClawdbotCanvasA2UIAction: Sendable {
public struct AgentMessageContext: Sendable {
public struct Session: Sendable {
public var key: String
public var surfaceId: String
public init(key: String, surfaceId: String) {
self.key = key
self.surfaceId = surfaceId
}
}
public struct Component: Sendable {
public var id: String
public var host: String
public var instanceId: String
public init(id: String, host: String, instanceId: String) {
self.id = id
self.host = host
self.instanceId = instanceId
}
}
public var actionName: String
public var session: Session
public var component: Component
public var contextJSON: String?
public init(actionName: String, session: Session, component: Component, contextJSON: String?) {
self.actionName = actionName
self.session = session
self.component = component
self.contextJSON = contextJSON
}
}
public static func extractActionName(_ userAction: [String: Any]) -> String? {
let keys = ["name", "action"]
for key in keys {
if let raw = userAction[key] as? String {
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty { return trimmed }
}
}
return nil
}
public static func sanitizeTagValue(_ value: String) -> String {
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
let nonEmpty = trimmed.isEmpty ? "-" : trimmed
let normalized = nonEmpty.replacingOccurrences(of: " ", with: "_")
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.:")
let scalars = normalized.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" }
return String(scalars)
}
public static func compactJSON(_ obj: Any?) -> String? {
guard let obj else { return nil }
guard JSONSerialization.isValidJSONObject(obj) else { return nil }
guard let data = try? JSONSerialization.data(withJSONObject: obj, options: []),
let str = String(data: data, encoding: .utf8)
else { return nil }
return str
}
public static func formatAgentMessage(_ context: AgentMessageContext) -> String {
let ctxSuffix = context.contextJSON.flatMap { $0.isEmpty ? nil : " ctx=\($0)" } ?? ""
return [
"CANVAS_A2UI",
"action=\(self.sanitizeTagValue(context.actionName))",
"session=\(self.sanitizeTagValue(context.session.key))",
"surface=\(self.sanitizeTagValue(context.session.surfaceId))",
"component=\(self.sanitizeTagValue(context.component.id))",
"host=\(self.sanitizeTagValue(context.component.host))",
"instance=\(self.sanitizeTagValue(context.component.instanceId))\(ctxSuffix)",
"default=update_canvas",
].joined(separator: " ")
}
public static func jsDispatchA2UIActionStatus(actionId: String, ok: Bool, error: String?) -> String {
let payload: [String: Any] = [
"id": actionId,
"ok": ok,
"error": error ?? "",
]
let json: String = {
if let data = try? JSONSerialization.data(withJSONObject: payload, options: []),
let str = String(data: data, encoding: .utf8)
{
return str
}
return "{\"id\":\"\(actionId)\",\"ok\":\(ok ? "true" : "false"),\"error\":\"\"}"
}()
return "window.dispatchEvent(new CustomEvent('clawdbot:a2ui-action-status', { detail: \(json) }));"
}
}

View File

@@ -0,0 +1,26 @@
import Foundation
public enum ClawdbotCanvasA2UICommand: String, Codable, Sendable {
/// Render A2UI content on the device canvas.
case push = "canvas.a2ui.push"
/// Legacy alias for `push` when sending JSONL.
case pushJSONL = "canvas.a2ui.pushJSONL"
/// Reset the A2UI renderer state.
case reset = "canvas.a2ui.reset"
}
public struct ClawdbotCanvasA2UIPushParams: Codable, Sendable, Equatable {
public var messages: [AnyCodable]
public init(messages: [AnyCodable]) {
self.messages = messages
}
}
public struct ClawdbotCanvasA2UIPushJSONLParams: Codable, Sendable, Equatable {
public var jsonl: String
public init(jsonl: String) {
self.jsonl = jsonl
}
}

View File

@@ -0,0 +1,81 @@
import Foundation
public enum ClawdbotCanvasA2UIJSONL: Sendable {
public struct ParsedItem: Sendable {
public var lineNumber: Int
public var message: AnyCodable
public init(lineNumber: Int, message: AnyCodable) {
self.lineNumber = lineNumber
self.message = message
}
}
public static func parse(_ text: String) throws -> [ParsedItem] {
var out: [ParsedItem] = []
var lineNumber = 0
for rawLine in text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {
lineNumber += 1
let line = String(rawLine).trimmingCharacters(in: .whitespacesAndNewlines)
if line.isEmpty { continue }
let data = Data(line.utf8)
let decoded = try JSONDecoder().decode(AnyCodable.self, from: data)
out.append(ParsedItem(lineNumber: lineNumber, message: decoded))
}
return out
}
public static func validateV0_8(_ items: [ParsedItem]) throws {
let allowed = Set([
"beginRendering",
"surfaceUpdate",
"dataModelUpdate",
"deleteSurface",
])
for item in items {
guard let dict = item.message.value as? [String: AnyCodable] else {
throw NSError(domain: "A2UI", code: 1, userInfo: [
NSLocalizedDescriptionKey: "A2UI JSONL line \(item.lineNumber): expected a JSON object",
])
}
if dict.keys.contains("createSurface") {
throw NSError(domain: "A2UI", code: 2, userInfo: [
NSLocalizedDescriptionKey: """
A2UI JSONL line \(item.lineNumber): looks like A2UI v0.9 (`createSurface`).
Canvas currently supports A2UI v0.8 server→client messages
(`beginRendering`, `surfaceUpdate`, `dataModelUpdate`, `deleteSurface`).
""",
])
}
let matched = dict.keys.filter { allowed.contains($0) }
if matched.count != 1 {
let found = dict.keys.sorted().joined(separator: ", ")
throw NSError(domain: "A2UI", code: 3, userInfo: [
NSLocalizedDescriptionKey: """
A2UI JSONL line \(item.lineNumber): expected exactly one of \(allowed.sorted()
.joined(separator: ", ")); found: \(found)
""",
])
}
}
}
public static func decodeMessagesFromJSONL(_ text: String) throws -> [AnyCodable] {
let items = try self.parse(text)
try self.validateV0_8(items)
return items.map(\.message)
}
public static func encodeMessagesJSONArray(_ messages: [AnyCodable]) throws -> String {
let data = try JSONEncoder().encode(messages)
guard let json = String(data: data, encoding: .utf8) else {
throw NSError(domain: "A2UI", code: 10, userInfo: [
NSLocalizedDescriptionKey: "Failed to encode messages payload as UTF-8",
])
}
return json
}
}

View File

@@ -0,0 +1,76 @@
import Foundation
public struct ClawdbotCanvasNavigateParams: Codable, Sendable, Equatable {
public var url: String
public init(url: String) {
self.url = url
}
}
public struct ClawdbotCanvasPlacement: Codable, Sendable, Equatable {
public var x: Double?
public var y: Double?
public var width: Double?
public var height: Double?
public init(x: Double? = nil, y: Double? = nil, width: Double? = nil, height: Double? = nil) {
self.x = x
self.y = y
self.width = width
self.height = height
}
}
public struct ClawdbotCanvasPresentParams: Codable, Sendable, Equatable {
public var url: String?
public var placement: ClawdbotCanvasPlacement?
public init(url: String? = nil, placement: ClawdbotCanvasPlacement? = nil) {
self.url = url
self.placement = placement
}
}
public struct ClawdbotCanvasEvalParams: Codable, Sendable, Equatable {
public var javaScript: String
public init(javaScript: String) {
self.javaScript = javaScript
}
}
public enum ClawdbotCanvasSnapshotFormat: String, Codable, Sendable {
case png
case jpeg
public init(from decoder: Decoder) throws {
let c = try decoder.singleValueContainer()
let raw = try c.decode(String.self).trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
switch raw {
case "png":
self = .png
case "jpeg", "jpg":
self = .jpeg
default:
throw DecodingError.dataCorruptedError(in: c, debugDescription: "Invalid snapshot format: \(raw)")
}
}
public func encode(to encoder: Encoder) throws {
var c = encoder.singleValueContainer()
try c.encode(self.rawValue)
}
}
public struct ClawdbotCanvasSnapshotParams: Codable, Sendable, Equatable {
public var maxWidth: Int?
public var quality: Double?
public var format: ClawdbotCanvasSnapshotFormat?
public init(maxWidth: Int? = nil, quality: Double? = nil, format: ClawdbotCanvasSnapshotFormat? = nil) {
self.maxWidth = maxWidth
self.quality = quality
self.format = format
}
}

View File

@@ -0,0 +1,9 @@
import Foundation
public enum ClawdbotCanvasCommand: String, Codable, Sendable {
case present = "canvas.present"
case hide = "canvas.hide"
case navigate = "canvas.navigate"
case evalJS = "canvas.eval"
case snapshot = "canvas.snapshot"
}

View File

@@ -0,0 +1,9 @@
import Foundation
public enum ClawdbotCapability: String, Codable, Sendable {
case canvas
case camera
case screen
case voiceWake
case location
}

View File

@@ -0,0 +1,5 @@
import Foundation
public enum ClawdbotKitResources {
public static let bundle: Bundle = .module
}

View File

@@ -0,0 +1,72 @@
import Foundation
public enum DeepLinkRoute: Sendable, Equatable {
case agent(AgentDeepLink)
}
public struct AgentDeepLink: Codable, Sendable, Equatable {
public let message: String
public let sessionKey: String?
public let thinking: String?
public let deliver: Bool
public let to: String?
public let channel: String?
public let timeoutSeconds: Int?
public let key: String?
public init(
message: String,
sessionKey: String?,
thinking: String?,
deliver: Bool,
to: String?,
channel: String?,
timeoutSeconds: Int?,
key: String?)
{
self.message = message
self.sessionKey = sessionKey
self.thinking = thinking
self.deliver = deliver
self.to = to
self.channel = channel
self.timeoutSeconds = timeoutSeconds
self.key = key
}
}
public enum DeepLinkParser {
public static func parse(_ url: URL) -> DeepLinkRoute? {
guard url.scheme?.lowercased() == "clawdbot" else { return nil }
guard let host = url.host?.lowercased(), !host.isEmpty else { return nil }
guard let comps = URLComponents(url: url, resolvingAgainstBaseURL: false) else { return nil }
let query = (comps.queryItems ?? []).reduce(into: [String: String]()) { dict, item in
guard let value = item.value else { return }
dict[item.name] = value
}
switch host {
case "agent":
guard let message = query["message"],
!message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
else {
return nil
}
let deliver = (query["deliver"] as NSString?)?.boolValue ?? false
let timeoutSeconds = query["timeoutSeconds"].flatMap { Int($0) }.flatMap { $0 >= 0 ? $0 : nil }
return .agent(
.init(
message: message,
sessionKey: query["sessionKey"],
thinking: query["thinking"],
deliver: deliver,
to: query["to"],
channel: query["channel"],
timeoutSeconds: timeoutSeconds,
key: query["key"]))
default:
return nil
}
}
}

View File

@@ -0,0 +1,9 @@
@_exported import ElevenLabsKit
public typealias ElevenLabsVoice = ElevenLabsKit.ElevenLabsVoice
public typealias ElevenLabsTTSRequest = ElevenLabsKit.ElevenLabsTTSRequest
public typealias ElevenLabsTTSClient = ElevenLabsKit.ElevenLabsTTSClient
public typealias TalkTTSValidation = ElevenLabsKit.TalkTTSValidation
public typealias StreamingAudioPlayer = ElevenLabsKit.StreamingAudioPlayer
public typealias PCMStreamingAudioPlayer = ElevenLabsKit.PCMStreamingAudioPlayer
public typealias StreamingPlaybackResult = ElevenLabsKit.StreamingPlaybackResult

View File

@@ -0,0 +1,135 @@
import CoreGraphics
import Foundation
import ImageIO
import UniformTypeIdentifiers
public enum JPEGTranscodeError: LocalizedError, Sendable {
case decodeFailed
case propertiesMissing
case encodeFailed
case sizeLimitExceeded(maxBytes: Int, actualBytes: Int)
public var errorDescription: String? {
switch self {
case .decodeFailed:
"Failed to decode image data"
case .propertiesMissing:
"Failed to read image properties"
case .encodeFailed:
"Failed to encode JPEG"
case let .sizeLimitExceeded(maxBytes, actualBytes):
"JPEG exceeds size limit (\(actualBytes) bytes > \(maxBytes) bytes)"
}
}
}
public struct JPEGTranscoder: Sendable {
public static func clampQuality(_ quality: Double) -> Double {
min(1.0, max(0.05, quality))
}
/// Re-encodes image data to JPEG, optionally downscaling so that the *oriented* pixel width is <= `maxWidthPx`.
///
/// - Important: This normalizes EXIF orientation (the output pixels are rotated if needed; orientation tag is not
/// relied on).
public static func transcodeToJPEG(
imageData: Data,
maxWidthPx: Int?,
quality: Double,
maxBytes: Int? = nil) throws -> (data: Data, widthPx: Int, heightPx: Int)
{
guard let src = CGImageSourceCreateWithData(imageData as CFData, nil) else {
throw JPEGTranscodeError.decodeFailed
}
guard
let props = CGImageSourceCopyPropertiesAtIndex(src, 0, nil) as? [CFString: Any],
let rawWidth = props[kCGImagePropertyPixelWidth] as? NSNumber,
let rawHeight = props[kCGImagePropertyPixelHeight] as? NSNumber
else {
throw JPEGTranscodeError.propertiesMissing
}
let pixelWidth = rawWidth.intValue
let pixelHeight = rawHeight.intValue
let orientation = (props[kCGImagePropertyOrientation] as? NSNumber)?.intValue ?? 1
guard pixelWidth > 0, pixelHeight > 0 else {
throw JPEGTranscodeError.propertiesMissing
}
let rotates90 = orientation == 5 || orientation == 6 || orientation == 7 || orientation == 8
let orientedWidth = rotates90 ? pixelHeight : pixelWidth
let orientedHeight = rotates90 ? pixelWidth : pixelHeight
let maxDim = max(orientedWidth, orientedHeight)
var targetMaxPixelSize: Int = {
guard let maxWidthPx, maxWidthPx > 0 else { return maxDim }
guard orientedWidth > maxWidthPx else { return maxDim } // never upscale
let scale = Double(maxWidthPx) / Double(orientedWidth)
return max(1, Int((Double(maxDim) * scale).rounded(.toNearestOrAwayFromZero)))
}()
func encode(maxPixelSize: Int, quality: Double) throws -> (data: Data, widthPx: Int, heightPx: Int) {
let thumbOpts: [CFString: Any] = [
kCGImageSourceCreateThumbnailFromImageAlways: true,
kCGImageSourceCreateThumbnailWithTransform: true,
kCGImageSourceThumbnailMaxPixelSize: maxPixelSize,
kCGImageSourceShouldCacheImmediately: true,
]
guard let img = CGImageSourceCreateThumbnailAtIndex(src, 0, thumbOpts as CFDictionary) else {
throw JPEGTranscodeError.decodeFailed
}
let out = NSMutableData()
guard let dest = CGImageDestinationCreateWithData(out, UTType.jpeg.identifier as CFString, 1, nil) else {
throw JPEGTranscodeError.encodeFailed
}
let q = self.clampQuality(quality)
let encodeProps = [kCGImageDestinationLossyCompressionQuality: q] as CFDictionary
CGImageDestinationAddImage(dest, img, encodeProps)
guard CGImageDestinationFinalize(dest) else {
throw JPEGTranscodeError.encodeFailed
}
return (out as Data, img.width, img.height)
}
guard let maxBytes, maxBytes > 0 else {
return try encode(maxPixelSize: targetMaxPixelSize, quality: quality)
}
let minQuality = max(0.2, self.clampQuality(quality) * 0.35)
let minPixelSize = 256
var best = try encode(maxPixelSize: targetMaxPixelSize, quality: quality)
if best.data.count <= maxBytes {
return best
}
for _ in 0..<6 {
var q = self.clampQuality(quality)
for _ in 0..<6 {
let candidate = try encode(maxPixelSize: targetMaxPixelSize, quality: q)
best = candidate
if candidate.data.count <= maxBytes {
return candidate
}
if q <= minQuality { break }
q = max(minQuality, q * 0.75)
}
let nextPixelSize = max(Int(Double(targetMaxPixelSize) * 0.85), minPixelSize)
if nextPixelSize == targetMaxPixelSize {
break
}
targetMaxPixelSize = nextPixelSize
}
if best.data.count > maxBytes {
throw JPEGTranscodeError.sizeLimitExceeded(maxBytes: maxBytes, actualBytes: best.data.count)
}
return best
}
}

View File

@@ -0,0 +1,57 @@
import Foundation
public enum ClawdbotLocationCommand: String, Codable, Sendable {
case get = "location.get"
}
public enum ClawdbotLocationAccuracy: String, Codable, Sendable {
case coarse
case balanced
case precise
}
public struct ClawdbotLocationGetParams: Codable, Sendable, Equatable {
public var timeoutMs: Int?
public var maxAgeMs: Int?
public var desiredAccuracy: ClawdbotLocationAccuracy?
public init(timeoutMs: Int? = nil, maxAgeMs: Int? = nil, desiredAccuracy: ClawdbotLocationAccuracy? = nil) {
self.timeoutMs = timeoutMs
self.maxAgeMs = maxAgeMs
self.desiredAccuracy = desiredAccuracy
}
}
public struct ClawdbotLocationPayload: Codable, Sendable, Equatable {
public var lat: Double
public var lon: Double
public var accuracyMeters: Double
public var altitudeMeters: Double?
public var speedMps: Double?
public var headingDeg: Double?
public var timestamp: String
public var isPrecise: Bool
public var source: String?
public init(
lat: Double,
lon: Double,
accuracyMeters: Double,
altitudeMeters: Double? = nil,
speedMps: Double? = nil,
headingDeg: Double? = nil,
timestamp: String,
isPrecise: Bool,
source: String? = nil)
{
self.lat = lat
self.lon = lon
self.accuracyMeters = accuracyMeters
self.altitudeMeters = altitudeMeters
self.speedMps = speedMps
self.headingDeg = headingDeg
self.timestamp = timestamp
self.isPrecise = isPrecise
self.source = source
}
}

View File

@@ -0,0 +1,7 @@
import Foundation
public enum ClawdbotLocationMode: String, Codable, Sendable, CaseIterable {
case off
case whileUsing
case always
}

View File

@@ -0,0 +1,28 @@
import Foundation
public enum ClawdbotNodeErrorCode: String, Codable, Sendable {
case notPaired = "NOT_PAIRED"
case unauthorized = "UNAUTHORIZED"
case backgroundUnavailable = "NODE_BACKGROUND_UNAVAILABLE"
case invalidRequest = "INVALID_REQUEST"
case unavailable = "UNAVAILABLE"
}
public struct ClawdbotNodeError: Error, Codable, Sendable, Equatable {
public var code: ClawdbotNodeErrorCode
public var message: String
public var retryable: Bool?
public var retryAfterMs: Int?
public init(
code: ClawdbotNodeErrorCode,
message: String,
retryable: Bool? = nil,
retryAfterMs: Int? = nil)
{
self.code = code
self.message = message
self.retryable = retryable
self.retryAfterMs = retryAfterMs
}
}

View File

@@ -0,0 +1,224 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
<title>Canvas</title>
<script>
(() => {
try {
const params = new URLSearchParams(window.location.search);
const platform = (params.get('platform') || '').trim().toLowerCase();
if (platform) {
document.documentElement.dataset.platform = platform;
return;
}
if (/android/i.test(navigator.userAgent || '')) {
document.documentElement.dataset.platform = 'android';
}
} catch (_) {}
})();
</script>
<style>
:root { color-scheme: dark; }
@media (prefers-reduced-motion: reduce) {
body::before, body::after { animation: none !important; }
}
html,body { height:100%; margin:0; }
body {
background:
radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.18), rgba(0,0,0,0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.14), rgba(0,0,0,0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.10), rgba(0,0,0,0) 60%),
#000;
overflow: hidden;
}
:root[data-platform="android"] body {
background:
radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.62), rgba(0,0,0,0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.52), rgba(0,0,0,0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.48), rgba(0,0,0,0) 60%),
#0b1328;
}
body::before {
content:"";
position: fixed;
inset: -20%;
background:
repeating-linear-gradient(0deg, rgba(255,255,255,0.03) 0, rgba(255,255,255,0.03) 1px,
transparent 1px, transparent 48px),
repeating-linear-gradient(90deg, rgba(255,255,255,0.03) 0, rgba(255,255,255,0.03) 1px,
transparent 1px, transparent 48px);
transform: translate3d(0,0,0) rotate(-7deg);
will-change: transform, opacity;
-webkit-backface-visibility: hidden;
backface-visibility: hidden;
opacity: 0.45;
pointer-events: none;
animation: clawdbot-grid-drift 140s ease-in-out infinite alternate;
}
:root[data-platform="android"] body::before { opacity: 0.80; }
body::after {
content:"";
position: fixed;
inset: -35%;
background:
radial-gradient(900px 700px at 30% 30%, rgba(42,113,255,0.16), rgba(0,0,0,0) 60%),
radial-gradient(800px 650px at 70% 35%, rgba(255,0,138,0.12), rgba(0,0,0,0) 62%),
radial-gradient(900px 800px at 55% 75%, rgba(0,209,255,0.10), rgba(0,0,0,0) 62%);
filter: blur(28px);
opacity: 0.52;
will-change: transform, opacity;
-webkit-backface-visibility: hidden;
backface-visibility: hidden;
transform: translate3d(0,0,0);
pointer-events: none;
animation: clawdbot-glow-drift 110s ease-in-out infinite alternate;
}
:root[data-platform="android"] body::after { opacity: 0.85; }
@supports (mix-blend-mode: screen) {
body::after { mix-blend-mode: screen; }
}
@supports not (mix-blend-mode: screen) {
body::after { opacity: 0.70; }
}
@keyframes clawdbot-grid-drift {
0% { transform: translate3d(-12px, 8px, 0) rotate(-7deg); opacity: 0.40; }
50% { transform: translate3d( 10px,-7px, 0) rotate(-6.6deg); opacity: 0.56; }
100% { transform: translate3d(-8px, 6px, 0) rotate(-7.2deg); opacity: 0.42; }
}
@keyframes clawdbot-glow-drift {
0% { transform: translate3d(-18px, 12px, 0) scale(1.02); opacity: 0.40; }
50% { transform: translate3d( 14px,-10px, 0) scale(1.05); opacity: 0.52; }
100% { transform: translate3d(-10px, 8px, 0) scale(1.03); opacity: 0.43; }
}
canvas {
position: fixed;
inset: 0;
display:block;
width:100vw;
height:100vh;
touch-action: none;
z-index: 1;
}
:root[data-platform="android"] #clawdbot-canvas {
background:
radial-gradient(1100px 800px at 20% 15%, rgba(42, 113, 255, 0.78), rgba(0,0,0,0) 58%),
radial-gradient(900px 650px at 82% 28%, rgba(255, 0, 138, 0.66), rgba(0,0,0,0) 62%),
radial-gradient(1000px 900px at 60% 88%, rgba(0, 209, 255, 0.58), rgba(0,0,0,0) 62%),
#141c33;
}
#clawdbot-status {
position: fixed;
inset: 0;
display: none;
align-items: center;
justify-content: flex-start;
flex-direction: column;
padding-top: calc(20px + env(safe-area-inset-top, 0px));
pointer-events: none;
z-index: 3;
}
#clawdbot-status .card {
text-align: center;
padding: 16px 18px;
border-radius: 14px;
background: rgba(18, 18, 22, 0.42);
border: 1px solid rgba(255,255,255,0.08);
box-shadow: 0 18px 60px rgba(0,0,0,0.55);
-webkit-backdrop-filter: blur(14px);
backdrop-filter: blur(14px);
}
#clawdbot-status .title {
font: 600 20px -apple-system, BlinkMacSystemFont, "SF Pro Display", "SF Pro Text", system-ui, sans-serif;
letter-spacing: 0.2px;
color: rgba(255,255,255,0.92);
text-shadow: 0 0 22px rgba(42, 113, 255, 0.35);
}
#clawdbot-status .subtitle {
margin-top: 6px;
font: 500 12px -apple-system, BlinkMacSystemFont, "SF Pro Text", system-ui, sans-serif;
color: rgba(255,255,255,0.58);
}
</style>
</head>
<body>
<canvas id="clawdbot-canvas"></canvas>
<div id="clawdbot-status">
<div class="card">
<div class="title" id="clawdbot-status-title">Ready</div>
<div class="subtitle" id="clawdbot-status-subtitle">Waiting for agent</div>
</div>
</div>
<script>
(() => {
const canvas = document.getElementById('clawdbot-canvas');
const ctx = canvas.getContext('2d');
const statusEl = document.getElementById('clawdbot-status');
const titleEl = document.getElementById('clawdbot-status-title');
const subtitleEl = document.getElementById('clawdbot-status-subtitle');
const debugStatusEnabledByQuery = (() => {
try {
const params = new URLSearchParams(window.location.search);
const raw = params.get('debugStatus') ?? params.get('debug');
if (!raw) return false;
const normalized = String(raw).trim().toLowerCase();
return normalized === '1' || normalized === 'true' || normalized === 'yes';
} catch (_) {
return false;
}
})();
let debugStatusEnabled = debugStatusEnabledByQuery;
function resize() {
const dpr = window.devicePixelRatio || 1;
const w = Math.max(1, Math.floor(window.innerWidth * dpr));
const h = Math.max(1, Math.floor(window.innerHeight * dpr));
canvas.width = w;
canvas.height = h;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
}
window.addEventListener('resize', resize);
resize();
const setDebugStatusEnabled = (enabled) => {
debugStatusEnabled = !!enabled;
if (!statusEl) return;
if (!debugStatusEnabled) {
statusEl.style.display = 'none';
}
};
if (statusEl && !debugStatusEnabled) {
statusEl.style.display = 'none';
}
window.__clawdbot = {
canvas,
ctx,
setDebugStatusEnabled,
setStatus: (title, subtitle) => {
if (!statusEl || !debugStatusEnabled) return;
if (!title && !subtitle) {
statusEl.style.display = 'none';
return;
}
statusEl.style.display = 'flex';
if (titleEl && typeof title === 'string') titleEl.textContent = title;
if (subtitleEl && typeof subtitle === 'string') subtitleEl.textContent = subtitle;
if (!debugStatusEnabled) {
clearTimeout(window.__statusTimeout);
window.__statusTimeout = setTimeout(() => {
statusEl.style.display = 'none';
}, 3000);
} else {
clearTimeout(window.__statusTimeout);
}
}
};
})();
</script>
</body>
</html>

View File

@@ -0,0 +1,27 @@
import Foundation
public enum ClawdbotScreenCommand: String, Codable, Sendable {
case record = "screen.record"
}
public struct ClawdbotScreenRecordParams: Codable, Sendable, Equatable {
public var screenIndex: Int?
public var durationMs: Int?
public var fps: Double?
public var format: String?
public var includeAudio: Bool?
public init(
screenIndex: Int? = nil,
durationMs: Int? = nil,
fps: Double? = nil,
format: String? = nil,
includeAudio: Bool? = nil)
{
self.screenIndex = screenIndex
self.durationMs = durationMs
self.fps = fps
self.format = format
self.includeAudio = includeAudio
}
}

View File

@@ -0,0 +1,37 @@
import Foundation
public enum ClawdbotNodeStorage {
public static func appSupportDir() throws -> URL {
let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first
guard let base else {
throw NSError(domain: "ClawdbotNodeStorage", code: 1, userInfo: [
NSLocalizedDescriptionKey: "Application Support directory unavailable",
])
}
return base.appendingPathComponent("Clawdbot", isDirectory: true)
}
public static func canvasRoot(sessionKey: String) throws -> URL {
let root = try appSupportDir().appendingPathComponent("canvas", isDirectory: true)
let safe = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
let session = safe.isEmpty ? "main" : safe
return root.appendingPathComponent(session, isDirectory: true)
}
public static func cachesDir() throws -> URL {
let base = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first
guard let base else {
throw NSError(domain: "ClawdbotNodeStorage", code: 2, userInfo: [
NSLocalizedDescriptionKey: "Caches directory unavailable",
])
}
return base.appendingPathComponent("Clawdbot", isDirectory: true)
}
public static func canvasSnapshotsRoot(sessionKey: String) throws -> URL {
let root = try cachesDir().appendingPathComponent("canvas-snapshots", isDirectory: true)
let safe = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
let session = safe.isEmpty ? "main" : safe
return root.appendingPathComponent(session, isDirectory: true)
}
}

View File

@@ -0,0 +1,62 @@
import Foundation
public enum ClawdbotSystemCommand: String, Codable, Sendable {
case run = "system.run"
case notify = "system.notify"
}
public enum ClawdbotNotificationPriority: String, Codable, Sendable {
case passive
case active
case timeSensitive
}
public enum ClawdbotNotificationDelivery: String, Codable, Sendable {
case system
case overlay
case auto
}
public struct ClawdbotSystemRunParams: Codable, Sendable, Equatable {
public var command: [String]
public var cwd: String?
public var env: [String: String]?
public var timeoutMs: Int?
public var needsScreenRecording: Bool?
public init(
command: [String],
cwd: String? = nil,
env: [String: String]? = nil,
timeoutMs: Int? = nil,
needsScreenRecording: Bool? = nil)
{
self.command = command
self.cwd = cwd
self.env = env
self.timeoutMs = timeoutMs
self.needsScreenRecording = needsScreenRecording
}
}
public struct ClawdbotSystemNotifyParams: Codable, Sendable, Equatable {
public var title: String
public var body: String
public var sound: String?
public var priority: ClawdbotNotificationPriority?
public var delivery: ClawdbotNotificationDelivery?
public init(
title: String,
body: String,
sound: String? = nil,
priority: ClawdbotNotificationPriority? = nil,
delivery: ClawdbotNotificationDelivery? = nil)
{
self.title = title
self.body = body
self.sound = sound
self.priority = priority
self.delivery = delivery
}
}

View File

@@ -0,0 +1,201 @@
import Foundation
public struct TalkDirective: Equatable, Sendable {
public var voiceId: String?
public var modelId: String?
public var speed: Double?
public var rateWPM: Int?
public var stability: Double?
public var similarity: Double?
public var style: Double?
public var speakerBoost: Bool?
public var seed: Int?
public var normalize: String?
public var language: String?
public var outputFormat: String?
public var latencyTier: Int?
public var once: Bool?
public init(
voiceId: String? = nil,
modelId: String? = nil,
speed: Double? = nil,
rateWPM: Int? = nil,
stability: Double? = nil,
similarity: Double? = nil,
style: Double? = nil,
speakerBoost: Bool? = nil,
seed: Int? = nil,
normalize: String? = nil,
language: String? = nil,
outputFormat: String? = nil,
latencyTier: Int? = nil,
once: Bool? = nil)
{
self.voiceId = voiceId
self.modelId = modelId
self.speed = speed
self.rateWPM = rateWPM
self.stability = stability
self.similarity = similarity
self.style = style
self.speakerBoost = speakerBoost
self.seed = seed
self.normalize = normalize
self.language = language
self.outputFormat = outputFormat
self.latencyTier = latencyTier
self.once = once
}
}
public struct TalkDirectiveParseResult: Equatable, Sendable {
public let directive: TalkDirective?
public let stripped: String
public let unknownKeys: [String]
public init(directive: TalkDirective?, stripped: String, unknownKeys: [String]) {
self.directive = directive
self.stripped = stripped
self.unknownKeys = unknownKeys
}
}
public enum TalkDirectiveParser {
public static func parse(_ text: String) -> TalkDirectiveParseResult {
let normalized = text.replacingOccurrences(of: "\r\n", with: "\n")
var lines = normalized.split(separator: "\n", omittingEmptySubsequences: false)
guard !lines.isEmpty else { return TalkDirectiveParseResult(directive: nil, stripped: text, unknownKeys: []) }
guard let firstNonEmptyIndex =
lines.firstIndex(where: { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty })
else {
return TalkDirectiveParseResult(directive: nil, stripped: text, unknownKeys: [])
}
var firstNonEmpty = firstNonEmptyIndex
if firstNonEmpty > 0 {
lines.removeSubrange(0..<firstNonEmpty)
firstNonEmpty = 0
}
let head = lines[firstNonEmpty].trimmingCharacters(in: .whitespacesAndNewlines)
guard head.hasPrefix("{"), head.hasSuffix("}") else {
return TalkDirectiveParseResult(directive: nil, stripped: text, unknownKeys: [])
}
guard let data = head.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
else {
return TalkDirectiveParseResult(directive: nil, stripped: text, unknownKeys: [])
}
let speakerBoost = self.boolValue(json, keys: ["speaker_boost", "speakerBoost"])
?? self.boolValue(json, keys: ["no_speaker_boost", "noSpeakerBoost"]).map { !$0 }
let directive = TalkDirective(
voiceId: stringValue(json, keys: ["voice", "voice_id", "voiceId"]),
modelId: stringValue(json, keys: ["model", "model_id", "modelId"]),
speed: doubleValue(json, keys: ["speed"]),
rateWPM: intValue(json, keys: ["rate", "wpm"]),
stability: doubleValue(json, keys: ["stability"]),
similarity: doubleValue(json, keys: ["similarity", "similarity_boost", "similarityBoost"]),
style: doubleValue(json, keys: ["style"]),
speakerBoost: speakerBoost,
seed: intValue(json, keys: ["seed"]),
normalize: stringValue(json, keys: ["normalize", "apply_text_normalization"]),
language: stringValue(json, keys: ["lang", "language_code", "language"]),
outputFormat: stringValue(json, keys: ["output_format", "format"]),
latencyTier: intValue(json, keys: ["latency", "latency_tier", "latencyTier"]),
once: boolValue(json, keys: ["once"]))
let hasDirective = [
directive.voiceId,
directive.modelId,
directive.speed.map { "\($0)" },
directive.rateWPM.map { "\($0)" },
directive.stability.map { "\($0)" },
directive.similarity.map { "\($0)" },
directive.style.map { "\($0)" },
directive.speakerBoost.map { "\($0)" },
directive.seed.map { "\($0)" },
directive.normalize,
directive.language,
directive.outputFormat,
directive.latencyTier.map { "\($0)" },
directive.once.map { "\($0)" },
].contains { $0 != nil }
guard hasDirective else {
return TalkDirectiveParseResult(directive: nil, stripped: text, unknownKeys: [])
}
let knownKeys = Set([
"voice", "voice_id", "voiceid",
"model", "model_id", "modelid",
"speed", "rate", "wpm",
"stability", "similarity", "similarity_boost", "similarityboost",
"style",
"speaker_boost", "speakerboost",
"no_speaker_boost", "nospeakerboost",
"seed",
"normalize", "apply_text_normalization",
"lang", "language_code", "language",
"output_format", "format",
"latency", "latency_tier", "latencytier",
"once",
])
let unknownKeys = json.keys.filter { !knownKeys.contains($0.lowercased()) }.sorted()
lines.remove(at: firstNonEmpty)
if firstNonEmpty < lines.count {
let next = lines[firstNonEmpty].trimmingCharacters(in: .whitespacesAndNewlines)
if next.isEmpty {
lines.remove(at: firstNonEmpty)
}
}
let stripped = lines.joined(separator: "\n")
return TalkDirectiveParseResult(directive: directive, stripped: stripped, unknownKeys: unknownKeys)
}
private static func stringValue(_ dict: [String: Any], keys: [String]) -> String? {
for key in keys {
if let value = dict[key] as? String {
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty { return trimmed }
}
}
return nil
}
private static func doubleValue(_ dict: [String: Any], keys: [String]) -> Double? {
for key in keys {
if let value = dict[key] as? Double { return value }
if let value = dict[key] as? Int { return Double(value) }
if let value = dict[key] as? String, let parsed = Double(value) { return parsed }
}
return nil
}
private static func intValue(_ dict: [String: Any], keys: [String]) -> Int? {
for key in keys {
if let value = dict[key] as? Int { return value }
if let value = dict[key] as? Double { return Int(value) }
if let value = dict[key] as? String, let parsed = Int(value) { return parsed }
}
return nil
}
private static func boolValue(_ dict: [String: Any], keys: [String]) -> Bool? {
for key in keys {
if let value = dict[key] as? Bool { return value }
if let value = dict[key] as? String {
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
if ["true", "yes", "1"].contains(trimmed) { return true }
if ["false", "no", "0"].contains(trimmed) { return false }
}
}
return nil
}
}

View File

@@ -0,0 +1,13 @@
public enum TalkHistoryTimestamp: Sendable {
/// Gateway history timestamps have historically been emitted as either seconds (Double, epoch seconds)
/// or milliseconds (Double, epoch ms). This helper accepts either.
public static func isAfter(_ timestamp: Double, sinceSeconds: Double) -> Bool {
let sinceMs = sinceSeconds * 1000
// ~2286-11-20 in epoch seconds. Anything bigger is almost certainly epoch milliseconds.
if timestamp > 10_000_000_000 {
return timestamp >= sinceMs - 500
}
return timestamp >= sinceSeconds - 0.5
}
}

View File

@@ -0,0 +1,17 @@
public enum TalkPromptBuilder: Sendable {
public static func build(transcript: String, interruptedAtSeconds: Double?) -> String {
var lines: [String] = [
"Talk Mode active. Reply in a concise, spoken tone.",
"You may optionally prefix the response with JSON (first line) to set ElevenLabs voice (id or alias), e.g. {\"voice\":\"<id>\",\"once\":true}.",
]
if let interruptedAtSeconds {
let formatted = String(format: "%.1f", interruptedAtSeconds)
lines.append("Assistant speech interrupted at \(formatted)s.")
}
lines.append("")
lines.append(transcript)
return lines.joined(separator: "\n")
}
}

View File

@@ -0,0 +1,110 @@
import AVFoundation
import Foundation
@MainActor
public final class TalkSystemSpeechSynthesizer: NSObject {
public enum SpeakError: Error {
case canceled
}
public static let shared = TalkSystemSpeechSynthesizer()
private let synth = AVSpeechSynthesizer()
private var speakContinuation: CheckedContinuation<Void, Error>?
private var currentUtterance: AVSpeechUtterance?
private var currentToken = UUID()
private var watchdog: Task<Void, Never>?
public var isSpeaking: Bool { self.synth.isSpeaking }
private override init() {
super.init()
self.synth.delegate = self
}
public func stop() {
self.currentToken = UUID()
self.watchdog?.cancel()
self.watchdog = nil
self.synth.stopSpeaking(at: .immediate)
self.finishCurrent(with: SpeakError.canceled)
}
public func speak(text: String, language: String? = nil) async throws {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
self.stop()
let token = UUID()
self.currentToken = token
let utterance = AVSpeechUtterance(string: trimmed)
if let language, let voice = AVSpeechSynthesisVoice(language: language) {
utterance.voice = voice
}
self.currentUtterance = utterance
let estimatedSeconds = max(3.0, min(180.0, Double(trimmed.count) * 0.08))
self.watchdog?.cancel()
self.watchdog = Task { @MainActor [weak self] in
guard let self else { return }
try? await Task.sleep(nanoseconds: UInt64(estimatedSeconds * 1_000_000_000))
if Task.isCancelled { return }
guard self.currentToken == token else { return }
if self.synth.isSpeaking {
self.synth.stopSpeaking(at: .immediate)
}
self.finishCurrent(
with: NSError(domain: "TalkSystemSpeechSynthesizer", code: 408, userInfo: [
NSLocalizedDescriptionKey: "system TTS timed out after \(estimatedSeconds)s",
]))
}
try await withTaskCancellationHandler(operation: {
try await withCheckedThrowingContinuation { cont in
self.speakContinuation = cont
self.synth.speak(utterance)
}
}, onCancel: {
Task { @MainActor in
self.stop()
}
})
if self.currentToken != token {
throw SpeakError.canceled
}
}
private func handleFinish(error: Error?) {
guard self.currentUtterance != nil else { return }
self.watchdog?.cancel()
self.watchdog = nil
self.finishCurrent(with: error)
}
private func finishCurrent(with error: Error?) {
self.currentUtterance = nil
let cont = self.speakContinuation
self.speakContinuation = nil
if let error {
cont?.resume(throwing: error)
} else {
cont?.resume(returning: ())
}
}
}
extension TalkSystemSpeechSynthesizer: AVSpeechSynthesizerDelegate {
public nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) {
Task { @MainActor in
self.handleFinish(error: nil)
}
}
public nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) {
Task { @MainActor in
self.handleFinish(error: SpeakError.canceled)
}
}
}

View File

@@ -0,0 +1,194 @@
import Foundation
public struct ToolDisplaySummary: Sendable, Equatable {
public let name: String
public let emoji: String
public let title: String
public let label: String
public let verb: String?
public let detail: String?
public var detailLine: String? {
var parts: [String] = []
if let verb, !verb.isEmpty { parts.append(verb) }
if let detail, !detail.isEmpty { parts.append(detail) }
return parts.isEmpty ? nil : parts.joined(separator: " · ")
}
public var summaryLine: String {
if let detailLine {
return "\(emoji) \(label): \(detailLine)"
}
return "\(emoji) \(label)"
}
}
public enum ToolDisplayRegistry {
private struct ToolDisplayActionSpec: Decodable {
let label: String?
let detailKeys: [String]?
}
private struct ToolDisplaySpec: Decodable {
let emoji: String?
let title: String?
let label: String?
let detailKeys: [String]?
let actions: [String: ToolDisplayActionSpec]?
}
private struct ToolDisplayConfig: Decodable {
let version: Int?
let fallback: ToolDisplaySpec?
let tools: [String: ToolDisplaySpec]?
}
private static let config: ToolDisplayConfig = loadConfig()
public static func resolve(name: String?, args: AnyCodable?, meta: String? = nil) -> ToolDisplaySummary {
let trimmedName = name?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "tool"
let key = trimmedName.lowercased()
let spec = config.tools?[key]
let fallback = config.fallback
let emoji = spec?.emoji ?? fallback?.emoji ?? "🧩"
let title = spec?.title ?? titleFromName(trimmedName)
let label = spec?.label ?? trimmedName
let actionRaw = valueForKeyPath(args, path: "action") as? String
let action = actionRaw?.trimmingCharacters(in: .whitespacesAndNewlines)
let actionSpec = action.flatMap { spec?.actions?[$0] }
let verb = normalizeVerb(actionSpec?.label ?? action)
var detail: String?
if key == "read" {
detail = readDetail(args)
} else if key == "write" || key == "edit" || key == "attach" {
detail = pathDetail(args)
}
let detailKeys = actionSpec?.detailKeys ?? spec?.detailKeys ?? fallback?.detailKeys ?? []
if detail == nil {
detail = firstValue(args, keys: detailKeys)
}
if detail == nil {
detail = meta
}
if let detailValue = detail {
detail = shortenHomeInString(detailValue)
}
return ToolDisplaySummary(
name: trimmedName,
emoji: emoji,
title: title,
label: label,
verb: verb,
detail: detail)
}
private static func loadConfig() -> ToolDisplayConfig {
guard let url = ClawdbotKitResources.bundle.url(forResource: "tool-display", withExtension: "json") else {
return ToolDisplayConfig(version: nil, fallback: nil, tools: nil)
}
do {
let data = try Data(contentsOf: url)
return try JSONDecoder().decode(ToolDisplayConfig.self, from: data)
} catch {
return ToolDisplayConfig(version: nil, fallback: nil, tools: nil)
}
}
private static func titleFromName(_ name: String) -> String {
let cleaned = name.replacingOccurrences(of: "_", with: " ").trimmingCharacters(in: .whitespaces)
guard !cleaned.isEmpty else { return "Tool" }
return cleaned
.split(separator: " ")
.map { part in
let upper = part.uppercased()
if part.count <= 2 && part == upper { return String(part) }
return String(upper.prefix(1)) + String(part.lowercased().dropFirst())
}
.joined(separator: " ")
}
private static func normalizeVerb(_ value: String?) -> String? {
let trimmed = value?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !trimmed.isEmpty else { return nil }
return trimmed.replacingOccurrences(of: "_", with: " ")
}
private static func readDetail(_ args: AnyCodable?) -> String? {
guard let path = valueForKeyPath(args, path: "path") as? String else { return nil }
let offset = valueForKeyPath(args, path: "offset") as? Double
let limit = valueForKeyPath(args, path: "limit") as? Double
if let offset, let limit {
let end = offset + limit
return "\(path):\(Int(offset))-\(Int(end))"
}
return path
}
private static func pathDetail(_ args: AnyCodable?) -> String? {
return valueForKeyPath(args, path: "path") as? String
}
private static func firstValue(_ args: AnyCodable?, keys: [String]) -> String? {
for key in keys {
if let value = valueForKeyPath(args, path: key),
let rendered = renderValue(value)
{
return rendered
}
}
return nil
}
private static func renderValue(_ value: Any) -> String? {
if let str = value as? String {
let trimmed = str.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let first = trimmed.split(whereSeparator: \.isNewline).first.map(String.init) ?? trimmed
if first.count > 160 { return String(first.prefix(157)) + "" }
return first
}
if let num = value as? Int { return String(num) }
if let num = value as? Double { return String(num) }
if let bool = value as? Bool { return bool ? "true" : "false" }
if let array = value as? [Any] {
let items = array.compactMap { renderValue($0) }
guard !items.isEmpty else { return nil }
let preview = items.prefix(3).joined(separator: ", ")
return items.count > 3 ? "\(preview)" : preview
}
if let dict = value as? [String: Any] {
if let label = dict["name"].flatMap({ renderValue($0) }) { return label }
if let label = dict["id"].flatMap({ renderValue($0) }) { return label }
}
return nil
}
private static func valueForKeyPath(_ args: AnyCodable?, path: String) -> Any? {
guard let args else { return nil }
let parts = path.split(separator: ".").map(String.init)
var current: Any? = args.value
for part in parts {
if let dict = current as? [String: AnyCodable] {
current = dict[part]?.value
} else if let dict = current as? [String: Any] {
current = dict[part]
} else {
return nil
}
}
return current
}
private static func shortenHomeInString(_ value: String) -> String {
let home = NSHomeDirectory()
guard !home.isEmpty else { return value }
return value.replacingOccurrences(of: home, with: "~")
}
}