let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

refactor(macos)!: remove clawdis-mac ui; host PeekabooBridge

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-13 23:49:19 +00:00
parent b508f642b2
commit cf3becfb2e
7 changed files with 20 additions and 825 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/macos/Package.swift
View File

@@ -20,7 +20,6 @@ let package = Package(
.package(path: "../shared/ClawdisKit"),
.package(path: "../../Peekaboo/Core/PeekabooCore"),
.package(path: "../../Peekaboo/Core/PeekabooAutomationKit"),
.package(path: "../../Peekaboo/Core/PeekabooVisualizer"),
],
targets: [
.target(
@@ -47,7 +46,6 @@ let package = Package(
.product(name: "Sparkle", package: "Sparkle"),
.product(name: "PeekabooBridge", package: "PeekabooCore"),
.product(name: "PeekabooAutomationKit", package: "PeekabooAutomationKit"),
.product(name: "PeekabooVisualizer", package: "PeekabooVisualizer"),
],
resources: [
.copy("Resources/Clawdis.icns"),
@@ -61,8 +59,6 @@ let package = Package(
dependencies: [
"ClawdisIPC",
"ClawdisProtocol",
.product(name: "PeekabooBridge", package: "PeekabooCore"),
.product(name: "PeekabooAutomationKit", package: "PeekabooAutomationKit"),
],
swiftSettings: [
.enableUpcomingFeature("StrictConcurrency"),

14
apps/macos/Sources/Clawdis/ControlRequestHandler.swift
View File

@@ -8,13 +8,13 @@ enum ControlRequestHandler {
notifier: NotificationManager = NotificationManager(),
logger: Logger = Logger(subsystem: "com.steipete.clawdis", category: "control")) async throws -> Response
{
// Keep `status` responsive even if the main actor is busy.
let paused = UserDefaults.standard.bool(forKey: pauseDefaultsKey)
if paused, case .status = request {
// allow status through
} else if paused {
return Response(ok: false, message: "clawdis paused")
}
// Keep `status` responsive even if the main actor is busy.
let paused = UserDefaults.standard.bool(forKey: pauseDefaultsKey)
if paused, case .status = request {
// allow status through
} else if paused {
return Response(ok: false, message: "clawdis paused")
}
switch request {
case let .notify(title, body, sound, priority, delivery):

2
apps/macos/Sources/Clawdis/GeneralSettings.swift
View File

@@ -60,7 +60,7 @@ struct GeneralSettings: View {
SettingsToggleRow(
title: "Enable Peekaboo Bridge",
subtitle: "Allow signed tools to drive UI automation via `clawdis-mac ui …`.",
subtitle: "Allow signed tools (e.g. `peekaboo`) to drive UI automation via PeekabooBridge.",
binding: self.$state.peekabooBridgeEnabled)
SettingsToggleRow(

173
apps/macos/Sources/Clawdis/PeekabooBridgeHostCoordinator.swift
View File

@@ -3,7 +3,6 @@ import os
import PeekabooAutomationKit
import PeekabooBridge
import PeekabooFoundation
import PeekabooVisualizer
@MainActor
final class PeekabooBridgeHostCoordinator {
@@ -72,16 +71,15 @@ private final class ClawdisPeekabooBridgeServices: PeekabooBridgeServiceProvidin
init() {
let logging = LoggingService(subsystem: "com.steipete.clawdis.peekaboo")
let visualizer = PeekabooVisualizerFeedbackClient(client: .shared)
let feedbackClient: any AutomationFeedbackClient = NoopAutomationFeedbackClient()
let snapshots = InMemorySnapshotManager(options: .init(
snapshotValidityWindow: 600,
maxSnapshots: 50,
deleteArtifactsOnCleanup: false))
let applications = ApplicationService(feedbackClient: visualizer)
let applications = ApplicationService(feedbackClient: feedbackClient)
let captureBase = ScreenCaptureService(loggingService: logging)
let screenCapture = FeedbackScreenCaptureService(base: captureBase, feedbackClient: visualizer)
let screenCapture = ScreenCaptureService(loggingService: logging)
self.permissions = PermissionsService()
self.snapshots = snapshots
@@ -91,165 +89,10 @@ private final class ClawdisPeekabooBridgeServices: PeekabooBridgeServiceProvidin
snapshotManager: snapshots,
loggingService: logging,
searchPolicy: .balanced,
feedbackClient: visualizer)
self.windows = WindowManagementService(applicationService: applications, feedbackClient: visualizer)
self.menu = MenuService(applicationService: applications, feedbackClient: visualizer)
self.dock = DockService(feedbackClient: visualizer)
self.dialogs = DialogService(feedbackClient: visualizer)
}
}
@MainActor
private final class PeekabooVisualizerFeedbackClient: AutomationFeedbackClient {
private let client: VisualizationClient
init(client: VisualizationClient) {
self.client = client
}
func connect() {
self.client.connect()
}
func showClickFeedback(at point: CGPoint, type: ClickType) async -> Bool {
await self.client.showClickFeedback(at: point, type: type)
}
func showTypingFeedback(keys: [String], duration: TimeInterval, cadence: TypingCadence) async -> Bool {
await self.client.showTypingFeedback(keys: keys, duration: duration, cadence: cadence)
}
func showScrollFeedback(at point: CGPoint, direction: ScrollDirection, amount: Int) async -> Bool {
await self.client.showScrollFeedback(at: point, direction: direction, amount: amount)
}
func showHotkeyDisplay(keys: [String], duration: TimeInterval) async -> Bool {
await self.client.showHotkeyDisplay(keys: keys, duration: duration)
}
func showSwipeGesture(from: CGPoint, to: CGPoint, duration: TimeInterval) async -> Bool {
await self.client.showSwipeGesture(from: from, to: to, duration: duration)
}
func showMouseMovement(from: CGPoint, to: CGPoint, duration: TimeInterval) async -> Bool {
await self.client.showMouseMovement(from: from, to: to, duration: duration)
}
func showWindowOperation(_ kind: WindowOperationKind, windowRect: CGRect, duration: TimeInterval) async -> Bool {
let mapped: WindowOperation = switch kind {
case .close: .close
case .minimize: .minimize
case .maximize: .maximize
case .move: .move
case .resize: .resize
case .setBounds: .setBounds
case .focus: .focus
}
return await self.client.showWindowOperation(mapped, windowRect: windowRect, duration: duration)
}
func showDialogInteraction(
element: DialogElementType,
elementRect: CGRect,
action: DialogActionType) async -> Bool
{
await self.client.showDialogInteraction(element: element, elementRect: elementRect, action: action)
}
func showMenuNavigation(menuPath: [String]) async -> Bool {
await self.client.showMenuNavigation(menuPath: menuPath)
}
func showSpaceSwitch(from: Int, to: Int, direction: SpaceSwitchDirection) async -> Bool {
let mapped: SpaceDirection = direction == .left ? .left : .right
return await self.client.showSpaceSwitch(from: from, to: to, direction: mapped)
}
func showAppLaunch(appName: String, iconPath: String?) async -> Bool {
await self.client.showAppLaunch(appName: appName, iconPath: iconPath)
}
func showAppQuit(appName: String, iconPath: String?) async -> Bool {
await self.client.showAppQuit(appName: appName, iconPath: iconPath)
}
func showScreenshotFlash(in rect: CGRect) async -> Bool {
await self.client.showScreenshotFlash(in: rect)
}
func showWatchCapture(in rect: CGRect) async -> Bool {
await self.client.showWatchCapture(in: rect)
}
}
@MainActor
private final class FeedbackScreenCaptureService: ScreenCaptureServiceProtocol {
private let base: any ScreenCaptureServiceProtocol
private let feedbackClient: any AutomationFeedbackClient
init(base: any ScreenCaptureServiceProtocol, feedbackClient: any AutomationFeedbackClient) {
self.base = base
self.feedbackClient = feedbackClient
}
func captureScreen(
displayIndex: Int?,
visualizerMode: CaptureVisualizerMode,
scale: CaptureScalePreference) async throws -> CaptureResult
{
let result = try await self.base.captureScreen(
displayIndex: displayIndex,
visualizerMode: visualizerMode,
scale: scale)
await self.showCaptureFeedback(mode: visualizerMode, rect: result.metadata.displayInfo?.bounds)
return result
}
func captureWindow(
appIdentifier: String,
windowIndex: Int?,
visualizerMode: CaptureVisualizerMode,
scale: CaptureScalePreference) async throws -> CaptureResult
{
let result = try await self.base.captureWindow(
appIdentifier: appIdentifier,
windowIndex: windowIndex,
visualizerMode: visualizerMode,
scale: scale)
await self.showCaptureFeedback(mode: visualizerMode, rect: result.metadata.windowInfo?.bounds)
return result
}
func captureFrontmost(
visualizerMode: CaptureVisualizerMode,
scale: CaptureScalePreference) async throws -> CaptureResult
{
let result = try await self.base.captureFrontmost(visualizerMode: visualizerMode, scale: scale)
await self.showCaptureFeedback(mode: visualizerMode, rect: result.metadata.windowInfo?.bounds)
return result
}
func captureArea(
_ rect: CGRect,
visualizerMode: CaptureVisualizerMode,
scale: CaptureScalePreference) async throws -> CaptureResult
{
let result = try await self.base.captureArea(rect, visualizerMode: visualizerMode, scale: scale)
await self.showCaptureFeedback(mode: visualizerMode, rect: rect)
return result
}
func hasScreenRecordingPermission() async -> Bool {
await self.base.hasScreenRecordingPermission()
}
private func showCaptureFeedback(mode: CaptureVisualizerMode, rect: CGRect?) async {
guard let rect else { return }
switch mode {
case .screenshotFlash:
_ = await self.feedbackClient.showScreenshotFlash(in: rect)
case .watchCapture:
_ = await self.feedbackClient.showWatchCapture(in: rect)
}
feedbackClient: feedbackClient)
self.windows = WindowManagementService(applicationService: applications, feedbackClient: feedbackClient)
self.menu = MenuService(applicationService: applications, feedbackClient: feedbackClient)
self.dock = DockService(feedbackClient: feedbackClient)
self.dialogs = DialogService(feedbackClient: feedbackClient)
}
}

17
apps/macos/Sources/ClawdisCLI/ClawdisCLI.swift
View File

@@ -15,11 +15,6 @@ struct ClawdisCLI {
exit(code)
}
if args.first == "ui" {
let code = try await UICLI.run(args: Array(args.dropFirst()), jsonOutput: jsonOutput)
exit(code)
}
let parsed = try parseCommandLine(args: args)
let response = try await send(request: parsed.request)
@@ -386,14 +381,6 @@ struct ClawdisCLI {
[--cap <notifications|accessibility|screenRecording|microphone|speechRecognition>]
[--interactive]
UI:
clawdis-mac ui screenshot [...]
clawdis-mac ui see [...]
clawdis-mac ui click ...
clawdis-mac ui type ...
clawdis-mac ui wait ...
clawdis-mac ui --help
Shell:
clawdis-mac run [--cwd <path>] [--env KEY=VAL] [--timeout <sec>]
[--needs-screen-recording] <command ...>
@@ -422,6 +409,10 @@ struct ClawdisCLI {
Browser (clawd):
clawdis-mac browser status|start|stop|tabs|open|focus|close|screenshot|eval|query|dom|snapshot
UI Automation (Peekaboo):
Install and use the `peekaboo` CLI; it will connect to Peekaboo.app (preferred) or Clawdis.app
(fallback) via PeekabooBridge. See `docs/mac/peekaboo.md`.
Browser notes:
- Uses clawd’s dedicated Chrome/Chromium profile (separate user-data dir).
- Talks to the gateway’s loopback browser-control server (config: ~/.clawdis/clawdis.json).

592
apps/macos/Sources/ClawdisCLI/UICLI.swift
View File

@@ -1,592 +0,0 @@
import Darwin
import Foundation
import PeekabooAutomationKit
import PeekabooBridge
import PeekabooFoundation
enum UICLI {
static func run(args: [String], jsonOutput: Bool) async throws -> Int32 {
var args = args
guard let sub = args.first else {
self.printHelp()
return 0
}
args.removeFirst()
if sub == "--help" || sub == "-h" || sub == "help" {
self.printHelp()
return 0
}
let context = try await self.resolveContext()
switch sub {
case "permissions":
return try await self.runPermissions(args: args, jsonOutput: jsonOutput, context: context)
case "frontmost":
return try await self.runFrontmost(args: args, jsonOutput: jsonOutput, context: context)
case "apps":
return try await self.runApps(args: args, jsonOutput: jsonOutput, context: context)
case "windows":
return try await self.runWindows(args: args, jsonOutput: jsonOutput, context: context)
case "screenshot":
return try await self.runScreenshot(args: args, jsonOutput: jsonOutput, context: context)
case "see":
return try await self.runSee(args: args, jsonOutput: jsonOutput, context: context)
case "click":
return try await self.runClick(args: args, jsonOutput: jsonOutput, context: context)
case "type":
return try await self.runType(args: args, jsonOutput: jsonOutput, context: context)
case "wait":
return try await self.runWait(args: args, jsonOutput: jsonOutput, context: context)
default:
self.printHelp()
return 1
}
}
// MARK: - Context
private struct Context {
let client: PeekabooBridgeClient
let hostDescription: String
}
private static func resolveContext() async throws -> Context {
let explicitSocket = ProcessInfo.processInfo.environment["PEEKABOO_BRIDGE_SOCKET"]
let candidates: [String] = if let explicitSocket, !explicitSocket.isEmpty {
[explicitSocket]
} else {
[
PeekabooBridgeConstants.peekabooSocketPath,
PeekabooBridgeConstants.clawdisSocketPath,
]
}
let identity = PeekabooBridgeClientIdentity(
bundleIdentifier: Bundle.main.bundleIdentifier,
teamIdentifier: nil,
processIdentifier: getpid(),
hostname: Host.current().name)
for socketPath in candidates {
let client = PeekabooBridgeClient(socketPath: socketPath, requestTimeoutSec: 10)
do {
let handshake = try await client.handshake(client: identity, requestedHost: nil)
return Context(
client: client,
hostDescription: "\(handshake.hostKind.rawValue) via \(socketPath)")
} catch let envelope as PeekabooBridgeErrorEnvelope {
if envelope.code == .unauthorizedClient {
throw envelope
}
} catch {
continue
}
}
throw NSError(domain: "clawdis.ui", code: 1, userInfo: [
NSLocalizedDescriptionKey: "No PeekabooBridge host reachable (run Peekaboo.app or Clawdis.app).",
])
}
// MARK: - Commands
private static func runPermissions(args: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
let sub = args.first ?? "status"
if sub != "status", sub != "--help", sub != "-h", sub != "help" {
self.printHelp()
return 1
}
let status = try await context.client.permissionsStatus()
if jsonOutput {
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
"result": self.toJSONObject(status),
])
} else {
FileHandle.standardOutput.write(Data((self.formatPermissions(status) + "\n").utf8))
}
return 0
}
private static func runFrontmost(args _: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
let app = try await context.client.getFrontmostApplication()
let window = try await context.client.getFocusedWindow()
if jsonOutput {
let windowObject: Any = if let window {
try self.toJSONObject(window)
} else {
NSNull()
}
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
"app": self.toJSONObject(app),
"window": windowObject,
])
} else {
let bundle = app.bundleIdentifier ?? "<unknown>"
let line = "\(bundle) (pid \(app.processIdentifier))"
FileHandle.standardOutput.write(Data((line + "\n").utf8))
if let window {
FileHandle.standardOutput.write(Data("window \(window.windowID): \(window.title)\n".utf8))
}
}
return 0
}
private static func runApps(args _: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
let apps = try await context.client.listApplications()
if jsonOutput {
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
"result": self.toJSONObject(apps),
])
} else {
for app in apps {
let bundle = app.bundleIdentifier ?? "<unknown>"
FileHandle.standardOutput.write(Data("\(bundle)\t\(app.name)\n".utf8))
}
}
return 0
}
private static func runWindows(args: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
var args = args
var bundleId: String?
while !args.isEmpty {
switch args.removeFirst() {
case "--bundle-id":
bundleId = args.popFirst()
case "--help", "-h", "help":
self.printHelp()
return 0
default:
break
}
}
let target: WindowTarget = if let bundleId, !bundleId.isEmpty { .application(bundleId) } else { .frontmost }
let windows = try await context.client.listWindows(target: target)
if jsonOutput {
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
"result": self.toJSONObject(windows),
])
} else {
for window in windows {
FileHandle.standardOutput.write(Data("\(window.windowID)\t\(window.title)\n".utf8))
}
}
return 0
}
private static func runScreenshot(args: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
var args = args
var displayIndex: Int?
var bundleId: String?
var windowIndex: Int?
var mode: CaptureVisualizerMode = .screenshotFlash
var scale: CaptureScalePreference = .logical1x
while !args.isEmpty {
let arg = args.removeFirst()
switch arg {
case "--screen-index":
displayIndex = args.popFirst().flatMap(Int.init)
case "--bundle-id":
bundleId = args.popFirst()
case "--window-index":
windowIndex = args.popFirst().flatMap(Int.init)
case "--watch":
mode = .watchCapture
case "--scale":
let raw = args.popFirst()?.lowercased()
if raw == "native" { scale = .native }
if raw == "1x" || raw == "logical" || raw == "logical1x" { scale = .logical1x }
case "--help", "-h", "help":
self.printHelp()
return 0
default:
break
}
}
let capture: CaptureResult = if let bundleId, !bundleId.isEmpty {
try await context.client.captureWindow(
appIdentifier: bundleId,
windowIndex: windowIndex,
visualizerMode: mode,
scale: scale)
} else if displayIndex != nil {
try await context.client.captureScreen(
displayIndex: displayIndex,
visualizerMode: mode,
scale: scale)
} else {
try await context.client.captureFrontmost(visualizerMode: mode, scale: scale)
}
let path = try self.writeTempPNG(capture.imageData)
if jsonOutput {
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
"path": path,
"metadata": self.toJSONObject(capture.metadata),
"warning": capture.warning ?? "",
])
} else {
FileHandle.standardOutput.write(Data((path + "\n").utf8))
}
return 0
}
private static func runSee(args: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
var args = args
var bundleId: String?
var windowIndex: Int?
var snapshotId: String?
while !args.isEmpty {
let arg = args.removeFirst()
switch arg {
case "--bundle-id":
bundleId = args.popFirst()
case "--window-index":
windowIndex = args.popFirst().flatMap(Int.init)
case "--snapshot-id":
snapshotId = args.popFirst()
case "--help", "-h", "help":
self.printHelp()
return 0
default:
break
}
}
let capture: CaptureResult
if let bundleId, !bundleId.isEmpty {
capture = try await context.client.captureWindow(
appIdentifier: bundleId,
windowIndex: windowIndex,
visualizerMode: .screenshotFlash,
scale: .logical1x)
} else {
capture = try await context.client.captureFrontmost(visualizerMode: .screenshotFlash, scale: .logical1x)
bundleId = capture.metadata.applicationInfo?.bundleIdentifier
}
let resolvedSnapshotId: String = if let snapshotId, !snapshotId.isEmpty {
snapshotId
} else if let bundleId, !bundleId.isEmpty, let existing = try? await context.client
.getMostRecentSnapshot(applicationBundleId: bundleId)
{
existing
} else {
try await context.client.createSnapshot()
}
let screenshotPath = try self.writeTempPNG(capture.imageData)
try await context.client.storeScreenshot(
snapshotId: resolvedSnapshotId,
screenshotPath: screenshotPath,
applicationBundleId: bundleId,
applicationProcessId: capture.metadata.applicationInfo?.processIdentifier,
applicationName: capture.metadata.applicationInfo?.name,
windowTitle: capture.metadata.windowInfo?.title,
windowBounds: capture.metadata.windowInfo?.bounds)
let windowContext = WindowContext(
applicationName: capture.metadata.applicationInfo?.name,
windowTitle: capture.metadata.windowInfo?.title,
windowBounds: capture.metadata.windowInfo?.bounds)
let detection = try await context.client.detectElements(
in: capture.imageData,
snapshotId: resolvedSnapshotId,
windowContext: windowContext)
try await context.client.storeDetectionResult(snapshotId: resolvedSnapshotId, result: detection)
if jsonOutput {
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
"snapshotId": resolvedSnapshotId,
"screenshotPath": screenshotPath,
"result": self.toJSONObject(detection),
])
} else {
FileHandle.standardOutput.write(Data((screenshotPath + "\n").utf8))
for el in detection.elements.all {
let b = el.bounds
let label = (el.label ?? el.value ?? "").replacingOccurrences(of: "\n", with: " ")
let coords = "\(Int(b.origin.x)),\(Int(b.origin.y))"
let size = "\(Int(b.size.width))x\(Int(b.size.height))"
let line = "\(el.id)\t\(el.type)\t\(coords) \(size)\t\(label)\n"
FileHandle.standardOutput.write(Data(line.utf8))
}
}
return 0
}
private static func runClick(args: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
var args = args
var bundleId: String?
var snapshotId: String?
var on: String?
var clickType: ClickType = .single
while !args.isEmpty {
let arg = args.removeFirst()
switch arg {
case "--bundle-id":
bundleId = args.popFirst()
case "--snapshot-id":
snapshotId = args.popFirst()
case "--on":
on = args.popFirst()
case "--double":
clickType = .double
case "--right":
clickType = .right
case "--help", "-h", "help":
self.printHelp()
return 0
default:
break
}
}
guard let on, !on.isEmpty else {
throw NSError(domain: "clawdis.ui", code: 2, userInfo: [
NSLocalizedDescriptionKey: "Missing --on <elementId> (run `clawdis-mac ui see` first).",
])
}
let effectiveSnapshotId = try await self.resolveImplicitSnapshotId(
snapshotId: snapshotId,
bundleId: bundleId,
client: context.client)
try await context.client.click(target: .elementId(on), clickType: clickType, snapshotId: effectiveSnapshotId)
if jsonOutput {
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
])
}
return 0
}
private static func runType(args: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
var args = args
var bundleId: String?
var snapshotId: String?
var into: String?
var clearExisting = false
var delayMs = 20
var textParts: [String] = []
while !args.isEmpty {
let arg = args.removeFirst()
switch arg {
case "--bundle-id":
bundleId = args.popFirst()
case "--snapshot-id":
snapshotId = args.popFirst()
case "--into":
into = args.popFirst()
case "--clear":
clearExisting = true
case "--delay-ms":
delayMs = args.popFirst().flatMap(Int.init) ?? delayMs
case "--text":
if let next = args.popFirst() {
textParts.append(next)
}
case "--help", "-h", "help":
self.printHelp()
return 0
default:
textParts.append(arg)
}
}
let text = textParts.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines)
guard !text.isEmpty else {
throw NSError(domain: "clawdis.ui", code: 3, userInfo: [
NSLocalizedDescriptionKey: "Missing text (use --text <value>).",
])
}
let effectiveSnapshotId = try await self.resolveImplicitSnapshotId(
snapshotId: snapshotId,
bundleId: bundleId,
client: context.client)
try await context.client.type(
text: text,
target: into,
clearExisting: clearExisting,
typingDelay: delayMs,
snapshotId: effectiveSnapshotId)
if jsonOutput {
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
])
}
return 0
}
private static func runWait(args: [String], jsonOutput: Bool, context: Context) async throws -> Int32 {
var args = args
var bundleId: String?
var snapshotId: String?
var on: String?
var timeoutSec: Double = 10
while !args.isEmpty {
let arg = args.removeFirst()
switch arg {
case "--bundle-id":
bundleId = args.popFirst()
case "--snapshot-id":
snapshotId = args.popFirst()
case "--on":
on = args.popFirst()
case "--timeout":
timeoutSec = args.popFirst().flatMap(Double.init) ?? timeoutSec
case "--help", "-h", "help":
self.printHelp()
return 0
default:
break
}
}
guard let on, !on.isEmpty else {
throw NSError(domain: "clawdis.ui", code: 4, userInfo: [
NSLocalizedDescriptionKey: "Missing --on <elementId>.",
])
}
let effectiveSnapshotId = try await self.resolveImplicitSnapshotId(
snapshotId: snapshotId,
bundleId: bundleId,
client: context.client)
let result = try await context.client.waitForElement(
target: .elementId(on),
timeout: timeoutSec,
snapshotId: effectiveSnapshotId)
if jsonOutput {
try self.writeJSON([
"ok": true,
"host": context.hostDescription,
"result": self.toJSONObject(result),
])
} else {
FileHandle.standardOutput.write(Data((result.found ? "found\n" : "not found\n").utf8))
}
return result.found ? 0 : 1
}
private static func resolveImplicitSnapshotId(
snapshotId: String?,
bundleId: String?,
client: PeekabooBridgeClient) async throws -> String
{
if let snapshotId, !snapshotId.isEmpty { return snapshotId }
let resolvedBundle: String? = if let bundleId, !bundleId.isEmpty {
bundleId
} else {
try await client.getFrontmostApplication().bundleIdentifier
}
guard let resolvedBundle, !resolvedBundle.isEmpty else {
throw NSError(domain: "clawdis.ui", code: 5, userInfo: [
NSLocalizedDescriptionKey: "Could not determine bundle id for implicit snapshot.",
])
}
do {
return try await client.getMostRecentSnapshot(applicationBundleId: resolvedBundle)
} catch {
let command = "clawdis-mac ui see --bundle-id \(resolvedBundle)"
let help = "No recent snapshot for \(resolvedBundle). Run `\(command)` first."
throw NSError(domain: "clawdis.ui", code: 6, userInfo: [
NSLocalizedDescriptionKey: help,
])
}
}
// MARK: - IO helpers
private static func writeTempPNG(_ data: Data) throws -> String {
let dir = FileManager.default.temporaryDirectory
let formatter = ISO8601DateFormatter()
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
let stamp = formatter.string(from: Date()).replacingOccurrences(of: ":", with: "-")
let url = dir.appendingPathComponent("clawdis-ui-\(stamp).png")
try data.write(to: url, options: [.atomic])
return url.path
}
private static func formatPermissions(_ status: PermissionsStatus) -> String {
let sr = status.screenRecording ? "screen-recording=ok" : "screen-recording=missing"
let ax = status.accessibility ? "accessibility=ok" : "accessibility=missing"
let ascr = status.appleScript ? "applescript=ok" : "applescript=missing"
return "\(sr) \(ax) \(ascr)"
}
private static func toJSONObject(_ value: some Encodable) throws -> Any {
let encoder = JSONEncoder()
encoder.dateEncodingStrategy = .iso8601
let data = try encoder.encode(value)
return try JSONSerialization.jsonObject(with: data)
}
private static func writeJSON(_ obj: [String: Any]) throws {
let data = try JSONSerialization.data(withJSONObject: obj, options: [.prettyPrinted])
FileHandle.standardOutput.write(data)
FileHandle.standardOutput.write(Data([0x0A]))
}
private static func printHelp() {
let usage = """
clawdis-mac ui — UI automation via PeekabooBridge
Usage:
clawdis-mac [--json] ui <command> ...
Commands:
permissions status
frontmost
apps
windows [--bundle-id <id>]
screenshot [--screen-index <n>] [--bundle-id <id>] [--window-index <n>] [--watch] [--scale native|1x]
see [--bundle-id <id>] [--window-index <n>] [--snapshot-id <id>]
click --on <elementId> [--bundle-id <id>] [--snapshot-id <id>] [--double|--right]
type --text <value> [--into <elementId>] [--bundle-id <id>] [--snapshot-id <id>] [--clear] [--delay-ms <n>]
wait --on <elementId> [--bundle-id <id>] [--snapshot-id <id>] [--timeout <sec>]
Notes:
- Prefers Peekaboo.app’s bridge, then Clawdis.app’s bridge.
- Default timeout is 10 seconds per action.
"""
FileHandle.standardError.write(Data((usage + "\n").utf8))
}
}

43
src/cli/program.ts
View File

@@ -22,7 +22,6 @@ import { sendCommand } from "../commands/send.js";
import { sessionsCommand } from "../commands/sessions.js";
import { statusCommand } from "../commands/status.js";
import { danger, info, setVerbose } from "../globals.js";
import { runClawdisMac } from "../infra/clawdis-mac.js";
import { loginWeb, logoutWeb } from "../provider-web.js";
import { defaultRuntime } from "../runtime.js";
import { VERSION } from "../version.js";
@@ -230,48 +229,6 @@ Examples:
registerNodesCli(program);
registerCronCli(program);
program
.command("ui")
.description("macOS UI automation via Clawdis.app (PeekabooBridge)")
.option("--json", "Output JSON (passthrough from clawdis-mac)", false)
.allowUnknownOption(true)
.argument(
"[uiArgs...]",
"Args passed through to: clawdis-mac ui <command> ...",
)
.addHelpText(
"after",
`
Examples:
clawdis ui permissions status
clawdis ui frontmost
clawdis ui screenshot
clawdis ui see --bundle-id com.apple.Safari
clawdis ui click --bundle-id com.apple.Safari --on B1
clawdis ui --json see --bundle-id com.apple.Safari
`,
)
.action(async (_unused: string[], opts, cmd) => {
try {
const raw = (cmd.parent?.rawArgs ?? []).map((a: unknown) => String(a));
const idx = raw.indexOf("ui");
const tail = idx >= 0 ? raw.slice(idx + 1) : [];
const forwarded =
tail.length > 0 && tail[0] === "--json" ? tail.slice(1) : tail;
const res = await runClawdisMac(["ui", ...forwarded], {
json: Boolean(opts.json),
timeoutMs: 45_000,
});
if (res.stdout) process.stdout.write(res.stdout);
if (res.stderr) process.stderr.write(res.stderr);
defaultRuntime.exit(res.code ?? 1);
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
program
.command("status")
.description("Show web session health and recent session recipients")