Canvas: fix A2UI click actions
This commit is contained in:
@@ -1,11 +1,14 @@
|
||||
import AppKit
|
||||
import ClawdisIPC
|
||||
import Foundation
|
||||
import OSLog
|
||||
|
||||
@MainActor
|
||||
final class CanvasManager {
|
||||
static let shared = CanvasManager()
|
||||
|
||||
private static let logger = Logger(subsystem: "com.steipete.clawdis", category: "CanvasManager")
|
||||
|
||||
private var panelController: CanvasWindowController?
|
||||
private var panelSessionKey: String?
|
||||
|
||||
@@ -24,6 +27,8 @@ final class CanvasManager {
|
||||
}
|
||||
|
||||
func showDetailed(sessionKey: String, target: String? = nil, placement: CanvasPlacement? = nil) throws -> CanvasShowResult {
|
||||
Self.logger.debug(
|
||||
"showDetailed start session=\(sessionKey, privacy: .public) target=\(target ?? "", privacy: .public) placement=\(placement != nil)")
|
||||
let anchorProvider = self.defaultAnchorProvider ?? Self.mouseAnchorProvider
|
||||
let session = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let normalizedTarget = target?
|
||||
@@ -31,6 +36,7 @@ final class CanvasManager {
|
||||
.nonEmpty
|
||||
|
||||
if let controller = self.panelController, self.panelSessionKey == session {
|
||||
Self.logger.debug("showDetailed reuse existing session=\(session, privacy: .public)")
|
||||
controller.onVisibilityChanged = { [weak self] visible in
|
||||
self?.onPanelVisibilityChanged?(visible)
|
||||
}
|
||||
@@ -54,15 +60,19 @@ final class CanvasManager {
|
||||
url: nil)
|
||||
}
|
||||
|
||||
Self.logger.debug("showDetailed creating new session=\(session, privacy: .public)")
|
||||
self.panelController?.close()
|
||||
self.panelController = nil
|
||||
self.panelSessionKey = nil
|
||||
|
||||
Self.logger.debug("showDetailed ensure canvas root dir")
|
||||
try FileManager.default.createDirectory(at: Self.canvasRoot, withIntermediateDirectories: true)
|
||||
Self.logger.debug("showDetailed init CanvasWindowController")
|
||||
let controller = try CanvasWindowController(
|
||||
sessionKey: session,
|
||||
root: Self.canvasRoot,
|
||||
presentation: .panel(anchorProvider: anchorProvider))
|
||||
Self.logger.debug("showDetailed CanvasWindowController init done")
|
||||
controller.onVisibilityChanged = { [weak self] visible in
|
||||
self?.onPanelVisibilityChanged?(visible)
|
||||
}
|
||||
@@ -72,7 +82,9 @@ final class CanvasManager {
|
||||
|
||||
// New session: default to "/" so the user sees either the welcome page or `index.html`.
|
||||
let effectiveTarget = normalizedTarget ?? "/"
|
||||
Self.logger.debug("showDetailed showCanvas effectiveTarget=\(effectiveTarget, privacy: .public)")
|
||||
controller.showCanvas(path: effectiveTarget)
|
||||
Self.logger.debug("showDetailed showCanvas done")
|
||||
|
||||
return self.makeShowResult(
|
||||
directory: controller.directoryPath,
|
||||
|
||||
@@ -50,18 +50,85 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
|
||||
self.root = root
|
||||
self.presentation = presentation
|
||||
|
||||
canvasWindowLogger.debug("CanvasWindowController init start session=\(sessionKey, privacy: .public)")
|
||||
let safeSessionKey = CanvasWindowController.sanitizeSessionKey(sessionKey)
|
||||
canvasWindowLogger.debug("CanvasWindowController init sanitized session=\(safeSessionKey, privacy: .public)")
|
||||
self.sessionDir = root.appendingPathComponent(safeSessionKey, isDirectory: true)
|
||||
try FileManager.default.createDirectory(at: self.sessionDir, withIntermediateDirectories: true)
|
||||
canvasWindowLogger.debug("CanvasWindowController init session dir ready")
|
||||
|
||||
self.schemeHandler = CanvasSchemeHandler(root: root)
|
||||
canvasWindowLogger.debug("CanvasWindowController init scheme handler ready")
|
||||
|
||||
let config = WKWebViewConfiguration()
|
||||
config.userContentController = WKUserContentController()
|
||||
config.preferences.isElementFullscreenEnabled = true
|
||||
config.preferences.setValue(true, forKey: "developerExtrasEnabled")
|
||||
canvasWindowLogger.debug("CanvasWindowController init config ready")
|
||||
config.setURLSchemeHandler(self.schemeHandler, forURLScheme: CanvasScheme.scheme)
|
||||
canvasWindowLogger.debug("CanvasWindowController init scheme handler installed")
|
||||
|
||||
// Bridge A2UI "a2uiaction" DOM events back into the native agent loop.
|
||||
//
|
||||
// Prefer WKScriptMessageHandler when WebKit exposes it, otherwise fall back to an unattended deep link
|
||||
// (includes the app-generated key so it won't prompt).
|
||||
canvasWindowLogger.debug("CanvasWindowController init building A2UI bridge script")
|
||||
let deepLinkKey = DeepLinkHandler.currentCanvasKey()
|
||||
let injectedSessionKey = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "main"
|
||||
let bridgeScript = """
|
||||
(() => {
|
||||
try {
|
||||
if (location.protocol !== '\(CanvasScheme.scheme):') return;
|
||||
if (globalThis.__clawdisA2UIBridgeInstalled) return;
|
||||
globalThis.__clawdisA2UIBridgeInstalled = true;
|
||||
|
||||
const deepLinkKey = \(Self.jsStringLiteral(deepLinkKey));
|
||||
const sessionKey = \(Self.jsStringLiteral(injectedSessionKey));
|
||||
|
||||
globalThis.addEventListener('a2uiaction', (evt) => {
|
||||
try {
|
||||
const payload = evt?.detail ?? evt?.payload ?? null;
|
||||
if (!payload || payload.eventType !== 'a2ui.action') return;
|
||||
|
||||
const action = payload.action ?? null;
|
||||
const name = action?.name ?? '';
|
||||
if (!name) return;
|
||||
|
||||
const context = Array.isArray(action?.context) ? action.context : [];
|
||||
const userAction = {
|
||||
name,
|
||||
surfaceId: payload.surfaceId ?? 'main',
|
||||
sourceComponentId: payload.sourceComponentId ?? '',
|
||||
dataContextPath: payload.dataContextPath ?? '',
|
||||
timestamp: new Date().toISOString(),
|
||||
...(context.length ? { context } : {}),
|
||||
};
|
||||
|
||||
const handler = globalThis.webkit?.messageHandlers?.clawdisCanvasA2UIAction;
|
||||
if (handler?.postMessage) {
|
||||
handler.postMessage({ userAction });
|
||||
return;
|
||||
}
|
||||
|
||||
const message = 'A2UI action: ' + name + '\\n\\n```json\\n' + JSON.stringify(userAction, null, 2) + '\\n```';
|
||||
const params = new URLSearchParams();
|
||||
params.set('message', message);
|
||||
params.set('sessionKey', sessionKey);
|
||||
params.set('thinking', 'low');
|
||||
params.set('deliver', 'false');
|
||||
params.set('channel', 'last');
|
||||
params.set('key', deepLinkKey);
|
||||
location.href = 'clawdis://agent?' + params.toString();
|
||||
} catch {}
|
||||
}, true);
|
||||
} catch {}
|
||||
})();
|
||||
"""
|
||||
config.userContentController.addUserScript(
|
||||
WKUserScript(source: bridgeScript, injectionTime: .atDocumentStart, forMainFrameOnly: true))
|
||||
canvasWindowLogger.debug("CanvasWindowController init A2UI bridge installed")
|
||||
|
||||
canvasWindowLogger.debug("CanvasWindowController init creating WKWebView")
|
||||
self.webView = WKWebView(frame: .zero, configuration: config)
|
||||
self.webView.setValue(false, forKey: "drawsBackground")
|
||||
|
||||
@@ -93,6 +160,7 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
|
||||
|
||||
self.container = HoverChromeContainerView(containing: self.webView)
|
||||
let window = Self.makeWindow(for: presentation, contentView: self.container)
|
||||
canvasWindowLogger.debug("CanvasWindowController init makeWindow done")
|
||||
super.init(window: window)
|
||||
|
||||
let handler = CanvasA2UIActionMessageHandler(sessionKey: sessionKey)
|
||||
@@ -106,6 +174,7 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
|
||||
}
|
||||
|
||||
self.watcher.start()
|
||||
canvasWindowLogger.debug("CanvasWindowController init done")
|
||||
}
|
||||
|
||||
@available(*, unavailable)
|
||||
@@ -409,6 +478,17 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
|
||||
}
|
||||
let scheme = url.scheme?.lowercased()
|
||||
|
||||
// Deep links: allow local Canvas content to invoke the agent without bouncing through NSWorkspace.
|
||||
if scheme == "clawdis" {
|
||||
if self.webView.url?.scheme == CanvasScheme.scheme {
|
||||
Task { await DeepLinkHandler.shared.handle(url: url) }
|
||||
} else {
|
||||
canvasWindowLogger.debug("ignoring deep link from non-canvas page \(url.absoluteString, privacy: .public)")
|
||||
}
|
||||
decisionHandler(.cancel)
|
||||
return
|
||||
}
|
||||
|
||||
// Keep web content inside the panel when reasonable.
|
||||
// `about:blank` and friends are common internal navigations for WKWebView; never send them to NSWorkspace.
|
||||
if scheme == CanvasScheme.scheme
|
||||
@@ -466,6 +546,11 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
|
||||
return String(scalars)
|
||||
}
|
||||
|
||||
private static func jsStringLiteral(_ value: String) -> String {
|
||||
let data = try? JSONEncoder().encode(value)
|
||||
return data.flatMap { String(data: $0, encoding: .utf8) } ?? "\"\""
|
||||
}
|
||||
|
||||
private static func storedFrameDefaultsKey(sessionKey: String) -> String {
|
||||
"clawdis.canvas.frame.\(self.sanitizeSessionKey(sessionKey))"
|
||||
}
|
||||
@@ -504,9 +589,6 @@ private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHan
|
||||
webView.url?.scheme == CanvasScheme.scheme
|
||||
else { return }
|
||||
|
||||
let path = webView.url?.path ?? ""
|
||||
guard path == "/" || path.isEmpty || path.hasPrefix("/__clawdis__/a2ui") else { return }
|
||||
|
||||
let body: [String: Any] = {
|
||||
if let dict = message.body as? [String: Any] { return dict }
|
||||
if let dict = message.body as? [AnyHashable: Any] {
|
||||
|
||||
@@ -113,9 +113,7 @@ enum ControlRequestHandler {
|
||||
priority: request.priority)
|
||||
return ok ? Response(ok: true) : Response(ok: false, message: "notification not authorized")
|
||||
case .overlay:
|
||||
await MainActor.run {
|
||||
NotifyOverlayController.shared.present(title: request.title, body: request.body)
|
||||
}
|
||||
await NotifyOverlayController.shared.present(title: request.title, body: request.body)
|
||||
return Response(ok: true)
|
||||
case .auto:
|
||||
let ok = await notifier.send(
|
||||
@@ -124,9 +122,7 @@ enum ControlRequestHandler {
|
||||
sound: chosenSound,
|
||||
priority: request.priority)
|
||||
if ok { return Response(ok: true) }
|
||||
await MainActor.run {
|
||||
NotifyOverlayController.shared.present(title: request.title, body: request.body)
|
||||
}
|
||||
await NotifyOverlayController.shared.present(title: request.title, body: request.body)
|
||||
return Response(ok: true, message: "notification not authorized; used overlay")
|
||||
}
|
||||
}
|
||||
@@ -194,28 +190,36 @@ enum ControlRequestHandler {
|
||||
placement: CanvasPlacement?) async -> Response
|
||||
{
|
||||
guard self.canvasEnabled() else { return Response(ok: false, message: "Canvas disabled by user") }
|
||||
let logger = Logger(subsystem: "com.steipete.clawdis", category: "CanvasControl")
|
||||
logger.info("canvas show start session=\(session, privacy: .public) path=\(path ?? "", privacy: .public)")
|
||||
do {
|
||||
let res = try await MainActor.run {
|
||||
try CanvasManager.shared.showDetailed(sessionKey: session, target: path, placement: placement)
|
||||
}
|
||||
logger.info("canvas show awaiting CanvasManager")
|
||||
let res = try await CanvasManager.shared.showDetailed(sessionKey: session, target: path, placement: placement)
|
||||
logger.info("canvas show done dir=\(res.directory, privacy: .public) status=\(String(describing: res.status), privacy: .public)")
|
||||
let payload = try? JSONEncoder().encode(res)
|
||||
return Response(ok: true, message: res.directory, payload: payload)
|
||||
} catch {
|
||||
logger.error("canvas show failed \(error.localizedDescription, privacy: .public)")
|
||||
return Response(ok: false, message: error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private static func handleCanvasHide(session: String) async -> Response {
|
||||
await MainActor.run { CanvasManager.shared.hide(sessionKey: session) }
|
||||
await CanvasManager.shared.hide(sessionKey: session)
|
||||
return Response(ok: true)
|
||||
}
|
||||
|
||||
private static func handleCanvasEval(session: String, javaScript: String) async -> Response {
|
||||
guard self.canvasEnabled() else { return Response(ok: false, message: "Canvas disabled by user") }
|
||||
let logger = Logger(subsystem: "com.steipete.clawdis", category: "CanvasControl")
|
||||
logger.info("canvas eval start session=\(session, privacy: .public) bytes=\(javaScript.utf8.count)")
|
||||
do {
|
||||
logger.info("canvas eval awaiting CanvasManager.eval")
|
||||
let result = try await CanvasManager.shared.eval(sessionKey: session, javaScript: javaScript)
|
||||
logger.info("canvas eval done bytes=\(result.utf8.count)")
|
||||
return Response(ok: true, payload: Data(result.utf8))
|
||||
} catch {
|
||||
logger.error("canvas eval failed \(error.localizedDescription, privacy: .public)")
|
||||
return Response(ok: false, message: error.localizedDescription)
|
||||
}
|
||||
}
|
||||
@@ -234,16 +238,12 @@ enum ControlRequestHandler {
|
||||
guard self.canvasEnabled() else { return Response(ok: false, message: "Canvas disabled by user") }
|
||||
do {
|
||||
// Ensure the Canvas is visible without forcing a navigation/reload.
|
||||
_ = try await MainActor.run {
|
||||
try CanvasManager.shared.show(sessionKey: session, path: nil)
|
||||
}
|
||||
_ = try await CanvasManager.shared.show(sessionKey: session, path: nil)
|
||||
|
||||
// Wait for the in-page A2UI bridge. If it doesn't appear, force-load the bundled A2UI shell once.
|
||||
var ready = await Self.waitForCanvasA2UI(session: session, requireBuiltinPath: false, timeoutMs: 2_000)
|
||||
if !ready {
|
||||
_ = try await MainActor.run {
|
||||
try CanvasManager.shared.show(sessionKey: session, path: "/__clawdis__/a2ui/")
|
||||
}
|
||||
_ = try await CanvasManager.shared.show(sessionKey: session, path: "/__clawdis__/a2ui/")
|
||||
ready = await Self.waitForCanvasA2UI(session: session, requireBuiltinPath: true, timeoutMs: 5_000)
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@ struct CritterStatusLabel: View {
|
||||
@State private var nextLegWiggle = Date().addingTimeInterval(Double.random(in: 5.0...11.0))
|
||||
@State private var earWiggle: CGFloat = 0
|
||||
@State private var nextEarWiggle = Date().addingTimeInterval(Double.random(in: 7.0...14.0))
|
||||
private let ticker = Timer.publish(every: 0.35, on: .main, in: .common).autoconnect()
|
||||
|
||||
private var isWorkingNow: Bool {
|
||||
self.iconState.isWorking || self.isWorking
|
||||
@@ -32,34 +31,18 @@ struct CritterStatusLabel: View {
|
||||
.frame(width: 18, height: 18)
|
||||
.rotationEffect(.degrees(self.wiggleAngle), anchor: .center)
|
||||
.offset(x: self.wiggleOffset)
|
||||
.onReceive(self.ticker) { now in
|
||||
// Avoid Combine's TimerPublisher here: on macOS 26.2 we've seen crashes inside executor checks
|
||||
// triggered by its callbacks. Drive periodic updates via a Swift-concurrency task instead.
|
||||
.task(id: self.tickTaskID) {
|
||||
guard self.animationsEnabled, !self.earBoostActive else {
|
||||
self.resetMotion()
|
||||
await MainActor.run { self.resetMotion() }
|
||||
return
|
||||
}
|
||||
|
||||
if now >= self.nextBlink {
|
||||
self.blink()
|
||||
self.nextBlink = now.addingTimeInterval(Double.random(in: 3.5...8.5))
|
||||
}
|
||||
|
||||
if now >= self.nextWiggle {
|
||||
self.wiggle()
|
||||
self.nextWiggle = now.addingTimeInterval(Double.random(in: 6.5...14))
|
||||
}
|
||||
|
||||
if now >= self.nextLegWiggle {
|
||||
self.wiggleLegs()
|
||||
self.nextLegWiggle = now.addingTimeInterval(Double.random(in: 5.0...11.0))
|
||||
}
|
||||
|
||||
if now >= self.nextEarWiggle {
|
||||
self.wiggleEars()
|
||||
self.nextEarWiggle = now.addingTimeInterval(Double.random(in: 7.0...14.0))
|
||||
}
|
||||
|
||||
if self.isWorkingNow {
|
||||
self.scurry()
|
||||
while !Task.isCancelled {
|
||||
let now = Date()
|
||||
await MainActor.run { self.tick(now) }
|
||||
try? await Task.sleep(nanoseconds: 350_000_000)
|
||||
}
|
||||
}
|
||||
.onChange(of: self.isPaused) { _, _ in self.resetMotion() }
|
||||
@@ -96,6 +79,42 @@ struct CritterStatusLabel: View {
|
||||
.frame(width: 18, height: 18)
|
||||
}
|
||||
|
||||
private var tickTaskID: Int {
|
||||
// Ensure SwiftUI restarts (and cancels) the task when these change.
|
||||
(self.animationsEnabled ? 1 : 0) | (self.earBoostActive ? 2 : 0)
|
||||
}
|
||||
|
||||
private func tick(_ now: Date) {
|
||||
guard self.animationsEnabled, !self.earBoostActive else {
|
||||
self.resetMotion()
|
||||
return
|
||||
}
|
||||
|
||||
if now >= self.nextBlink {
|
||||
self.blink()
|
||||
self.nextBlink = now.addingTimeInterval(Double.random(in: 3.5...8.5))
|
||||
}
|
||||
|
||||
if now >= self.nextWiggle {
|
||||
self.wiggle()
|
||||
self.nextWiggle = now.addingTimeInterval(Double.random(in: 6.5...14))
|
||||
}
|
||||
|
||||
if now >= self.nextLegWiggle {
|
||||
self.wiggleLegs()
|
||||
self.nextLegWiggle = now.addingTimeInterval(Double.random(in: 5.0...11.0))
|
||||
}
|
||||
|
||||
if now >= self.nextEarWiggle {
|
||||
self.wiggleEars()
|
||||
self.nextEarWiggle = now.addingTimeInterval(Double.random(in: 7.0...14.0))
|
||||
}
|
||||
|
||||
if self.isWorkingNow {
|
||||
self.scurry()
|
||||
}
|
||||
}
|
||||
|
||||
private var iconImage: Image {
|
||||
let badge: CritterIconRenderer.Badge? = if let prominence = self.iconState.badgeProminence, !self.isPaused {
|
||||
CritterIconRenderer.Badge(
|
||||
@@ -128,7 +147,8 @@ struct CritterStatusLabel: View {
|
||||
|
||||
private func blink() {
|
||||
withAnimation(.easeInOut(duration: 0.08)) { self.blinkAmount = 1 }
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.16) {
|
||||
Task { @MainActor in
|
||||
try? await Task.sleep(nanoseconds: 160_000_000)
|
||||
withAnimation(.easeOut(duration: 0.12)) { self.blinkAmount = 0 }
|
||||
}
|
||||
}
|
||||
@@ -140,7 +160,8 @@ struct CritterStatusLabel: View {
|
||||
self.wiggleAngle = targetAngle
|
||||
self.wiggleOffset = targetOffset
|
||||
}
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.36) {
|
||||
Task { @MainActor in
|
||||
try? await Task.sleep(nanoseconds: 360_000_000)
|
||||
withAnimation(.interpolatingSpring(stiffness: 220, damping: 18)) {
|
||||
self.wiggleAngle = 0
|
||||
self.wiggleOffset = 0
|
||||
@@ -153,7 +174,8 @@ struct CritterStatusLabel: View {
|
||||
withAnimation(.easeInOut(duration: 0.14)) {
|
||||
self.legWiggle = target
|
||||
}
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.22) {
|
||||
Task { @MainActor in
|
||||
try? await Task.sleep(nanoseconds: 220_000_000)
|
||||
withAnimation(.easeOut(duration: 0.18)) { self.legWiggle = 0 }
|
||||
}
|
||||
}
|
||||
@@ -164,7 +186,8 @@ struct CritterStatusLabel: View {
|
||||
self.legWiggle = target
|
||||
self.wiggleOffset = CGFloat.random(in: -0.6...0.6)
|
||||
}
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.18) {
|
||||
Task { @MainActor in
|
||||
try? await Task.sleep(nanoseconds: 180_000_000)
|
||||
withAnimation(.easeOut(duration: 0.16)) {
|
||||
self.legWiggle = 0.25
|
||||
self.wiggleOffset = 0
|
||||
@@ -177,8 +200,11 @@ struct CritterStatusLabel: View {
|
||||
withAnimation(.interpolatingSpring(stiffness: 260, damping: 19)) {
|
||||
self.earWiggle = target
|
||||
}
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.32) {
|
||||
withAnimation(.interpolatingSpring(stiffness: 260, damping: 19)) { self.earWiggle = 0 }
|
||||
Task { @MainActor in
|
||||
try? await Task.sleep(nanoseconds: 320_000_000)
|
||||
withAnimation(.interpolatingSpring(stiffness: 260, damping: 19)) {
|
||||
self.earWiggle = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,11 @@ final class DeepLinkHandler {
|
||||
|
||||
private var lastPromptAt: Date = .distantPast
|
||||
|
||||
// Ephemeral, in-memory key used for unattended deep links originating from the in-app Canvas.
|
||||
// This avoids blocking Canvas init on UserDefaults and doesn't weaken the external deep-link prompt:
|
||||
// outside callers can't know this randomly generated key.
|
||||
private nonisolated static let canvasUnattendedKey: String = DeepLinkHandler.generateRandomKey()
|
||||
|
||||
func handle(url: URL) async {
|
||||
guard let route = DeepLinkParser.parse(url) else {
|
||||
deepLinkLogger.debug("ignored url \(url.absoluteString, privacy: .public)")
|
||||
@@ -35,7 +40,7 @@ final class DeepLinkHandler {
|
||||
return
|
||||
}
|
||||
|
||||
let allowUnattended = link.key == Self.expectedKey()
|
||||
let allowUnattended = link.key == Self.canvasUnattendedKey || link.key == Self.expectedKey()
|
||||
if !allowUnattended {
|
||||
if Date().timeIntervalSince(self.lastPromptAt) < 1.0 {
|
||||
deepLinkLogger.debug("throttling deep link prompt")
|
||||
@@ -83,6 +88,10 @@ final class DeepLinkHandler {
|
||||
self.expectedKey()
|
||||
}
|
||||
|
||||
static func currentCanvasKey() -> String {
|
||||
self.canvasUnattendedKey
|
||||
}
|
||||
|
||||
private static func expectedKey() -> String {
|
||||
let defaults = UserDefaults.standard
|
||||
if let key = defaults.string(forKey: deepLinkKeyKey), !key.isEmpty {
|
||||
@@ -100,6 +109,17 @@ final class DeepLinkHandler {
|
||||
return key
|
||||
}
|
||||
|
||||
private nonisolated static func generateRandomKey() -> String {
|
||||
var bytes = [UInt8](repeating: 0, count: 32)
|
||||
_ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes)
|
||||
let data = Data(bytes)
|
||||
return data
|
||||
.base64EncodedString()
|
||||
.replacingOccurrences(of: "+", with: "-")
|
||||
.replacingOccurrences(of: "/", with: "_")
|
||||
.replacingOccurrences(of: "=", with: "")
|
||||
}
|
||||
|
||||
// MARK: - UI
|
||||
|
||||
private func confirm(title: String, message: String) -> Bool {
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import AppKit
|
||||
import AVFoundation
|
||||
import Dispatch
|
||||
import OSLog
|
||||
import Speech
|
||||
|
||||
/// Observes right Option and starts a push-to-talk capture while it is held.
|
||||
@MainActor
|
||||
final class VoicePushToTalkHotkey {
|
||||
final class VoicePushToTalkHotkey: @unchecked Sendable {
|
||||
static let shared = VoicePushToTalkHotkey()
|
||||
|
||||
private var globalMonitor: Any?
|
||||
@@ -13,29 +13,48 @@ final class VoicePushToTalkHotkey {
|
||||
private var optionDown = false // right option only
|
||||
private var active = false
|
||||
|
||||
private let beginAction: @Sendable () async -> Void
|
||||
private let endAction: @Sendable () async -> Void
|
||||
|
||||
init(
|
||||
beginAction: @escaping @Sendable () async -> Void = { await VoicePushToTalk.shared.begin() },
|
||||
endAction: @escaping @Sendable () async -> Void = { await VoicePushToTalk.shared.end() }
|
||||
) {
|
||||
self.beginAction = beginAction
|
||||
self.endAction = endAction
|
||||
}
|
||||
|
||||
func setEnabled(_ enabled: Bool) {
|
||||
if enabled {
|
||||
self.startMonitoring()
|
||||
} else {
|
||||
self.stopMonitoring()
|
||||
self.withMainThread { [weak self] in
|
||||
guard let self else { return }
|
||||
if enabled {
|
||||
self.startMonitoring()
|
||||
} else {
|
||||
self.stopMonitoring()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func startMonitoring() {
|
||||
assert(Thread.isMainThread)
|
||||
guard self.globalMonitor == nil, self.localMonitor == nil else { return }
|
||||
// Listen-only global monitor; we rely on Input Monitoring permission to receive events.
|
||||
self.globalMonitor = NSEvent.addGlobalMonitorForEvents(matching: .flagsChanged) { [weak self] event in
|
||||
guard let self else { return }
|
||||
self.updateModifierState(from: event)
|
||||
let keyCode = event.keyCode
|
||||
let flags = event.modifierFlags
|
||||
self?.handleFlagsChanged(keyCode: keyCode, modifierFlags: flags)
|
||||
}
|
||||
// Also listen locally so we still catch events when the app is active/focused.
|
||||
self.localMonitor = NSEvent.addLocalMonitorForEvents(matching: .flagsChanged) { [weak self] event in
|
||||
self?.updateModifierState(from: event)
|
||||
let keyCode = event.keyCode
|
||||
let flags = event.modifierFlags
|
||||
self?.handleFlagsChanged(keyCode: keyCode, modifierFlags: flags)
|
||||
return event
|
||||
}
|
||||
}
|
||||
|
||||
private func stopMonitoring() {
|
||||
assert(Thread.isMainThread)
|
||||
if let globalMonitor {
|
||||
NSEvent.removeMonitor(globalMonitor)
|
||||
self.globalMonitor = nil
|
||||
@@ -48,10 +67,25 @@ final class VoicePushToTalkHotkey {
|
||||
self.active = false
|
||||
}
|
||||
|
||||
private func updateModifierState(from event: NSEvent) {
|
||||
private func handleFlagsChanged(keyCode: UInt16, modifierFlags: NSEvent.ModifierFlags) {
|
||||
self.withMainThread { [weak self] in
|
||||
self?.updateModifierState(keyCode: keyCode, modifierFlags: modifierFlags)
|
||||
}
|
||||
}
|
||||
|
||||
private func withMainThread(_ block: @escaping @Sendable () -> Void) {
|
||||
if Thread.isMainThread {
|
||||
block()
|
||||
} else {
|
||||
DispatchQueue.main.async(execute: block)
|
||||
}
|
||||
}
|
||||
|
||||
private func updateModifierState(keyCode: UInt16, modifierFlags: NSEvent.ModifierFlags) {
|
||||
assert(Thread.isMainThread)
|
||||
// Right Option (keyCode 61) acts as a hold-to-talk modifier.
|
||||
if event.keyCode == 61 {
|
||||
self.optionDown = event.modifierFlags.contains(.option)
|
||||
if keyCode == 61 {
|
||||
self.optionDown = modifierFlags.contains(.option)
|
||||
}
|
||||
|
||||
let chordActive = self.optionDown
|
||||
@@ -60,17 +94,21 @@ final class VoicePushToTalkHotkey {
|
||||
Task {
|
||||
Logger(subsystem: "com.steipete.clawdis", category: "voicewake.ptt")
|
||||
.info("ptt hotkey down")
|
||||
await VoicePushToTalk.shared.begin()
|
||||
await self.beginAction()
|
||||
}
|
||||
} else if !chordActive, self.active {
|
||||
self.active = false
|
||||
Task {
|
||||
Logger(subsystem: "com.steipete.clawdis", category: "voicewake.ptt")
|
||||
.info("ptt hotkey up")
|
||||
await VoicePushToTalk.shared.end()
|
||||
await self.endAction()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func _testUpdateModifierState(keyCode: UInt16, modifierFlags: NSEvent.ModifierFlags) {
|
||||
self.updateModifierState(keyCode: keyCode, modifierFlags: modifierFlags)
|
||||
}
|
||||
}
|
||||
|
||||
/// Short-lived speech recognizer that records while the hotkey is held.
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import AppKit
|
||||
import Testing
|
||||
@testable import Clawdis
|
||||
|
||||
@Suite(.serialized) struct VoicePushToTalkHotkeyTests {
|
||||
actor Counter {
|
||||
private(set) var began = 0
|
||||
private(set) var ended = 0
|
||||
|
||||
func incBegin() { self.began += 1 }
|
||||
func incEnd() { self.ended += 1 }
|
||||
func snapshot() -> (began: Int, ended: Int) { (self.began, self.ended) }
|
||||
}
|
||||
|
||||
@Test func beginEndFiresOncePerHold() async {
|
||||
let counter = Counter()
|
||||
let hotkey = VoicePushToTalkHotkey(
|
||||
beginAction: { await counter.incBegin() },
|
||||
endAction: { await counter.incEnd() })
|
||||
|
||||
await MainActor.run {
|
||||
hotkey._testUpdateModifierState(keyCode: 61, modifierFlags: [.option])
|
||||
hotkey._testUpdateModifierState(keyCode: 61, modifierFlags: [.option])
|
||||
hotkey._testUpdateModifierState(keyCode: 61, modifierFlags: [])
|
||||
}
|
||||
|
||||
for _ in 0..<50 {
|
||||
let snap = await counter.snapshot()
|
||||
if snap.began == 1, snap.ended == 1 { break }
|
||||
try? await Task.sleep(nanoseconds: 10_000_000)
|
||||
}
|
||||
|
||||
let snap = await counter.snapshot()
|
||||
#expect(snap.began == 1)
|
||||
#expect(snap.ended == 1)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,6 @@ import Testing
|
||||
#expect(opts.thinking == "low")
|
||||
#expect(opts.deliver == true)
|
||||
#expect(opts.to == nil)
|
||||
#expect(opts.channel == "last")
|
||||
#expect(opts.channel == .last)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,15 +108,15 @@ open 'clawdis://agent?message=Hello%20from%20deep%20link'
|
||||
Query parameters:
|
||||
- `message` (required): the agent prompt (URL-encoded).
|
||||
- `sessionKey` (optional): explicit session key to use.
|
||||
- `thinking` (optional): `off|minimal|low|medium|high` (or omit for default).
|
||||
- `thinking` (optional): thinking hint (e.g. `low`; omit for default).
|
||||
- `deliver` (optional): `true|false` (default: false).
|
||||
- `to` / `channel` (optional): forwarded to the Gateway `agent` method (only meaningful with `deliver=true`).
|
||||
- `timeoutSeconds` (optional): timeout hint forwarded to the Gateway.
|
||||
- `key` (optional): unattended mode key (see below).
|
||||
|
||||
Safety/guardrails:
|
||||
- Disabled by default; enable in **Clawdis → Settings → Debug** (“Allow URL scheme (agent)”).
|
||||
- Without `key`, Clawdis prompts with a confirmation dialog before invoking the agent.
|
||||
- Always enabled.
|
||||
- Without a `key` query param, the app will prompt for confirmation before invoking the agent.
|
||||
- With `key=<value>`, Clawdis runs without prompting (intended for personal automations).
|
||||
- The current key is shown in Debug Settings and stored locally in UserDefaults.
|
||||
|
||||
|
||||
@@ -139,8 +139,9 @@ Implementation note (important):
|
||||
- In `WKWebView`, intercept `clawdis://…` navigations in `WKNavigationDelegate` and forward them to the app, e.g. by calling `DeepLinkHandler.shared.handle(url:)` and returning `.cancel` for the navigation.
|
||||
|
||||
Safety:
|
||||
- `clawdis://agent` is disabled by default and must be enabled in **Clawdis → Settings → Debug** (“Allow URL scheme (agent)”).
|
||||
- Deep links (`clawdis://agent?...`) are always enabled.
|
||||
- Without a `key` query param, the app will prompt for confirmation before invoking the agent.
|
||||
- With a valid `key`, the run is unattended (no prompt). For Canvas-originated actions, the app injects an internal key automatically.
|
||||
|
||||
## Security / guardrails
|
||||
|
||||
|
||||
@@ -12,7 +12,16 @@
|
||||
|
||||
## Fixes (2025-12-17)
|
||||
- Close button: render a small vibrancy/material pill behind the “x” and reduce the button size for less visual weight.
|
||||
- Click reliability: `GatewayConnection` auto-starts the local gateway (and retries briefly) when a request fails in `.local` mode, so Canvas actions don’t silently fail if the gateway isn’t running yet.
|
||||
- Click reliability:
|
||||
- Allow A2UI clicks from any local Canvas path (not just `/` or the built-in A2UI shell).
|
||||
- Inject an A2UI → native bridge at document start that listens for `a2uiaction` and forwards it:
|
||||
- Prefer `WKScriptMessageHandler` when available.
|
||||
- Otherwise fall back to an unattended `clawdis://agent?...&key=...` deep link (no prompt).
|
||||
- Intercept `clawdis://…` navigations inside the Canvas WKWebView and route them through `DeepLinkHandler` (no NSWorkspace bounce).
|
||||
- `GatewayConnection` auto-starts the local gateway (and retries briefly) when a request fails in `.local` mode, so Canvas actions don’t silently fail if the gateway isn’t running yet.
|
||||
- Fix a crash that made `clawdis-mac canvas show`/`eval` look “hung”:
|
||||
- `VoicePushToTalkHotkey`’s NSEvent monitor could call `@MainActor` code off-main, triggering executor checks / EXC_BAD_ACCESS on macOS 26.2.
|
||||
- Now it hops back to the main actor before mutating state.
|
||||
|
||||
## Follow-ups
|
||||
- Add a small “action sent / failed” debug overlay in the A2UI shell (dev-only) to make failures obvious.
|
||||
|
||||
Reference in New Issue
Block a user