Merge remote-tracking branch 'origin/main' into upstream-preview-nix-2025-12-20

This commit is contained in:
Peter Steinberger
2026-01-01 09:15:28 +01:00
163 changed files with 10867 additions and 1712 deletions

View File

@@ -15,6 +15,7 @@ let package = Package(
dependencies: [
.package(url: "https://github.com/orchetect/MenuBarExtraAccess", exact: "1.2.2"),
.package(url: "https://github.com/swiftlang/swift-subprocess.git", from: "0.1.0"),
.package(url: "https://github.com/apple/swift-log.git", from: "1.8.0"),
.package(url: "https://github.com/sparkle-project/Sparkle", from: "2.8.1"),
.package(path: "../shared/ClawdisKit"),
.package(path: "../../Swabble"),
@@ -45,6 +46,7 @@ let package = Package(
.product(name: "SwabbleKit", package: "swabble"),
.product(name: "MenuBarExtraAccess", package: "MenuBarExtraAccess"),
.product(name: "Subprocess", package: "swift-subprocess"),
.product(name: "Logging", package: "swift-log"),
.product(name: "Sparkle", package: "Sparkle"),
.product(name: "PeekabooBridge", package: "PeekabooCore"),
.product(name: "PeekabooAutomationKit", package: "PeekabooAutomationKit"),

View File

@@ -121,6 +121,18 @@ final class AppState {
forKey: voicePushToTalkEnabledKey) } }
}
var talkEnabled: Bool {
didSet {
self.ifNotPreview {
UserDefaults.standard.set(self.talkEnabled, forKey: talkEnabledKey)
Task { await TalkModeController.shared.setEnabled(self.talkEnabled) }
}
}
}
/// Gateway-provided UI accent color (hex). Optional; clients provide a default.
var seamColorHex: String?
var iconOverride: IconOverrideSelection {
didSet { self.ifNotPreview { UserDefaults.standard.set(self.iconOverride.rawValue, forKey: iconOverrideKey) } }
}
@@ -216,6 +228,8 @@ final class AppState {
.stringArray(forKey: voiceWakeAdditionalLocalesKey) ?? []
self.voicePushToTalkEnabled = UserDefaults.standard
.object(forKey: voicePushToTalkEnabledKey) as? Bool ?? false
self.talkEnabled = UserDefaults.standard.bool(forKey: talkEnabledKey)
self.seamColorHex = nil
if let storedHeartbeats = UserDefaults.standard.object(forKey: heartbeatsEnabledKey) as? Bool {
self.heartbeatsEnabled = storedHeartbeats
} else {
@@ -256,9 +270,13 @@ final class AppState {
if self.swabbleEnabled, !PermissionManager.voiceWakePermissionsGranted() {
self.swabbleEnabled = false
}
if self.talkEnabled, !PermissionManager.voiceWakePermissionsGranted() {
self.talkEnabled = false
}
if !self.isPreview {
Task { await VoiceWakeRuntime.shared.refresh(state: self) }
Task { await TalkModeController.shared.setEnabled(self.talkEnabled) }
}
}
@@ -312,6 +330,31 @@ final class AppState {
Task { await VoiceWakeRuntime.shared.refresh(state: self) }
}
func setTalkEnabled(_ enabled: Bool) async {
guard voiceWakeSupported else {
self.talkEnabled = false
await GatewayConnection.shared.talkMode(enabled: false, phase: "disabled")
return
}
self.talkEnabled = enabled
guard !self.isPreview else { return }
if !enabled {
await GatewayConnection.shared.talkMode(enabled: false, phase: "disabled")
return
}
if PermissionManager.voiceWakePermissionsGranted() {
await GatewayConnection.shared.talkMode(enabled: true, phase: "enabled")
return
}
let granted = await PermissionManager.ensureVoiceWakePermissions(interactive: true)
self.talkEnabled = granted
await GatewayConnection.shared.talkMode(enabled: granted, phase: granted ? "enabled" : "denied")
}
// MARK: - Global wake words sync (Gateway-owned)
func applyGlobalVoiceWakeTriggers(_ triggers: [String]) {
@@ -367,6 +410,7 @@ extension AppState {
state.voiceWakeLocaleID = Locale.current.identifier
state.voiceWakeAdditionalLocaleIDs = ["en-US", "de-DE"]
state.voicePushToTalkEnabled = false
state.talkEnabled = false
state.iconOverride = .system
state.heartbeatsEnabled = true
state.connectionMode = .local

View File

@@ -79,7 +79,14 @@ actor CameraCaptureService {
}
withExtendedLifetime(delegate) {}
let res = try JPEGTranscoder.transcodeToJPEG(imageData: rawData, maxWidthPx: maxWidth, quality: quality)
let maxPayloadBytes = 5 * 1024 * 1024
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
let maxEncodedBytes = (maxPayloadBytes / 4) * 3
let res = try JPEGTranscoder.transcodeToJPEG(
imageData: rawData,
maxWidthPx: maxWidth,
quality: quality,
maxBytes: maxEncodedBytes)
return (data: res.data, size: CGSize(width: res.widthPx, height: res.heightPx))
}

View File

@@ -190,7 +190,7 @@ final class CanvasManager {
private static func resolveA2UIHostUrl(from raw: String?) -> String? {
let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString
return base.appendingPathComponent("__clawdis__/a2ui/").absoluteString + "?platform=macos"
}
// MARK: - Anchoring

View File

@@ -1,5 +1,4 @@
import AppKit
import OSLog
let canvasWindowLogger = Logger(subsystem: "com.steipete.clawdis", category: "Canvas")

View File

@@ -1,6 +1,8 @@
import Foundation
enum ClawdisConfigFile {
private static let logger = Logger(subsystem: "com.steipete.clawdis", category: "config")
static func url() -> URL {
FileManager.default.homeDirectoryForCurrentUser
.appendingPathComponent(".clawdis")
@@ -15,8 +17,18 @@ enum ClawdisConfigFile {
static func loadDict() -> [String: Any] {
let url = self.url()
guard let data = try? Data(contentsOf: url) else { return [:] }
return (try? JSONSerialization.jsonObject(with: data) as? [String: Any]) ?? [:]
guard FileManager.default.fileExists(atPath: url.path) else { return [:] }
do {
let data = try Data(contentsOf: url)
guard let root = try JSONSerialization.jsonObject(with: data) as? [String: Any] else {
self.logger.warning("config JSON root invalid")
return [:]
}
return root
} catch {
self.logger.warning("config read failed: \(error.localizedDescription)")
return [:]
}
}
static func saveDict(_ dict: [String: Any]) {
@@ -28,7 +40,9 @@ enum ClawdisConfigFile {
at: url.deletingLastPathComponent(),
withIntermediateDirectories: true)
try data.write(to: url, options: [.atomic])
} catch {}
} catch {
self.logger.error("config save failed: \(error.localizedDescription)")
}
}
static func loadGatewayDict() -> [String: Any] {
@@ -60,6 +74,7 @@ enum ClawdisConfigFile {
browser["enabled"] = enabled
root["browser"] = browser
self.saveDict(root)
self.logger.debug("browser control updated enabled=\(enabled)")
}
static func agentWorkspace() -> String? {
@@ -79,5 +94,6 @@ enum ClawdisConfigFile {
}
root["agent"] = agent
self.saveDict(root)
self.logger.debug("agent workspace updated set=\(!trimmed.isEmpty)")
}
}

View File

@@ -16,6 +16,10 @@ enum CommandResolver {
RuntimeLocator.resolve(searchPaths: self.preferredPaths())
}
static func runtimeResolution(searchPaths: [String]?) -> Result<RuntimeResolution, RuntimeResolutionError> {
RuntimeLocator.resolve(searchPaths: searchPaths ?? self.preferredPaths())
}
static func makeRuntimeCommand(
runtime: RuntimeResolution,
entrypoint: String,
@@ -152,8 +156,8 @@ enum CommandResolver {
return paths
}
static func findExecutable(named name: String) -> String? {
for dir in self.preferredPaths() {
static func findExecutable(named name: String, searchPaths: [String]? = nil) -> String? {
for dir in (searchPaths ?? self.preferredPaths()) {
let candidate = (dir as NSString).appendingPathComponent(name)
if FileManager.default.isExecutableFile(atPath: candidate) {
return candidate
@@ -162,8 +166,14 @@ enum CommandResolver {
return nil
}
static func clawdisExecutable() -> String? {
self.findExecutable(named: self.helperName)
static func clawdisExecutable(searchPaths: [String]? = nil) -> String? {
self.findExecutable(named: self.helperName, searchPaths: searchPaths)
}
static func projectClawdisExecutable(projectRoot: URL? = nil) -> String? {
let root = projectRoot ?? self.projectRoot()
let candidate = root.appendingPathComponent("node_modules/.bin").appendingPathComponent(self.helperName).path
return FileManager.default.isExecutableFile(atPath: candidate) ? candidate : nil
}
static func nodeCliPath() -> String? {
@@ -171,17 +181,18 @@ enum CommandResolver {
return FileManager.default.isReadableFile(atPath: candidate) ? candidate : nil
}
static func hasAnyClawdisInvoker() -> Bool {
if self.clawdisExecutable() != nil { return true }
if self.findExecutable(named: "pnpm") != nil { return true }
if self.findExecutable(named: "node") != nil, self.nodeCliPath() != nil { return true }
static func hasAnyClawdisInvoker(searchPaths: [String]? = nil) -> Bool {
if self.clawdisExecutable(searchPaths: searchPaths) != nil { return true }
if self.findExecutable(named: "pnpm", searchPaths: searchPaths) != nil { return true }
if self.findExecutable(named: "node", searchPaths: searchPaths) != nil, self.nodeCliPath() != nil { return true }
return false
}
static func clawdisNodeCommand(
subcommand: String,
extraArgs: [String] = [],
defaults: UserDefaults = .standard) -> [String]
defaults: UserDefaults = .standard,
searchPaths: [String]? = nil) -> [String]
{
let settings = self.connectionSettings(defaults: defaults)
if settings.mode == .remote, let ssh = self.sshNodeCommand(
@@ -192,25 +203,29 @@ enum CommandResolver {
return ssh
}
let runtimeResult = self.runtimeResolution()
let runtimeResult = self.runtimeResolution(searchPaths: searchPaths)
switch runtimeResult {
case let .success(runtime):
if let clawdisPath = self.clawdisExecutable() {
let root = self.projectRoot()
if let clawdisPath = self.projectClawdisExecutable(projectRoot: root) {
return [clawdisPath, subcommand] + extraArgs
}
if let entry = self.gatewayEntrypoint(in: self.projectRoot()) {
if let entry = self.gatewayEntrypoint(in: root) {
return self.makeRuntimeCommand(
runtime: runtime,
entrypoint: entry,
subcommand: subcommand,
extraArgs: extraArgs)
}
if let pnpm = self.findExecutable(named: "pnpm") {
if let pnpm = self.findExecutable(named: "pnpm", searchPaths: searchPaths) {
// Use --silent to avoid pnpm lifecycle banners that would corrupt JSON outputs.
return [pnpm, "--silent", "clawdis", subcommand] + extraArgs
}
if let clawdisPath = self.clawdisExecutable(searchPaths: searchPaths) {
return [clawdisPath, subcommand] + extraArgs
}
let missingEntry = """
clawdis entrypoint missing (looked for dist/index.js or bin/clawdis.js); run pnpm build.
@@ -226,9 +241,10 @@ enum CommandResolver {
static func clawdisCommand(
subcommand: String,
extraArgs: [String] = [],
defaults: UserDefaults = .standard) -> [String]
defaults: UserDefaults = .standard,
searchPaths: [String]? = nil) -> [String]
{
self.clawdisNodeCommand(subcommand: subcommand, extraArgs: extraArgs, defaults: defaults)
self.clawdisNodeCommand(subcommand: subcommand, extraArgs: extraArgs, defaults: defaults, searchPaths: searchPaths)
}
// MARK: - SSH helpers
@@ -258,7 +274,7 @@ enum CommandResolver {
"/bin",
"/usr/sbin",
"/sbin",
"/Users/steipete/Library/pnpm",
"$HOME/Library/pnpm",
"$PATH",
].joined(separator: ":")
let quotedArgs = ([subcommand] + extraArgs).map(self.shellQuote).joined(separator: " ")

View File

@@ -31,6 +31,12 @@ struct ConfigSettings: View {
@State private var browserColorHex: String = "#FF4500"
@State private var browserAttachOnly: Bool = false
// Talk mode settings (stored in ~/.clawdis/clawdis.json under "talk")
@State private var talkVoiceId: String = ""
@State private var talkInterruptOnSpeech: Bool = true
@State private var talkApiKey: String = ""
@State private var gatewayApiKeyFound = false
var body: some View {
ScrollView { self.content }
.onChange(of: self.modelCatalogPath) { _, _ in
@@ -45,6 +51,7 @@ struct ConfigSettings: View {
self.hasLoaded = true
self.loadConfig()
await self.loadModels()
await self.refreshGatewayTalkApiKey()
self.allowAutosave = true
}
}
@@ -56,6 +63,8 @@ struct ConfigSettings: View {
.disabled(self.isNixMode)
self.heartbeatSection
.disabled(self.isNixMode)
self.talkSection
.disabled(self.isNixMode)
self.browserSection
.disabled(self.isNixMode)
Spacer(minLength: 0)
@@ -272,18 +281,101 @@ struct ConfigSettings: View {
.frame(maxWidth: .infinity, alignment: .leading)
}
private var talkSection: some View {
GroupBox("Talk Mode") {
Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) {
GridRow {
self.gridLabel("Voice ID")
VStack(alignment: .leading, spacing: 6) {
HStack(spacing: 8) {
TextField("ElevenLabs voice ID", text: self.$talkVoiceId)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
.onChange(of: self.talkVoiceId) { _, _ in self.autosaveConfig() }
if !self.talkVoiceSuggestions.isEmpty {
Menu {
ForEach(self.talkVoiceSuggestions, id: \.self) { value in
Button(value) {
self.talkVoiceId = value
self.autosaveConfig()
}
}
} label: {
Label("Suggestions", systemImage: "chevron.up.chevron.down")
}
.fixedSize()
}
}
Text("Defaults to ELEVENLABS_VOICE_ID / SAG_VOICE_ID if unset.")
.font(.footnote)
.foregroundStyle(.secondary)
}
}
GridRow {
self.gridLabel("API key")
VStack(alignment: .leading, spacing: 6) {
HStack(spacing: 8) {
SecureField("ELEVENLABS_API_KEY", text: self.$talkApiKey)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
.disabled(self.hasEnvApiKey)
.onChange(of: self.talkApiKey) { _, _ in self.autosaveConfig() }
if !self.hasEnvApiKey && !self.talkApiKey.isEmpty {
Button("Clear") {
self.talkApiKey = ""
self.autosaveConfig()
}
}
}
self.statusLine(label: self.apiKeyStatusLabel, color: self.apiKeyStatusColor)
if self.hasEnvApiKey {
Text("Using ELEVENLABS_API_KEY from the environment.")
.font(.footnote)
.foregroundStyle(.secondary)
} else if self.gatewayApiKeyFound && self.talkApiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
Text("Using API key from the gateway profile.")
.font(.footnote)
.foregroundStyle(.secondary)
}
}
}
GridRow {
self.gridLabel("Interrupt")
Toggle("Stop speaking when you start talking", isOn: self.$talkInterruptOnSpeech)
.labelsHidden()
.toggleStyle(.checkbox)
.onChange(of: self.talkInterruptOnSpeech) { _, _ in self.autosaveConfig() }
}
}
}
.frame(maxWidth: .infinity, alignment: .leading)
}
private func gridLabel(_ text: String) -> some View {
Text(text)
.foregroundStyle(.secondary)
.frame(width: self.labelColumnWidth, alignment: .leading)
}
private func statusLine(label: String, color: Color) -> some View {
HStack(spacing: 6) {
Circle()
.fill(color)
.frame(width: 6, height: 6)
Text(label)
.font(.footnote)
.foregroundStyle(.secondary)
}
.padding(.top, 2)
}
private func loadConfig() {
let parsed = self.loadConfigDict()
let agent = parsed["agent"] as? [String: Any]
let heartbeatMinutes = agent?["heartbeatMinutes"] as? Int
let heartbeatBody = agent?["heartbeatBody"] as? String
let browser = parsed["browser"] as? [String: Any]
let talk = parsed["talk"] as? [String: Any]
let loadedModel = (agent?["model"] as? String) ?? ""
if !loadedModel.isEmpty {
@@ -303,6 +395,28 @@ struct ConfigSettings: View {
if let color = browser["color"] as? String, !color.isEmpty { self.browserColorHex = color }
if let attachOnly = browser["attachOnly"] as? Bool { self.browserAttachOnly = attachOnly }
}
if let talk {
if let voice = talk["voiceId"] as? String { self.talkVoiceId = voice }
if let apiKey = talk["apiKey"] as? String { self.talkApiKey = apiKey }
if let interrupt = talk["interruptOnSpeech"] as? Bool {
self.talkInterruptOnSpeech = interrupt
}
}
}
private func refreshGatewayTalkApiKey() async {
do {
let snap: ConfigSnapshot = try await GatewayConnection.shared.requestDecoded(
method: .configGet,
params: nil,
timeoutMs: 8000)
let talk = snap.config?["talk"]?.dictionaryValue
let apiKey = talk?["apiKey"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines)
self.gatewayApiKeyFound = !(apiKey ?? "").isEmpty
} catch {
self.gatewayApiKeyFound = false
}
}
private func autosaveConfig() {
@@ -318,6 +432,7 @@ struct ConfigSettings: View {
var root = self.loadConfigDict()
var agent = root["agent"] as? [String: Any] ?? [:]
var browser = root["browser"] as? [String: Any] ?? [:]
var talk = root["talk"] as? [String: Any] ?? [:]
let chosenModel = (self.configModel == "__custom__" ? self.customModel : self.configModel)
.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -343,6 +458,21 @@ struct ConfigSettings: View {
browser["attachOnly"] = self.browserAttachOnly
root["browser"] = browser
let trimmedVoice = self.talkVoiceId.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmedVoice.isEmpty {
talk.removeValue(forKey: "voiceId")
} else {
talk["voiceId"] = trimmedVoice
}
let trimmedApiKey = self.talkApiKey.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmedApiKey.isEmpty {
talk.removeValue(forKey: "apiKey")
} else {
talk["apiKey"] = trimmedApiKey
}
talk["interruptOnSpeech"] = self.talkInterruptOnSpeech
root["talk"] = talk
ClawdisConfigFile.saveDict(root)
}
@@ -360,6 +490,41 @@ struct ConfigSettings: View {
return Color(red: r, green: g, blue: b)
}
private var talkVoiceSuggestions: [String] {
let env = ProcessInfo.processInfo.environment
let candidates = [
self.talkVoiceId,
env["ELEVENLABS_VOICE_ID"] ?? "",
env["SAG_VOICE_ID"] ?? "",
]
var seen = Set<String>()
return candidates
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
.filter { !$0.isEmpty }
.filter { seen.insert($0).inserted }
}
private var hasEnvApiKey: Bool {
let raw = ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"] ?? ""
return !raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}
private var apiKeyStatusLabel: String {
if self.hasEnvApiKey { return "ElevenLabs API key: found (environment)" }
if !self.talkApiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return "ElevenLabs API key: stored in config"
}
if self.gatewayApiKeyFound { return "ElevenLabs API key: found (gateway)" }
return "ElevenLabs API key: missing"
}
private var apiKeyStatusColor: Color {
if self.hasEnvApiKey { return .green }
if !self.talkApiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return .green }
if self.gatewayApiKeyFound { return .green }
return .red
}
private var browserPathLabel: String? {
guard self.browserEnabled else { return nil }

View File

@@ -294,6 +294,11 @@ final class ConnectionsStore {
: nil
self.configRoot = snap.config?.mapValues { $0.foundationValue } ?? [:]
self.configLoaded = true
let ui = snap.config?["ui"]?.dictionaryValue
let rawSeam = ui?["seamColor"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
AppStateStore.shared.seamColorHex = rawSeam.isEmpty ? nil : rawSeam
let telegram = snap.config?["telegram"]?.dictionaryValue
self.telegramToken = telegram?["botToken"]?.stringValue ?? ""
self.telegramRequireMention = telegram?["requireMention"]?.boolValue ?? true

View File

@@ -16,6 +16,7 @@ let voiceWakeMicKey = "clawdis.voiceWakeMicID"
let voiceWakeLocaleKey = "clawdis.voiceWakeLocaleID"
let voiceWakeAdditionalLocalesKey = "clawdis.voiceWakeAdditionalLocaleIDs"
let voicePushToTalkEnabledKey = "clawdis.voicePushToTalkEnabled"
let talkEnabledKey = "clawdis.talkEnabled"
let iconOverrideKey = "clawdis.iconOverride"
let connectionModeKey = "clawdis.connectionMode"
let remoteTargetKey = "clawdis.remoteTarget"
@@ -31,5 +32,6 @@ let modelCatalogReloadKey = "clawdis.modelCatalogReload"
let attachExistingGatewayOnlyKey = "clawdis.gateway.attachExistingOnly"
let heartbeatsEnabledKey = "clawdis.heartbeatsEnabled"
let debugFileLogEnabledKey = "clawdis.debug.fileLogEnabled"
let appLogLevelKey = "clawdis.debug.appLogLevel"
let voiceWakeSupported: Bool = ProcessInfo.processInfo.operatingSystemVersion.majorVersion >= 26
let cliHelperSearchPaths = ["/usr/local/bin", "/opt/homebrew/bin"]

View File

@@ -1,7 +1,6 @@
import ClawdisProtocol
import Foundation
import Observation
import OSLog
import SwiftUI
struct ControlHeartbeatEvent: Codable {

View File

@@ -1,8 +1,10 @@
import AppKit
import Observation
import SwiftUI
import UniformTypeIdentifiers
struct DebugSettings: View {
@Bindable var state: AppState
private let isPreview = ProcessInfo.processInfo.isPreview
private let labelColumnWidth: CGFloat = 140
@AppStorage(modelCatalogPathKey) private var modelCatalogPath: String = ModelCatalogLoader.defaultPath
@@ -28,6 +30,7 @@ struct DebugSettings: View {
@State private var pendingKill: DebugActions.PortListener?
@AppStorage(attachExistingGatewayOnlyKey) private var attachExistingGatewayOnly: Bool = false
@AppStorage(debugFileLogEnabledKey) private var diagnosticsFileLogEnabled: Bool = false
@AppStorage(appLogLevelKey) private var appLogLevelRaw: String = AppLogLevel.default.rawValue
@State private var canvasSessionKey: String = "main"
@State private var canvasStatus: String?
@@ -36,6 +39,10 @@ struct DebugSettings: View {
@State private var canvasEvalResult: String?
@State private var canvasSnapshotPath: String?
init(state: AppState = AppStateStore.shared) {
self.state = state
}
var body: some View {
ScrollView(.vertical) {
VStack(alignment: .leading, spacing: 14) {
@@ -194,7 +201,9 @@ struct DebugSettings: View {
.overlay(RoundedRectangle(cornerRadius: 6).stroke(Color.secondary.opacity(0.2)))
HStack(spacing: 8) {
Button("Restart Gateway") { DebugActions.restartGateway() }
if self.canRestartGateway {
Button("Restart Gateway") { DebugActions.restartGateway() }
}
Button("Clear log") { GatewayProcessManager.shared.clearLog() }
Spacer(minLength: 0)
}
@@ -224,13 +233,23 @@ struct DebugSettings: View {
}
GridRow {
self.gridLabel("Diagnostics")
VStack(alignment: .leading, spacing: 6) {
self.gridLabel("App logging")
VStack(alignment: .leading, spacing: 8) {
Picker("Verbosity", selection: self.$appLogLevelRaw) {
ForEach(AppLogLevel.allCases) { level in
Text(level.title).tag(level.rawValue)
}
}
.pickerStyle(.menu)
.labelsHidden()
.help("Controls the macOS app log verbosity.")
Toggle("Write rolling diagnostics log (JSONL)", isOn: self.$diagnosticsFileLogEnabled)
.toggleStyle(.checkbox)
.help(
"Writes a rotating, local-only diagnostics log under ~/Library/Logs/Clawdis/. " +
"Writes a rotating, local-only log under ~/Library/Logs/Clawdis/. " +
"Enable only while actively debugging.")
HStack(spacing: 8) {
Button("Open folder") {
NSWorkspace.shared.open(DiagnosticsFileLog.logDirectoryURL())
@@ -762,6 +781,10 @@ struct DebugSettings: View {
CommandResolver.connectionSettings().mode == .remote
}
private var canRestartGateway: Bool {
self.state.connectionMode == .local && !self.attachExistingGatewayOnly
}
private func configURL() -> URL {
FileManager.default.homeDirectoryForCurrentUser
.appendingPathComponent(".clawdis")
@@ -902,7 +925,7 @@ private struct PlainSettingsGroupBoxStyle: GroupBoxStyle {
#if DEBUG
struct DebugSettings_Previews: PreviewProvider {
static var previews: some View {
DebugSettings()
DebugSettings(state: .preview)
.frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight)
}
}
@@ -910,7 +933,7 @@ struct DebugSettings_Previews: PreviewProvider {
@MainActor
extension DebugSettings {
static func exerciseForTesting() async {
let view = DebugSettings()
let view = DebugSettings(state: .preview)
view.modelsCount = 3
view.modelsLoading = false
view.modelsError = "Failed to load models"

View File

@@ -7,6 +7,8 @@ struct DevicePresentation: Sendable {
enum DeviceModelCatalog {
private static let modelIdentifierToName: [String: String] = loadModelIdentifierToName()
private static let resourceBundle: Bundle? = locateResourceBundle()
private static let resourceSubdirectory = "DeviceModels"
static func presentation(deviceFamily: String?, modelIdentifier: String?) -> DevicePresentation? {
let family = (deviceFamily ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
@@ -104,13 +106,11 @@ enum DeviceModelCatalog {
}
private static func loadMapping(resourceName: String) -> [String: String] {
guard let url = self.resourceURL(
resourceName: resourceName,
guard let url = self.resourceBundle?.url(
forResource: resourceName,
withExtension: "json",
subdirectory: "DeviceModels")
else {
return [:]
}
subdirectory: self.resourceSubdirectory)
else { return [:] }
do {
let data = try Data(contentsOf: url)
@@ -121,37 +121,48 @@ enum DeviceModelCatalog {
}
}
private static func resourceURL(
resourceName: String,
withExtension ext: String,
subdirectory: String
) -> URL? {
let bundledSubdir = "Clawdis_Clawdis.bundle/\(subdirectory)"
let mainBundle = Bundle.main
if let url = mainBundle.url(forResource: resourceName, withExtension: ext, subdirectory: bundledSubdir)
?? mainBundle.url(forResource: resourceName, withExtension: ext, subdirectory: subdirectory)
{
return url
private static func locateResourceBundle() -> Bundle? {
if let bundle = self.bundleIfContainsDeviceModels(Bundle.module) {
return bundle
}
let fallbackBases = [
mainBundle.resourceURL,
mainBundle.bundleURL.appendingPathComponent("Contents/Resources"),
mainBundle.bundleURL.deletingLastPathComponent(),
].compactMap { $0 }
if let bundle = self.bundleIfContainsDeviceModels(Bundle.main) {
return bundle
}
let fileName = "\(resourceName).\(ext)"
for base in fallbackBases {
let bundled = base.appendingPathComponent(bundledSubdir).appendingPathComponent(fileName)
if FileManager.default.fileExists(atPath: bundled.path) { return bundled }
let loose = base.appendingPathComponent(subdirectory).appendingPathComponent(fileName)
if FileManager.default.fileExists(atPath: loose.path) { return loose }
if let resourceURL = Bundle.main.resourceURL {
if let enumerator = FileManager.default.enumerator(
at: resourceURL,
includingPropertiesForKeys: [.isDirectoryKey],
options: [.skipsHiddenFiles]) {
for case let url as URL in enumerator {
guard url.pathExtension == "bundle" else { continue }
if let bundle = Bundle(url: url),
self.bundleIfContainsDeviceModels(bundle) != nil {
return bundle
}
}
}
}
return nil
}
private static func bundleIfContainsDeviceModels(_ bundle: Bundle) -> Bundle? {
if bundle.url(
forResource: "ios-device-identifiers",
withExtension: "json",
subdirectory: self.resourceSubdirectory) != nil {
return bundle
}
if bundle.url(
forResource: "mac-device-identifiers",
withExtension: "json",
subdirectory: self.resourceSubdirectory) != nil {
return bundle
}
return nil
}
private enum NameValue: Decodable {
case string(String)
case stringArray([String])

View File

@@ -1,5 +1,4 @@
import AppKit
import OSLog
/// Central manager for Dock icon visibility.
/// Shows the Dock icon while any windows are visible, regardless of user preference.

View File

@@ -51,6 +51,7 @@ actor GatewayConnection {
case providersStatus = "providers.status"
case configGet = "config.get"
case configSet = "config.set"
case talkMode = "talk.mode"
case webLoginStart = "web.login.start"
case webLoginWait = "web.login.wait"
case webLogout = "web.logout"
@@ -472,7 +473,10 @@ extension GatewayConnection {
params["attachments"] = AnyCodable(encoded)
}
return try await self.requestDecoded(method: .chatSend, params: params)
return try await self.requestDecoded(
method: .chatSend,
params: params,
timeoutMs: Double(timeoutMs))
}
func chatAbort(sessionKey: String, runId: String) async throws -> Bool {
@@ -483,6 +487,12 @@ extension GatewayConnection {
return res.aborted ?? false
}
func talkMode(enabled: Bool, phase: String? = nil) async {
var params: [String: AnyCodable] = ["enabled": AnyCodable(enabled)]
if let phase { params["phase"] = AnyCodable(phase) }
try? await self.requestVoid(method: .talkMode, params: params)
}
// MARK: - VoiceWake
func voiceWakeGetTriggers() async throws -> [String] {

View File

@@ -1,6 +1,7 @@
import Foundation
enum GatewayLaunchAgentManager {
private static let logger = Logger(subsystem: "com.steipete.clawdis", category: "gateway.launchd")
private static let supportedBindModes: Set<String> = ["loopback", "tailnet", "lan", "auto"]
private static var plistURL: URL {
@@ -26,12 +27,16 @@ enum GatewayLaunchAgentManager {
if enabled {
let gatewayBin = self.gatewayExecutablePath(bundlePath: bundlePath)
guard FileManager.default.isExecutableFile(atPath: gatewayBin) else {
self.logger.error("launchd enable failed: gateway missing at \(gatewayBin)")
return "Embedded gateway missing in bundle; rebuild via scripts/package-mac-app.sh"
}
self.logger.info("launchd enable requested port=\(port)")
self.writePlist(bundlePath: bundlePath, port: port)
_ = await self.runLaunchctl(["bootout", "gui/\(getuid())/\(gatewayLaunchdLabel)"])
let bootstrap = await self.runLaunchctl(["bootstrap", "gui/\(getuid())", self.plistURL.path])
if bootstrap.status != 0 {
let msg = bootstrap.output.trimmingCharacters(in: .whitespacesAndNewlines)
self.logger.error("launchd bootstrap failed: \(msg)")
return bootstrap.output.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
? "Failed to bootstrap gateway launchd job"
: bootstrap.output.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -42,6 +47,7 @@ enum GatewayLaunchAgentManager {
return nil
}
self.logger.info("launchd disable requested")
_ = await self.runLaunchctl(["bootout", "gui/\(getuid())/\(gatewayLaunchdLabel)"])
try? FileManager.default.removeItem(at: self.plistURL)
return nil
@@ -103,7 +109,11 @@ enum GatewayLaunchAgentManager {
</dict>
</plist>
"""
try? plist.write(to: self.plistURL, atomically: true, encoding: .utf8)
do {
try plist.write(to: self.plistURL, atomically: true, encoding: .utf8)
} catch {
self.logger.error("launchd plist write failed: \(error.localizedDescription)")
}
}
private static func preferredGatewayBind() -> String? {

View File

@@ -42,6 +42,7 @@ final class GatewayProcessManager {
private var environmentRefreshTask: Task<Void, Never>?
private var lastEnvironmentRefresh: Date?
private var logRefreshTask: Task<Void, Never>?
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "gateway.process")
private let logLimit = 20000 // characters to keep in-memory
private let environmentRefreshMinInterval: TimeInterval = 30
@@ -53,8 +54,10 @@ final class GatewayProcessManager {
self.stop()
self.status = .stopped
self.appendLog("[gateway] remote mode active; skipping local gateway\n")
self.logger.info("gateway process skipped: remote mode active")
return
}
self.logger.debug("gateway active requested active=\(active)")
self.desiredActive = active
self.refreshEnvironmentStatus()
if active {
@@ -86,6 +89,7 @@ final class GatewayProcessManager {
return
}
self.status = .starting
self.logger.debug("gateway start requested")
// First try to latch onto an already-running gateway to avoid spawning a duplicate.
Task { [weak self] in
@@ -98,6 +102,7 @@ final class GatewayProcessManager {
await MainActor.run {
self.status = .failed("Attach-only enabled; no gateway to attach")
self.appendLog("[gateway] attach-only enabled; not spawning local gateway\n")
self.logger.warning("gateway attach-only enabled; not spawning")
}
return
}
@@ -110,6 +115,7 @@ final class GatewayProcessManager {
self.existingGatewayDetails = nil
self.lastFailureReason = nil
self.status = .stopped
self.logger.info("gateway stop requested")
let bundlePath = Bundle.main.bundleURL.path
Task {
_ = await GatewayLaunchAgentManager.set(
@@ -182,6 +188,7 @@ final class GatewayProcessManager {
self.existingGatewayDetails = details
self.status = .attachedExisting(details: details)
self.appendLog("[gateway] using existing instance: \(details)\n")
self.logger.info("gateway using existing instance details=\(details)")
self.refreshControlChannelIfNeeded(reason: "attach existing")
self.refreshLog()
return true
@@ -197,6 +204,7 @@ final class GatewayProcessManager {
self.status = .failed(reason)
self.lastFailureReason = reason
self.appendLog("[gateway] existing listener on port \(port) but attach failed: \(reason)\n")
self.logger.warning("gateway attach failed reason=\(reason)")
return true
}
@@ -268,16 +276,19 @@ final class GatewayProcessManager {
await MainActor.run {
self.status = .failed(resolution.status.message)
}
self.logger.error("gateway command resolve failed: \(resolution.status.message)")
return
}
let bundlePath = Bundle.main.bundleURL.path
let port = GatewayEnvironment.gatewayPort()
self.appendLog("[gateway] enabling launchd job (\(gatewayLaunchdLabel)) on port \(port)\n")
self.logger.info("gateway enabling launchd port=\(port)")
let err = await GatewayLaunchAgentManager.set(enabled: true, bundlePath: bundlePath, port: port)
if let err {
self.status = .failed(err)
self.lastFailureReason = err
self.logger.error("gateway launchd enable failed: \(err)")
return
}
@@ -290,6 +301,7 @@ final class GatewayProcessManager {
let instance = await PortGuardian.shared.describe(port: port)
let details = instance.map { "pid \($0.pid)" }
self.status = .running(details: details)
self.logger.info("gateway started details=\(details ?? "ok")")
self.refreshControlChannelIfNeeded(reason: "gateway started")
self.refreshLog()
return
@@ -300,6 +312,7 @@ final class GatewayProcessManager {
self.status = .failed("Gateway did not start in time")
self.lastFailureReason = "launchd start timeout"
self.logger.warning("gateway start timed out")
}
private func appendLog(_ chunk: String) {
@@ -317,6 +330,7 @@ final class GatewayProcessManager {
break
}
self.appendLog("[gateway] refreshing control channel (\(reason))\n")
self.logger.debug("gateway control channel refresh reason=\(reason)")
Task { await ControlChannel.shared.configure() }
}
@@ -332,12 +346,14 @@ final class GatewayProcessManager {
}
}
self.appendLog("[gateway] readiness wait timed out\n")
self.logger.warning("gateway readiness wait timed out")
return false
}
func clearLog() {
self.log = ""
try? FileManager.default.removeItem(atPath: LogLocator.launchdGatewayLogPath)
self.logger.debug("gateway log cleared")
}
func setProjectRoot(path: String) {

View File

@@ -1,7 +1,6 @@
import Foundation
import Network
import Observation
import OSLog
import SwiftUI
struct HealthSnapshot: Codable, Sendable {

View File

@@ -0,0 +1,229 @@
@_exported import Logging
import Foundation
import OSLog
typealias Logger = Logging.Logger
enum AppLogSettings {
static let logLevelKey = appLogLevelKey
static func logLevel() -> Logger.Level {
if let raw = UserDefaults.standard.string(forKey: self.logLevelKey),
let level = Logger.Level(rawValue: raw)
{
return level
}
return .info
}
static func setLogLevel(_ level: Logger.Level) {
UserDefaults.standard.set(level.rawValue, forKey: self.logLevelKey)
}
static func fileLoggingEnabled() -> Bool {
UserDefaults.standard.bool(forKey: debugFileLogEnabledKey)
}
}
enum AppLogLevel: String, CaseIterable, Identifiable {
case trace
case debug
case info
case notice
case warning
case error
case critical
static let `default`: AppLogLevel = .info
var id: String { self.rawValue }
var title: String {
switch self {
case .trace: "Trace"
case .debug: "Debug"
case .info: "Info"
case .notice: "Notice"
case .warning: "Warning"
case .error: "Error"
case .critical: "Critical"
}
}
}
enum ClawdisLogging {
private static let labelSeparator = "::"
private static let didBootstrap: Void = {
LoggingSystem.bootstrap { label in
let (subsystem, category) = Self.parseLabel(label)
let osHandler = ClawdisOSLogHandler(subsystem: subsystem, category: category)
let fileHandler = ClawdisFileLogHandler(label: label)
return MultiplexLogHandler([osHandler, fileHandler])
}
}()
static func bootstrapIfNeeded() {
_ = Self.didBootstrap
}
static func makeLabel(subsystem: String, category: String) -> String {
"\(subsystem)\(Self.labelSeparator)\(category)"
}
static func parseLabel(_ label: String) -> (String, String) {
guard let range = label.range(of: Self.labelSeparator) else {
return ("com.steipete.clawdis", label)
}
let subsystem = String(label[..<range.lowerBound])
let category = String(label[range.upperBound...])
return (subsystem, category)
}
}
extension Logging.Logger {
init(subsystem: String, category: String) {
ClawdisLogging.bootstrapIfNeeded()
let label = ClawdisLogging.makeLabel(subsystem: subsystem, category: category)
self.init(label: label)
}
}
extension Logger.Message.StringInterpolation {
mutating func appendInterpolation<T>(_ value: T, privacy: OSLogPrivacy) {
self.appendInterpolation(String(describing: value))
}
}
struct ClawdisOSLogHandler: LogHandler {
private let osLogger: OSLog.Logger
var metadata: Logger.Metadata = [:]
var logLevel: Logger.Level {
get { AppLogSettings.logLevel() }
set { AppLogSettings.setLogLevel(newValue) }
}
init(subsystem: String, category: String) {
self.osLogger = OSLog.Logger(subsystem: subsystem, category: category)
}
subscript(metadataKey key: String) -> Logger.Metadata.Value? {
get { self.metadata[key] }
set { self.metadata[key] = newValue }
}
func log(
level: Logger.Level,
message: Logger.Message,
metadata: Logger.Metadata?,
source: String,
file: String,
function: String,
line: UInt)
{
let merged = Self.mergeMetadata(self.metadata, metadata)
let rendered = Self.renderMessage(message, metadata: merged)
self.osLogger.log(level: Self.osLogType(for: level), "\(rendered, privacy: .public)")
}
private static func osLogType(for level: Logger.Level) -> OSLogType {
switch level {
case .trace, .debug:
return .debug
case .info, .notice:
return .info
case .warning:
return .default
case .error:
return .error
case .critical:
return .fault
}
}
private static func mergeMetadata(
_ base: Logger.Metadata,
_ extra: Logger.Metadata?) -> Logger.Metadata
{
guard let extra else { return base }
return base.merging(extra, uniquingKeysWith: { _, new in new })
}
private static func renderMessage(_ message: Logger.Message, metadata: Logger.Metadata) -> String {
guard !metadata.isEmpty else { return message.description }
let meta = metadata
.sorted(by: { $0.key < $1.key })
.map { "\($0.key)=\(stringify($0.value))" }
.joined(separator: " ")
return "\(message.description) [\(meta)]"
}
private static func stringify(_ value: Logger.Metadata.Value) -> String {
switch value {
case let .string(text):
text
case let .stringConvertible(value):
String(describing: value)
case let .array(values):
"[" + values.map { stringify($0) }.joined(separator: ",") + "]"
case let .dictionary(entries):
"{" + entries.map { "\($0.key)=\(stringify($0.value))" }.joined(separator: ",") + "}"
}
}
}
struct ClawdisFileLogHandler: LogHandler {
let label: String
var metadata: Logger.Metadata = [:]
var logLevel: Logger.Level {
get { AppLogSettings.logLevel() }
set { AppLogSettings.setLogLevel(newValue) }
}
subscript(metadataKey key: String) -> Logger.Metadata.Value? {
get { self.metadata[key] }
set { self.metadata[key] = newValue }
}
func log(
level: Logger.Level,
message: Logger.Message,
metadata: Logger.Metadata?,
source: String,
file: String,
function: String,
line: UInt)
{
guard AppLogSettings.fileLoggingEnabled() else { return }
let (subsystem, category) = ClawdisLogging.parseLabel(self.label)
var fields: [String: String] = [
"subsystem": subsystem,
"category": category,
"level": level.rawValue,
"source": source,
"file": file,
"function": function,
"line": "\(line)",
]
let merged = self.metadata.merging(metadata ?? [:], uniquingKeysWith: { _, new in new })
for (key, value) in merged {
fields["meta.\(key)"] = Self.stringify(value)
}
DiagnosticsFileLog.shared.log(category: category, event: message.description, fields: fields)
}
private static func stringify(_ value: Logger.Metadata.Value) -> String {
switch value {
case let .string(text):
text
case let .stringConvertible(value):
String(describing: value)
case let .array(values):
"[" + values.map { stringify($0) }.joined(separator: ",") + "]"
case let .dictionary(entries):
"{" + entries.map { "\($0.key)=\(stringify($0.value))" }.joined(separator: ",") + "}"
}
}
}

View File

@@ -3,7 +3,6 @@ import Darwin
import Foundation
import MenuBarExtraAccess
import Observation
import OSLog
import Security
import SwiftUI
@@ -30,6 +29,7 @@ struct ClawdisApp: App {
}
init() {
ClawdisLogging.bootstrapIfNeeded()
_state = State(initialValue: AppStateStore.shared)
}

View File

@@ -14,11 +14,14 @@ struct MenuContent: View {
private let heartbeatStore = HeartbeatStore.shared
private let controlChannel = ControlChannel.shared
private let activityStore = WorkActivityStore.shared
@Bindable private var pairingPrompter = NodePairingApprovalPrompter.shared
@Environment(\.openSettings) private var openSettings
@State private var availableMics: [AudioInputDevice] = []
@State private var loadingMics = false
@State private var browserControlEnabled = true
@AppStorage(cameraEnabledKey) private var cameraEnabled: Bool = false
@AppStorage(appLogLevelKey) private var appLogLevelRaw: String = AppLogLevel.default.rawValue
@AppStorage(debugFileLogEnabledKey) private var appFileLoggingEnabled: Bool = false
init(state: AppState, updater: UpdaterProviding?) {
self._state = Bindable(wrappedValue: state)
@@ -32,6 +35,13 @@ struct MenuContent: View {
VStack(alignment: .leading, spacing: 2) {
Text(self.connectionLabel)
self.statusLine(label: self.healthStatus.label, color: self.healthStatus.color)
if self.pairingPrompter.pendingCount > 0 {
let repairCount = self.pairingPrompter.pendingRepairCount
let repairSuffix = repairCount > 0 ? " · \(repairCount) repair" : ""
self.statusLine(
label: "Pairing approval pending (\(self.pairingPrompter.pendingCount))\(repairSuffix)",
color: .orange)
}
}
}
.disabled(self.state.connectionMode == .unconfigured)
@@ -102,6 +112,13 @@ struct MenuContent: View {
systemImage: "rectangle.inset.filled.on.rectangle")
}
}
Button {
Task { await self.state.setTalkEnabled(!self.state.talkEnabled) }
} label: {
Label(self.state.talkEnabled ? "Stop Talk Mode" : "Talk Mode", systemImage: "waveform.circle.fill")
}
.disabled(!voiceWakeSupported)
.opacity(voiceWakeSupported ? 1 : 0.5)
Divider()
Button("Settings…") { self.open(tab: .general) }
.keyboardShortcut(",", modifiers: [.command])
@@ -167,6 +184,20 @@ struct MenuContent: View {
: "Verbose Logging (Main): Off",
systemImage: "text.alignleft")
}
Menu("App Logging") {
Picker("Verbosity", selection: self.$appLogLevelRaw) {
ForEach(AppLogLevel.allCases) { level in
Text(level.title).tag(level.rawValue)
}
}
Toggle(isOn: self.$appFileLoggingEnabled) {
Label(
self.appFileLoggingEnabled
? "File Logging: On"
: "File Logging: Off",
systemImage: "doc.text.magnifyingglass")
}
}
Button {
DebugActions.openSessionStore()
} label: {
@@ -194,10 +225,12 @@ struct MenuContent: View {
Label("Send Test Notification", systemImage: "bell")
}
Divider()
Button {
DebugActions.restartGateway()
} label: {
Label("Restart Gateway", systemImage: "arrow.clockwise")
if self.state.connectionMode == .local, !AppStateStore.attachExistingGatewayOnly {
Button {
DebugActions.restartGateway()
} label: {
Label("Restart Gateway", systemImage: "arrow.clockwise")
}
}
Button {
DebugActions.restartApp()

View File

@@ -22,8 +22,7 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
private var cachedErrorText: String?
private var cacheUpdatedAt: Date?
private let refreshIntervalSeconds: TimeInterval = 12
private let nodesStore = InstancesStore.shared
private let gatewayDiscovery = GatewayDiscoveryModel()
private let nodesStore = NodesStore.shared
#if DEBUG
private var testControlChannelConnected: Bool?
#endif
@@ -43,7 +42,6 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
}
self.nodesStore.start()
self.gatewayDiscovery.start()
}
func menuWillOpen(_ menu: NSMenu) {
@@ -218,7 +216,7 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
}
if entries.isEmpty {
let title = self.nodesStore.isLoading ? "Loading nodes..." : "No nodes yet"
let title = self.nodesStore.isLoading ? "Loading devices..." : "No devices yet"
menu.insertItem(self.makeMessageItem(text: title, symbolName: "circle.dashed", width: width), at: cursor)
cursor += 1
} else {
@@ -231,7 +229,7 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
item.view = HighlightedMenuItemHostView(
rootView: AnyView(NodeMenuRowView(entry: entry, width: width)),
width: width)
item.submenu = self.buildNodeSubmenu(entry: entry)
item.submenu = self.buildNodeSubmenu(entry: entry, width: width)
menu.insertItem(item, at: cursor)
cursor += 1
}
@@ -239,7 +237,7 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
if entries.count > 8 {
let moreItem = NSMenuItem()
moreItem.tag = self.nodesTag
moreItem.title = "More Nodes..."
moreItem.title = "More Devices..."
moreItem.image = NSImage(systemSymbolName: "ellipsis.circle", accessibilityDescription: nil)
let overflow = Array(entries.dropFirst(8))
moreItem.submenu = self.buildNodesOverflowMenu(entries: overflow, width: width)
@@ -436,7 +434,7 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
return menu
}
private func buildNodesOverflowMenu(entries: [InstanceInfo], width: CGFloat) -> NSMenu {
private func buildNodesOverflowMenu(entries: [NodeInfo], width: CGFloat) -> NSMenu {
let menu = NSMenu()
for entry in entries {
let item = NSMenuItem()
@@ -446,27 +444,27 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
item.view = HighlightedMenuItemHostView(
rootView: AnyView(NodeMenuRowView(entry: entry, width: width)),
width: width)
item.submenu = self.buildNodeSubmenu(entry: entry)
item.submenu = self.buildNodeSubmenu(entry: entry, width: width)
menu.addItem(item)
}
return menu
}
private func buildNodeSubmenu(entry: InstanceInfo) -> NSMenu {
private func buildNodeSubmenu(entry: NodeInfo, width: CGFloat) -> NSMenu {
let menu = NSMenu()
menu.autoenablesItems = false
menu.addItem(self.makeNodeCopyItem(label: "ID", value: entry.id))
menu.addItem(self.makeNodeCopyItem(label: "Node ID", value: entry.nodeId))
if let host = entry.host?.nonEmpty {
menu.addItem(self.makeNodeCopyItem(label: "Host", value: host))
if let name = entry.displayName?.nonEmpty {
menu.addItem(self.makeNodeCopyItem(label: "Name", value: name))
}
if let ip = entry.ip?.nonEmpty {
if let ip = entry.remoteIp?.nonEmpty {
menu.addItem(self.makeNodeCopyItem(label: "IP", value: ip))
}
menu.addItem(self.makeNodeCopyItem(label: "Role", value: NodeMenuEntryFormatter.roleText(entry)))
menu.addItem(self.makeNodeCopyItem(label: "Status", value: NodeMenuEntryFormatter.roleText(entry)))
if let platform = NodeMenuEntryFormatter.platformText(entry) {
menu.addItem(self.makeNodeCopyItem(label: "Platform", value: platform))
@@ -476,19 +474,20 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
menu.addItem(self.makeNodeCopyItem(label: "Version", value: self.formatVersionLabel(version)))
}
menu.addItem(self.makeNodeDetailItem(label: "Last seen", value: entry.ageDescription))
menu.addItem(self.makeNodeDetailItem(label: "Connected", value: entry.isConnected ? "Yes" : "No"))
menu.addItem(self.makeNodeDetailItem(label: "Paired", value: entry.isPaired ? "Yes" : "No"))
if entry.lastInputSeconds != nil {
menu.addItem(self.makeNodeDetailItem(label: "Last input", value: entry.lastInputDescription))
if let caps = entry.caps?.filter({ !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }),
!caps.isEmpty {
menu.addItem(self.makeNodeCopyItem(label: "Caps", value: caps.joined(separator: ", ")))
}
if let reason = entry.reason?.nonEmpty {
menu.addItem(self.makeNodeDetailItem(label: "Reason", value: reason))
}
if let sshURL = self.sshURL(for: entry) {
menu.addItem(.separator())
menu.addItem(self.makeNodeActionItem(title: "Open SSH", url: sshURL))
if let commands = entry.commands?.filter({ !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }),
!commands.isEmpty {
menu.addItem(self.makeNodeMultilineItem(
label: "Commands",
value: commands.joined(separator: ", "),
width: width))
}
return menu
@@ -507,12 +506,17 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
return item
}
private func makeNodeActionItem(title: String, url: URL) -> NSMenuItem {
let item = NSMenuItem(title: title, action: #selector(self.openNodeSSH(_:)), keyEquivalent: "")
private func makeNodeMultilineItem(label: String, value: String, width: CGFloat) -> NSMenuItem {
let item = NSMenuItem()
item.target = self
item.representedObject = url
item.action = #selector(self.copyNodeValue(_:))
item.representedObject = value
item.view = HighlightedMenuItemHostView(
rootView: AnyView(NodeMenuMultilineView(label: label, value: value, width: width)),
width: width)
return item
}
private func formatVersionLabel(_ version: String) -> String {
let trimmed = version.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return version }
@@ -638,104 +642,6 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
NSPasteboard.general.setString(value, forType: .string)
}
@objc
private func openNodeSSH(_ sender: NSMenuItem) {
guard let url = sender.representedObject as? URL else { return }
if let appURL = self.preferredTerminalAppURL() {
NSWorkspace.shared.open(
[url],
withApplicationAt: appURL,
configuration: NSWorkspace.OpenConfiguration(),
completionHandler: nil)
} else {
NSWorkspace.shared.open(url)
}
}
private func preferredTerminalAppURL() -> URL? {
if let ghosty = self.ghostyAppURL() { return ghosty }
return NSWorkspace.shared.urlForApplication(withBundleIdentifier: "com.apple.Terminal")
}
private func ghostyAppURL() -> URL? {
let candidates = [
"/Applications/Ghosty.app",
("~/Applications/Ghosty.app" as NSString).expandingTildeInPath,
]
for path in candidates where FileManager.default.fileExists(atPath: path) {
return URL(fileURLWithPath: path)
}
return nil
}
private func sshURL(for entry: InstanceInfo) -> URL? {
guard NodeMenuEntryFormatter.isGateway(entry) else { return nil }
guard let gateway = self.matchingGateway(for: entry) else { return nil }
guard let host = self.sanitizedTailnetHost(gateway.tailnetDns) ?? gateway.lanHost else { return nil }
let user = NSUserName()
return self.buildSSHURL(user: user, host: host, port: gateway.sshPort)
}
private func matchingGateway(for entry: InstanceInfo) -> GatewayDiscoveryModel.DiscoveredGateway? {
let candidates = self.entryHostCandidates(entry)
guard !candidates.isEmpty else { return nil }
return self.gatewayDiscovery.gateways.first { gateway in
let gatewayTokens = self.gatewayHostTokens(gateway)
return candidates.contains { gatewayTokens.contains($0) }
}
}
private func entryHostCandidates(_ entry: InstanceInfo) -> [String] {
let raw: [String?] = [
entry.host,
entry.ip,
NodeMenuEntryFormatter.primaryName(entry),
]
return raw.compactMap(self.normalizedHostToken(_:))
}
private func gatewayHostTokens(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) -> [String] {
let raw: [String?] = [
gateway.displayName,
gateway.lanHost,
gateway.tailnetDns,
]
return raw.compactMap(self.normalizedHostToken(_:))
}
private func normalizedHostToken(_ value: String?) -> String? {
guard let value else { return nil }
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty { return nil }
let lower = trimmed.lowercased().trimmingCharacters(in: CharacterSet(charactersIn: "."))
if lower.hasSuffix(".localdomain") {
return lower.replacingOccurrences(of: ".localdomain", with: ".local")
}
return lower
}
private func sanitizedTailnetHost(_ host: String?) -> String? {
guard let host else { return nil }
let trimmed = host.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty { return nil }
if trimmed.hasSuffix(".internal.") || trimmed.hasSuffix(".internal") {
return nil
}
return trimmed
}
private func buildSSHURL(user: String, host: String, port: Int) -> URL? {
var components = URLComponents()
components.scheme = "ssh"
components.user = user
components.host = host
if port != 22 {
components.port = port
}
return components.url
}
// MARK: - Width + placement
private func findInsertIndex(in menu: NSMenu) -> Int? {
@@ -790,23 +696,14 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
return width
}
private func sortedNodeEntries() -> [InstanceInfo] {
let entries = self.nodesStore.instances.filter { entry in
let mode = entry.mode?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
return mode != "health"
}
private func sortedNodeEntries() -> [NodeInfo] {
let entries = self.nodesStore.nodes.filter { $0.isConnected }
return entries.sorted { lhs, rhs in
let lhsGateway = NodeMenuEntryFormatter.isGateway(lhs)
let rhsGateway = NodeMenuEntryFormatter.isGateway(rhs)
if lhsGateway != rhsGateway { return lhsGateway }
let lhsLocal = NodeMenuEntryFormatter.isLocal(lhs)
let rhsLocal = NodeMenuEntryFormatter.isLocal(rhs)
if lhsLocal != rhsLocal { return lhsLocal }
if lhs.isConnected != rhs.isConnected { return lhs.isConnected }
if lhs.isPaired != rhs.isPaired { return lhs.isPaired }
let lhsName = NodeMenuEntryFormatter.primaryName(lhs).lowercased()
let rhsName = NodeMenuEntryFormatter.primaryName(rhs).lowercased()
if lhsName == rhsName { return lhs.ts > rhs.ts }
if lhsName == rhsName { return lhs.nodeId < rhs.nodeId }
return lhsName < rhsName
}
}

View File

@@ -4,18 +4,23 @@ import JavaScriptCore
enum ModelCatalogLoader {
static let defaultPath: String = FileManager.default.homeDirectoryForCurrentUser
.appendingPathComponent("Projects/pi-mono/packages/ai/src/models.generated.ts").path
private static let logger = Logger(subsystem: "com.steipete.clawdis", category: "models")
static func load(from path: String) async throws -> [ModelChoice] {
let expanded = (path as NSString).expandingTildeInPath
self.logger.debug("model catalog load start file=\(URL(fileURLWithPath: expanded).lastPathComponent)")
let source = try String(contentsOfFile: expanded, encoding: .utf8)
let sanitized = self.sanitize(source: source)
let ctx = JSContext()
ctx?.exceptionHandler = { _, exception in
if let exception { print("JS exception: \(exception)") }
if let exception {
self.logger.warning("model catalog JS exception: \(exception)")
}
}
ctx?.evaluateScript(sanitized)
guard let rawModels = ctx?.objectForKeyedSubscript("MODELS")?.toDictionary() as? [String: Any] else {
self.logger.error("model catalog parse failed: MODELS missing")
throw NSError(
domain: "ModelCatalogLoader",
code: 1,
@@ -33,12 +38,14 @@ enum ModelCatalogLoader {
}
}
return choices.sorted { lhs, rhs in
let sorted = choices.sorted { lhs, rhs in
if lhs.provider == rhs.provider {
return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending
}
return lhs.provider.localizedCaseInsensitiveCompare(rhs.provider) == .orderedAscending
}
self.logger.debug("model catalog loaded providers=\(rawModels.count) models=\(sorted.count)")
return sorted
}
private static func sanitize(source: String) -> String {

View File

@@ -265,7 +265,7 @@ actor MacNodeRuntime {
guard let raw = await GatewayConnection.shared.canvasHostUrl() else { return nil }
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, let baseUrl = URL(string: trimmed) else { return nil }
return baseUrl.appendingPathComponent("__clawdis__/a2ui/").absoluteString
return baseUrl.appendingPathComponent("__clawdis__/a2ui/").absoluteString + "?platform=macos"
}
private func isA2UIReady(poll: Bool = false) async -> Bool {

View File

@@ -2,6 +2,7 @@ import AppKit
import ClawdisIPC
import ClawdisProtocol
import Foundation
import Observation
import OSLog
import UserNotifications
@@ -15,6 +16,7 @@ enum NodePairingReconcilePolicy {
}
@MainActor
@Observable
final class NodePairingApprovalPrompter {
static let shared = NodePairingApprovalPrompter()
@@ -26,6 +28,8 @@ final class NodePairingApprovalPrompter {
private var isStopping = false
private var isPresenting = false
private var queue: [PendingRequest] = []
var pendingCount: Int = 0
var pendingRepairCount: Int = 0
private var activeAlert: NSAlert?
private var activeRequestId: String?
private var alertHostWindow: NSWindow?
@@ -104,6 +108,7 @@ final class NodePairingApprovalPrompter {
self.reconcileOnceTask?.cancel()
self.reconcileOnceTask = nil
self.queue.removeAll(keepingCapacity: false)
self.updatePendingCounts()
self.isPresenting = false
self.activeRequestId = nil
self.alertHostWindow?.orderOut(nil)
@@ -292,6 +297,7 @@ final class NodePairingApprovalPrompter {
private func enqueue(_ req: PendingRequest) {
if self.queue.contains(req) { return }
self.queue.append(req)
self.updatePendingCounts()
self.presentNextIfNeeded()
self.updateReconcileLoop()
}
@@ -362,6 +368,7 @@ final class NodePairingApprovalPrompter {
} else {
self.queue.removeAll { $0 == request }
}
self.updatePendingCounts()
self.isPresenting = false
self.presentNextIfNeeded()
self.updateReconcileLoop()
@@ -501,6 +508,8 @@ final class NodePairingApprovalPrompter {
} else {
self.queue.removeAll { $0 == req }
}
self.updatePendingCounts()
self.isPresenting = false
self.presentNextIfNeeded()
self.updateReconcileLoop()
@@ -599,6 +608,12 @@ final class NodePairingApprovalPrompter {
}
}
private func updatePendingCounts() {
// Keep a cheap observable summary for the menu bar status line.
self.pendingCount = self.queue.count
self.pendingRepairCount = self.queue.filter { $0.isRepair == true }.count
}
private func reconcileOnce(timeoutMs: Double) async {
if self.isStopping { return }
if self.reconcileInFlight { return }
@@ -643,6 +658,7 @@ final class NodePairingApprovalPrompter {
return
}
self.queue.removeAll { $0.requestId == resolved.requestId }
self.updatePendingCounts()
Task { @MainActor in
await self.notify(resolution: resolution, request: request, via: "remote")
}

View File

@@ -2,51 +2,53 @@ import AppKit
import SwiftUI
struct NodeMenuEntryFormatter {
static func isGateway(_ entry: InstanceInfo) -> Bool {
entry.mode?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "gateway"
static func isConnected(_ entry: NodeInfo) -> Bool {
entry.isConnected
}
static func isLocal(_ entry: InstanceInfo) -> Bool {
entry.mode?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "local"
static func primaryName(_ entry: NodeInfo) -> String {
entry.displayName?.nonEmpty ?? entry.nodeId
}
static func primaryName(_ entry: InstanceInfo) -> String {
if self.isGateway(entry) {
let host = entry.host?.nonEmpty
if let host, host.lowercased() != "gateway" { return host }
return "Gateway"
static func summaryText(_ entry: NodeInfo) -> String {
let name = self.primaryName(entry)
var prefix = "Node: \(name)"
if let ip = entry.remoteIp?.nonEmpty {
prefix += " (\(ip))"
}
var parts = [prefix]
if let platform = self.platformText(entry) {
parts.append("platform \(platform)")
}
return entry.host?.nonEmpty ?? entry.id
}
static func summaryText(_ entry: InstanceInfo) -> String {
entry.text.nonEmpty ?? self.primaryName(entry)
}
static func roleText(_ entry: InstanceInfo) -> String {
if self.isGateway(entry) { return "gateway" }
if let mode = entry.mode?.nonEmpty { return mode }
return "node"
}
static func detailLeft(_ entry: InstanceInfo) -> String {
let role = self.roleText(entry)
if let ip = entry.ip?.nonEmpty { return "\(ip) · \(role)" }
return role
}
static func detailRight(_ entry: InstanceInfo) -> String? {
var parts: [String] = []
if let platform = self.platformText(entry) { parts.append(platform) }
if let version = entry.version?.nonEmpty {
let short = self.compactVersion(version)
parts.append("v\(short)")
parts.append("app \(self.compactVersion(version))")
}
if parts.isEmpty { return nil }
parts.append("status \(self.roleText(entry))")
return parts.joined(separator: " · ")
}
static func platformText(_ entry: InstanceInfo) -> String? {
static func roleText(_ entry: NodeInfo) -> String {
if entry.isConnected { return "connected" }
if entry.isPaired { return "paired" }
return "unpaired"
}
static func detailLeft(_ entry: NodeInfo) -> String {
let role = self.roleText(entry)
if let ip = entry.remoteIp?.nonEmpty { return "\(ip) · \(role)" }
return role
}
static func headlineRight(_ entry: NodeInfo) -> String? {
self.platformText(entry)
}
static func detailRightVersion(_ entry: NodeInfo) -> String? {
guard let version = entry.version?.nonEmpty else { return nil }
return self.shortVersionLabel(version)
}
static func platformText(_ entry: NodeInfo) -> String? {
if let raw = entry.platform?.nonEmpty {
return self.prettyPlatform(raw) ?? raw
}
@@ -99,8 +101,17 @@ struct NodeMenuEntryFormatter {
return trimmed
}
static func leadingSymbol(_ entry: InstanceInfo) -> String {
if self.isGateway(entry) { return self.safeSystemSymbol("dot.radiowaves.left.and.right", fallback: "network") }
private static func shortVersionLabel(_ raw: String) -> String {
let compact = self.compactVersion(raw)
if compact.isEmpty { return compact }
if compact.lowercased().hasPrefix("v") { return compact }
if let first = compact.unicodeScalars.first, CharacterSet.decimalDigits.contains(first) {
return "v\(compact)"
}
return compact
}
static func leadingSymbol(_ entry: NodeInfo) -> String {
if let family = entry.deviceFamily?.lowercased() {
if family.contains("mac") {
return self.safeSystemSymbol("laptopcomputer", fallback: "laptopcomputer")
@@ -116,9 +127,11 @@ struct NodeMenuEntryFormatter {
return "cpu"
}
static func isAndroid(_ entry: InstanceInfo) -> Bool {
static func isAndroid(_ entry: NodeInfo) -> Bool {
let family = entry.deviceFamily?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
return family == "android"
if family == "android" { return true }
let platform = entry.platform?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
return platform?.contains("android") == true
}
private static func safeSystemSymbol(_ preferred: String, fallback: String) -> String {
@@ -128,7 +141,7 @@ struct NodeMenuEntryFormatter {
}
struct NodeMenuRowView: View {
let entry: InstanceInfo
let entry: NodeInfo
let width: CGFloat
@Environment(\.menuItemHighlighted) private var isHighlighted
@@ -146,11 +159,32 @@ struct NodeMenuRowView: View {
.frame(width: 22, height: 22, alignment: .center)
VStack(alignment: .leading, spacing: 2) {
Text(NodeMenuEntryFormatter.primaryName(self.entry))
.font(.callout.weight(NodeMenuEntryFormatter.isGateway(self.entry) ? .semibold : .regular))
.foregroundStyle(self.primaryColor)
.lineLimit(1)
.truncationMode(.middle)
HStack(alignment: .firstTextBaseline, spacing: 8) {
Text(NodeMenuEntryFormatter.primaryName(self.entry))
.font(.callout.weight(NodeMenuEntryFormatter.isConnected(self.entry) ? .semibold : .regular))
.foregroundStyle(self.primaryColor)
.lineLimit(1)
.truncationMode(.middle)
.layoutPriority(1)
Spacer(minLength: 8)
HStack(alignment: .firstTextBaseline, spacing: 6) {
if let right = NodeMenuEntryFormatter.headlineRight(self.entry) {
Text(right)
.font(.caption.monospacedDigit())
.foregroundStyle(self.secondaryColor)
.lineLimit(1)
.truncationMode(.middle)
.layoutPriority(2)
}
Image(systemName: "chevron.right")
.font(.caption.weight(.semibold))
.foregroundStyle(self.secondaryColor)
.padding(.leading, 2)
}
}
HStack(alignment: .firstTextBaseline, spacing: 8) {
Text(NodeMenuEntryFormatter.detailLeft(self.entry))
@@ -161,21 +195,15 @@ struct NodeMenuRowView: View {
Spacer(minLength: 0)
HStack(alignment: .firstTextBaseline, spacing: 6) {
if let right = NodeMenuEntryFormatter.detailRight(self.entry) {
Text(right)
.font(.caption.monospacedDigit())
.foregroundStyle(self.secondaryColor)
.lineLimit(1)
.truncationMode(.middle)
}
Image(systemName: "chevron.right")
.font(.caption.weight(.semibold))
if let version = NodeMenuEntryFormatter.detailRightVersion(self.entry) {
Text(version)
.font(.caption.monospacedDigit())
.foregroundStyle(self.secondaryColor)
.padding(.leading, 2)
.lineLimit(1)
.truncationMode(.middle)
}
}
.frame(maxWidth: .infinity, alignment: .leading)
}
.frame(maxWidth: .infinity, alignment: .leading)
@@ -215,3 +243,36 @@ struct AndroidMark: View {
}
}
}
struct NodeMenuMultilineView: View {
let label: String
let value: String
let width: CGFloat
@Environment(\.menuItemHighlighted) private var isHighlighted
private var primaryColor: Color {
self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor) : .primary
}
private var secondaryColor: Color {
self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor).opacity(0.85) : .secondary
}
var body: some View {
VStack(alignment: .leading, spacing: 4) {
Text("\(self.label):")
.font(.caption.weight(.semibold))
.foregroundStyle(self.secondaryColor)
Text(self.value)
.font(.caption)
.foregroundStyle(self.primaryColor)
.multilineTextAlignment(.leading)
.fixedSize(horizontal: false, vertical: true)
}
.padding(.vertical, 6)
.padding(.leading, 18)
.padding(.trailing, 12)
.frame(width: max(1, self.width), alignment: .leading)
}
}

View File

@@ -0,0 +1,84 @@
import Foundation
import Observation
import OSLog
struct NodeInfo: Identifiable, Codable {
let nodeId: String
let displayName: String?
let platform: String?
let version: String?
let deviceFamily: String?
let modelIdentifier: String?
let remoteIp: String?
let caps: [String]?
let commands: [String]?
let permissions: [String: Bool]?
let paired: Bool?
let connected: Bool?
var id: String { self.nodeId }
var isConnected: Bool { self.connected ?? false }
var isPaired: Bool { self.paired ?? false }
}
private struct NodeListResponse: Codable {
let ts: Double?
let nodes: [NodeInfo]
}
@MainActor
@Observable
final class NodesStore {
static let shared = NodesStore()
var nodes: [NodeInfo] = []
var lastError: String?
var statusMessage: String?
var isLoading = false
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "nodes")
private var task: Task<Void, Never>?
private let interval: TimeInterval = 30
private var startCount = 0
func start() {
self.startCount += 1
guard self.startCount == 1 else { return }
guard self.task == nil else { return }
self.task = Task.detached { [weak self] in
guard let self else { return }
await self.refresh()
while !Task.isCancelled {
try? await Task.sleep(nanoseconds: UInt64(self.interval * 1_000_000_000))
await self.refresh()
}
}
}
func stop() {
guard self.startCount > 0 else { return }
self.startCount -= 1
guard self.startCount == 0 else { return }
self.task?.cancel()
self.task = nil
}
func refresh() async {
if self.isLoading { return }
self.statusMessage = nil
self.isLoading = true
defer { self.isLoading = false }
do {
let data = try await GatewayConnection.shared.requestRaw(method: "node.list", params: nil, timeoutMs: 8000)
let decoded = try JSONDecoder().decode(NodeListResponse.self, from: data)
self.nodes = decoded.nodes
self.lastError = nil
self.statusMessage = nil
} catch {
self.logger.error("node.list failed \(error.localizedDescription, privacy: .public)")
self.nodes = []
self.lastError = error.localizedDescription
self.statusMessage = nil
}
}
}

View File

@@ -5,6 +5,8 @@ import UserNotifications
@MainActor
struct NotificationManager {
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "notifications")
private static let hasTimeSensitiveEntitlement: Bool = {
guard let task = SecTaskCreateFromSelf(nil) else { return false }
let key = "com.apple.developer.usernotifications.time-sensitive" as CFString
@@ -17,8 +19,12 @@ struct NotificationManager {
let status = await center.notificationSettings()
if status.authorizationStatus == .notDetermined {
let granted = try? await center.requestAuthorization(options: [.alert, .sound, .badge])
if granted != true { return false }
if granted != true {
self.logger.warning("notification permission denied (request)")
return false
}
} else if status.authorizationStatus != .authorized {
self.logger.warning("notification permission denied status=\(status.authorizationStatus.rawValue)")
return false
}
@@ -37,15 +43,22 @@ struct NotificationManager {
case .active:
content.interruptionLevel = .active
case .timeSensitive:
content.interruptionLevel = Self.hasTimeSensitiveEntitlement ? .timeSensitive : .active
if Self.hasTimeSensitiveEntitlement {
content.interruptionLevel = .timeSensitive
} else {
self.logger.debug("time-sensitive notification requested without entitlement; falling back to active")
content.interruptionLevel = .active
}
}
}
let req = UNNotificationRequest(identifier: UUID().uuidString, content: content, trigger: nil)
do {
try await center.add(req)
self.logger.debug("notification queued")
return true
} catch {
self.logger.error("notification send failed: \(error.localizedDescription)")
return false
}
}

View File

@@ -5,7 +5,6 @@ import ClawdisIPC
import CoreGraphics
import Foundation
import Observation
import OSLog
import Speech
import UserNotifications

View File

@@ -21,7 +21,6 @@ struct SettingsRootView: View {
if self.isNixMode {
self.nixManagedBanner
}
TabView(selection: self.$selectedTab) {
GeneralSettings(state: self.state)
.tabItem { Label("General", systemImage: "gearshape") }
@@ -63,7 +62,7 @@ struct SettingsRootView: View {
.tag(SettingsTab.permissions)
if self.state.debugPaneEnabled {
DebugSettings()
DebugSettings(state: self.state)
.tabItem { Label("Debug", systemImage: "ant") }
.tag(SettingsTab.debug)
}

View File

@@ -0,0 +1,158 @@
import AVFoundation
import Foundation
import OSLog
@MainActor
final class TalkAudioPlayer: NSObject, @preconcurrency AVAudioPlayerDelegate {
static let shared = TalkAudioPlayer()
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "talk.tts")
private var player: AVAudioPlayer?
private var playback: Playback?
private final class Playback: @unchecked Sendable {
private let lock = NSLock()
private var finished = false
private var continuation: CheckedContinuation<TalkPlaybackResult, Never>?
private var watchdog: Task<Void, Never>?
func setContinuation(_ continuation: CheckedContinuation<TalkPlaybackResult, Never>) {
self.lock.lock()
defer { self.lock.unlock() }
self.continuation = continuation
}
func setWatchdog(_ task: Task<Void, Never>?) {
self.lock.lock()
let old = self.watchdog
self.watchdog = task
self.lock.unlock()
old?.cancel()
}
func cancelWatchdog() {
self.setWatchdog(nil)
}
func finish(_ result: TalkPlaybackResult) {
let continuation: CheckedContinuation<TalkPlaybackResult, Never>?
self.lock.lock()
if self.finished {
continuation = nil
} else {
self.finished = true
continuation = self.continuation
self.continuation = nil
}
self.lock.unlock()
continuation?.resume(returning: result)
}
}
func play(data: Data) async -> TalkPlaybackResult {
self.stopInternal()
let playback = Playback()
self.playback = playback
return await withCheckedContinuation { continuation in
playback.setContinuation(continuation)
do {
let player = try AVAudioPlayer(data: data)
self.player = player
player.delegate = self
player.prepareToPlay()
self.armWatchdog(playback: playback)
let ok = player.play()
if !ok {
self.logger.error("talk audio player refused to play")
self.finish(playback: playback, result: TalkPlaybackResult(finished: false, interruptedAt: nil))
}
} catch {
self.logger.error("talk audio player failed: \(error.localizedDescription, privacy: .public)")
self.finish(playback: playback, result: TalkPlaybackResult(finished: false, interruptedAt: nil))
}
}
}
func stop() -> Double? {
guard let player else { return nil }
let time = player.currentTime
self.stopInternal(interruptedAt: time)
return time
}
func audioPlayerDidFinishPlaying(_: AVAudioPlayer, successfully flag: Bool) {
self.stopInternal(finished: flag)
}
private func stopInternal(finished: Bool = false, interruptedAt: Double? = nil) {
guard let playback else { return }
let result = TalkPlaybackResult(finished: finished, interruptedAt: interruptedAt)
self.finish(playback: playback, result: result)
}
private func finish(playback: Playback, result: TalkPlaybackResult) {
playback.cancelWatchdog()
playback.finish(result)
guard self.playback === playback else { return }
self.playback = nil
self.player?.stop()
self.player = nil
}
private func stopInternal() {
if let playback = self.playback {
let interruptedAt = self.player?.currentTime
self.finish(
playback: playback,
result: TalkPlaybackResult(finished: false, interruptedAt: interruptedAt))
return
}
self.player?.stop()
self.player = nil
}
private func armWatchdog(playback: Playback) {
playback.setWatchdog(Task { @MainActor [weak self] in
guard let self else { return }
do {
try await Task.sleep(nanoseconds: 650_000_000)
} catch {
return
}
if Task.isCancelled { return }
guard self.playback === playback else { return }
if self.player?.isPlaying != true {
self.logger.error("talk audio player did not start playing")
self.finish(playback: playback, result: TalkPlaybackResult(finished: false, interruptedAt: nil))
return
}
let duration = self.player?.duration ?? 0
let timeoutSeconds = min(max(2.0, duration + 2.0), 5 * 60.0)
do {
try await Task.sleep(nanoseconds: UInt64(timeoutSeconds * 1_000_000_000))
} catch {
return
}
if Task.isCancelled { return }
guard self.playback === playback else { return }
guard self.player?.isPlaying == true else { return }
self.logger.error("talk audio player watchdog fired")
self.finish(playback: playback, result: TalkPlaybackResult(finished: false, interruptedAt: nil))
})
}
}
struct TalkPlaybackResult: Sendable {
let finished: Bool
let interruptedAt: Double?
}

View File

@@ -0,0 +1,61 @@
import Observation
@MainActor
@Observable
final class TalkModeController {
static let shared = TalkModeController()
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "talk.controller")
private(set) var phase: TalkModePhase = .idle
private(set) var isPaused: Bool = false
func setEnabled(_ enabled: Bool) async {
self.logger.info("talk enabled=\(enabled)")
if enabled {
TalkOverlayController.shared.present()
} else {
TalkOverlayController.shared.dismiss()
}
await TalkModeRuntime.shared.setEnabled(enabled)
}
func updatePhase(_ phase: TalkModePhase) {
self.phase = phase
TalkOverlayController.shared.updatePhase(phase)
let effectivePhase = self.isPaused ? "paused" : phase.rawValue
Task { await GatewayConnection.shared.talkMode(enabled: AppStateStore.shared.talkEnabled, phase: effectivePhase) }
}
func updateLevel(_ level: Double) {
TalkOverlayController.shared.updateLevel(level)
}
func setPaused(_ paused: Bool) {
guard self.isPaused != paused else { return }
self.logger.info("talk paused=\(paused)")
self.isPaused = paused
TalkOverlayController.shared.updatePaused(paused)
let effectivePhase = paused ? "paused" : self.phase.rawValue
Task { await GatewayConnection.shared.talkMode(enabled: AppStateStore.shared.talkEnabled, phase: effectivePhase) }
Task { await TalkModeRuntime.shared.setPaused(paused) }
}
func togglePaused() {
self.setPaused(!self.isPaused)
}
func stopSpeaking(reason: TalkStopReason = .userTap) {
Task { await TalkModeRuntime.shared.stopSpeaking(reason: reason) }
}
func exitTalkMode() {
Task { await AppStateStore.shared.setTalkEnabled(false) }
}
}
enum TalkStopReason {
case userTap
case speech
case manual
}

View File

@@ -0,0 +1,890 @@
import AVFoundation
import ClawdisChatUI
import ClawdisKit
import Foundation
import OSLog
import Speech
actor TalkModeRuntime {
static let shared = TalkModeRuntime()
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "talk.runtime")
private let ttsLogger = Logger(subsystem: "com.steipete.clawdis", category: "talk.tts")
private static let defaultModelIdFallback = "eleven_v3"
private final class RMSMeter: @unchecked Sendable {
private let lock = NSLock()
private var latestRMS: Double = 0
func set(_ rms: Double) {
self.lock.lock()
self.latestRMS = rms
self.lock.unlock()
}
func get() -> Double {
self.lock.lock()
let value = self.latestRMS
self.lock.unlock()
return value
}
}
private var recognizer: SFSpeechRecognizer?
private var audioEngine: AVAudioEngine?
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
private var recognitionTask: SFSpeechRecognitionTask?
private var recognitionGeneration: Int = 0
private var rmsTask: Task<Void, Never>?
private let rmsMeter = RMSMeter()
private var captureTask: Task<Void, Never>?
private var silenceTask: Task<Void, Never>?
private var phase: TalkModePhase = .idle
private var isEnabled = false
private var isPaused = false
private var lifecycleGeneration: Int = 0
private var lastHeard: Date?
private var noiseFloorRMS: Double = 1e-4
private var lastTranscript: String = ""
private var lastSpeechEnergyAt: Date?
private var defaultVoiceId: String?
private var currentVoiceId: String?
private var defaultModelId: String?
private var currentModelId: String?
private var voiceOverrideActive = false
private var modelOverrideActive = false
private var defaultOutputFormat: String?
private var interruptOnSpeech: Bool = true
private var lastInterruptedAtSeconds: Double?
private var voiceAliases: [String: String] = [:]
private var lastSpokenText: String?
private var apiKey: String?
private var fallbackVoiceId: String?
private var lastPlaybackWasPCM: Bool = false
private let silenceWindow: TimeInterval = 0.7
private let minSpeechRMS: Double = 1e-3
private let speechBoostFactor: Double = 6.0
// MARK: - Lifecycle
func setEnabled(_ enabled: Bool) async {
guard enabled != self.isEnabled else { return }
self.isEnabled = enabled
self.lifecycleGeneration &+= 1
if enabled {
await self.start()
} else {
await self.stop()
}
}
func setPaused(_ paused: Bool) async {
guard paused != self.isPaused else { return }
self.isPaused = paused
await MainActor.run { TalkModeController.shared.updateLevel(0) }
guard self.isEnabled else { return }
if paused {
self.lastTranscript = ""
self.lastHeard = nil
self.lastSpeechEnergyAt = nil
await self.stopRecognition()
return
}
if self.phase == .idle || self.phase == .listening {
await self.startRecognition()
self.phase = .listening
await MainActor.run { TalkModeController.shared.updatePhase(.listening) }
self.startSilenceMonitor()
}
}
private func isCurrent(_ generation: Int) -> Bool {
generation == self.lifecycleGeneration && self.isEnabled
}
private func start() async {
let gen = self.lifecycleGeneration
guard voiceWakeSupported else { return }
guard PermissionManager.voiceWakePermissionsGranted() else {
self.logger.debug("talk runtime not starting: permissions missing")
return
}
await self.reloadConfig()
guard self.isCurrent(gen) else { return }
if self.isPaused {
self.phase = .idle
await MainActor.run {
TalkModeController.shared.updateLevel(0)
TalkModeController.shared.updatePhase(.idle)
}
return
}
await self.startRecognition()
guard self.isCurrent(gen) else { return }
self.phase = .listening
await MainActor.run { TalkModeController.shared.updatePhase(.listening) }
self.startSilenceMonitor()
}
private func stop() async {
self.captureTask?.cancel()
self.captureTask = nil
self.silenceTask?.cancel()
self.silenceTask = nil
// Stop audio before changing phase (stopSpeaking is gated on .speaking).
await self.stopSpeaking(reason: .manual)
self.lastTranscript = ""
self.lastHeard = nil
self.lastSpeechEnergyAt = nil
self.phase = .idle
await self.stopRecognition()
await MainActor.run {
TalkModeController.shared.updateLevel(0)
TalkModeController.shared.updatePhase(.idle)
}
}
// MARK: - Speech recognition
private struct RecognitionUpdate {
let transcript: String?
let hasConfidence: Bool
let isFinal: Bool
let errorDescription: String?
let generation: Int
}
private func startRecognition() async {
await self.stopRecognition()
self.recognitionGeneration &+= 1
let generation = self.recognitionGeneration
let locale = await MainActor.run { AppStateStore.shared.voiceWakeLocaleID }
self.recognizer = SFSpeechRecognizer(locale: Locale(identifier: locale))
guard let recognizer, recognizer.isAvailable else {
self.logger.error("talk recognizer unavailable")
return
}
self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
self.recognitionRequest?.shouldReportPartialResults = true
guard let request = self.recognitionRequest else { return }
if self.audioEngine == nil {
self.audioEngine = AVAudioEngine()
}
guard let audioEngine = self.audioEngine else { return }
let input = audioEngine.inputNode
let format = input.outputFormat(forBus: 0)
input.removeTap(onBus: 0)
let meter = self.rmsMeter
input.installTap(onBus: 0, bufferSize: 2048, format: format) { [weak request, meter] buffer, _ in
request?.append(buffer)
if let rms = Self.rmsLevel(buffer: buffer) {
meter.set(rms)
}
}
audioEngine.prepare()
do {
try audioEngine.start()
} catch {
self.logger.error("talk audio engine start failed: \(error.localizedDescription, privacy: .public)")
return
}
self.startRMSTicker(meter: meter)
self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self, generation] result, error in
guard let self else { return }
let segments = result?.bestTranscription.segments ?? []
let transcript = result?.bestTranscription.formattedString
let update = RecognitionUpdate(
transcript: transcript,
hasConfidence: segments.contains { $0.confidence > 0.6 },
isFinal: result?.isFinal ?? false,
errorDescription: error?.localizedDescription,
generation: generation)
Task { await self.handleRecognition(update) }
}
}
private func stopRecognition() async {
self.recognitionGeneration &+= 1
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.recognitionRequest?.endAudio()
self.recognitionRequest = nil
self.audioEngine?.inputNode.removeTap(onBus: 0)
self.audioEngine?.stop()
self.audioEngine = nil
self.recognizer = nil
self.rmsTask?.cancel()
self.rmsTask = nil
}
private func startRMSTicker(meter: RMSMeter) {
self.rmsTask?.cancel()
self.rmsTask = Task { [weak self, meter] in
while let self {
try? await Task.sleep(nanoseconds: 50_000_000)
if Task.isCancelled { return }
await self.noteAudioLevel(rms: meter.get())
}
}
}
private func handleRecognition(_ update: RecognitionUpdate) async {
guard update.generation == self.recognitionGeneration else { return }
guard !self.isPaused else { return }
if let errorDescription = update.errorDescription {
self.logger.debug("talk recognition error: \(errorDescription, privacy: .public)")
}
guard let transcript = update.transcript else { return }
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
if self.phase == .speaking, self.interruptOnSpeech {
if await self.shouldInterrupt(transcript: trimmed, hasConfidence: update.hasConfidence) {
await self.stopSpeaking(reason: .speech)
self.lastTranscript = ""
self.lastHeard = nil
await self.startListening()
}
return
}
guard self.phase == .listening else { return }
if !trimmed.isEmpty {
self.lastTranscript = trimmed
self.lastHeard = Date()
}
if update.isFinal {
self.lastTranscript = trimmed
}
}
// MARK: - Silence handling
private func startSilenceMonitor() {
self.silenceTask?.cancel()
self.silenceTask = Task { [weak self] in
await self?.silenceLoop()
}
}
private func silenceLoop() async {
while self.isEnabled {
try? await Task.sleep(nanoseconds: 200_000_000)
await self.checkSilence()
}
}
private func checkSilence() async {
guard !self.isPaused else { return }
guard self.phase == .listening else { return }
let transcript = self.lastTranscript.trimmingCharacters(in: .whitespacesAndNewlines)
guard !transcript.isEmpty else { return }
guard let lastHeard else { return }
let elapsed = Date().timeIntervalSince(lastHeard)
guard elapsed >= self.silenceWindow else { return }
await self.finalizeTranscript(transcript)
}
private func startListening() async {
self.phase = .listening
self.lastTranscript = ""
self.lastHeard = nil
await MainActor.run {
TalkModeController.shared.updatePhase(.listening)
TalkModeController.shared.updateLevel(0)
}
}
private func finalizeTranscript(_ text: String) async {
self.lastTranscript = ""
self.lastHeard = nil
self.phase = .thinking
await MainActor.run { TalkModeController.shared.updatePhase(.thinking) }
await self.stopRecognition()
await self.sendAndSpeak(text)
}
// MARK: - Gateway + TTS
private func sendAndSpeak(_ transcript: String) async {
let gen = self.lifecycleGeneration
await self.reloadConfig()
guard self.isCurrent(gen) else { return }
let prompt = self.buildPrompt(transcript: transcript)
let activeSessionKey = await MainActor.run { WebChatManager.shared.activeSessionKey }
let sessionKey: String = if let activeSessionKey {
activeSessionKey
} else {
await GatewayConnection.shared.mainSessionKey()
}
let runId = UUID().uuidString
let startedAt = Date().timeIntervalSince1970
self.logger.info(
"talk send start runId=\(runId, privacy: .public) session=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
do {
let response = try await GatewayConnection.shared.chatSend(
sessionKey: sessionKey,
message: prompt,
thinking: "low",
idempotencyKey: runId,
attachments: [])
guard self.isCurrent(gen) else { return }
self.logger.info(
"talk chat.send ok runId=\(response.runId, privacy: .public) session=\(sessionKey, privacy: .public)")
guard let assistantText = await self.waitForAssistantText(
sessionKey: sessionKey,
since: startedAt,
timeoutSeconds: 45)
else {
self.logger.warning("talk assistant text missing after timeout")
await self.startListening()
await self.startRecognition()
return
}
guard self.isCurrent(gen) else { return }
self.logger.info("talk assistant text len=\(assistantText.count, privacy: .public)")
await self.playAssistant(text: assistantText)
guard self.isCurrent(gen) else { return }
await self.resumeListeningIfNeeded()
return
} catch {
self.logger.error("talk chat.send failed: \(error.localizedDescription, privacy: .public)")
await self.resumeListeningIfNeeded()
return
}
}
private func resumeListeningIfNeeded() async {
if self.isPaused {
self.lastTranscript = ""
self.lastHeard = nil
self.lastSpeechEnergyAt = nil
await MainActor.run {
TalkModeController.shared.updateLevel(0)
}
return
}
await self.startListening()
await self.startRecognition()
}
private func buildPrompt(transcript: String) -> String {
let interrupted = self.lastInterruptedAtSeconds
self.lastInterruptedAtSeconds = nil
return TalkPromptBuilder.build(transcript: transcript, interruptedAtSeconds: interrupted)
}
private func waitForAssistantText(
sessionKey: String,
since: Double,
timeoutSeconds: Int) async -> String?
{
let deadline = Date().addingTimeInterval(TimeInterval(timeoutSeconds))
while Date() < deadline {
if let text = await self.latestAssistantText(sessionKey: sessionKey, since: since) {
return text
}
try? await Task.sleep(nanoseconds: 300_000_000)
}
return nil
}
private func latestAssistantText(sessionKey: String, since: Double? = nil) async -> String? {
do {
let history = try await GatewayConnection.shared.chatHistory(sessionKey: sessionKey)
let messages = history.messages ?? []
let decoded: [ClawdisChatMessage] = messages.compactMap { item in
guard let data = try? JSONEncoder().encode(item) else { return nil }
return try? JSONDecoder().decode(ClawdisChatMessage.self, from: data)
}
let assistant = decoded.last { message in
guard message.role == "assistant" else { return false }
guard let since else { return true }
guard let timestamp = message.timestamp else { return false }
return TalkHistoryTimestamp.isAfter(timestamp, sinceSeconds: since)
}
guard let assistant else { return nil }
let text = assistant.content.compactMap(\.text).joined(separator: "\n")
let trimmed = text.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines)
return trimmed.isEmpty ? nil : trimmed
} catch {
self.logger.error("talk history fetch failed: \(error.localizedDescription, privacy: .public)")
return nil
}
}
private func playAssistant(text: String) async {
let gen = self.lifecycleGeneration
let parse = TalkDirectiveParser.parse(text)
let directive = parse.directive
let cleaned = parse.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
guard !cleaned.isEmpty else { return }
guard self.isCurrent(gen) else { return }
if !parse.unknownKeys.isEmpty {
self.logger
.warning("talk directive ignored keys: \(parse.unknownKeys.joined(separator: ","), privacy: .public)")
}
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
if let requestedVoice, !requestedVoice.isEmpty, resolvedVoice == nil {
self.logger.warning("talk unknown voice alias \(requestedVoice, privacy: .public)")
}
if let voice = resolvedVoice {
if directive?.once == true {
self.logger.info("talk voice override (once) voiceId=\(voice, privacy: .public)")
} else {
self.currentVoiceId = voice
self.voiceOverrideActive = true
self.logger.info("talk voice override voiceId=\(voice, privacy: .public)")
}
}
if let model = directive?.modelId {
if directive?.once == true {
self.logger.info("talk model override (once) modelId=\(model, privacy: .public)")
} else {
self.currentModelId = model
self.modelOverrideActive = true
}
}
let apiKey = self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines)
let preferredVoice =
resolvedVoice ??
self.currentVoiceId ??
self.defaultVoiceId
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
let voiceId: String? = if let apiKey, !apiKey.isEmpty {
await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey)
} else {
nil
}
if apiKey?.isEmpty != false {
self.ttsLogger.warning("talk missing ELEVENLABS_API_KEY; falling back to system voice")
} else if voiceId == nil {
self.ttsLogger.warning("talk missing voiceId; falling back to system voice")
} else if let voiceId {
self.ttsLogger
.info("talk TTS request voiceId=\(voiceId, privacy: .public) chars=\(cleaned.count, privacy: .public)")
}
self.lastSpokenText = cleaned
let synthTimeoutSeconds = max(20.0, min(90.0, Double(cleaned.count) * 0.12))
do {
if let apiKey, !apiKey.isEmpty, let voiceId {
let desiredOutputFormat = directive?.outputFormat ?? self.defaultOutputFormat ?? "pcm_44100"
let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(desiredOutputFormat)
if outputFormat == nil, !desiredOutputFormat.isEmpty {
self.logger
.warning(
"talk output_format unsupported for local playback: \(desiredOutputFormat, privacy: .public)")
}
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
ElevenLabsTTSRequest(
text: cleaned,
modelId: modelId,
outputFormat: outputFormat,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
style: TalkTTSValidation.validatedUnit(directive?.style),
speakerBoost: directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
}
let request = makeRequest(outputFormat: outputFormat)
self.ttsLogger.info("talk TTS synth timeout=\(synthTimeoutSeconds, privacy: .public)s")
let client = ElevenLabsTTSClient(apiKey: apiKey)
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
guard self.isCurrent(gen) else { return }
if self.interruptOnSpeech {
await self.startRecognition()
guard self.isCurrent(gen) else { return }
}
await MainActor.run { TalkModeController.shared.updatePhase(.speaking) }
self.phase = .speaking
let sampleRate = TalkTTSValidation.pcmSampleRate(from: outputFormat)
var result: StreamingPlaybackResult
if let sampleRate {
self.lastPlaybackWasPCM = true
result = await self.playPCM(stream: stream, sampleRate: sampleRate)
if !result.finished, result.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
self.ttsLogger.warning("talk pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: makeRequest(outputFormat: mp3Format))
result = await self.playMP3(stream: mp3Stream)
}
} else {
self.lastPlaybackWasPCM = false
result = await self.playMP3(stream: stream)
}
self.ttsLogger
.info(
"talk audio result finished=\(result.finished, privacy: .public) interruptedAt=\(String(describing: result.interruptedAt), privacy: .public)")
if !result.finished, result.interruptedAt == nil {
throw NSError(domain: "StreamingAudioPlayer", code: 1, userInfo: [
NSLocalizedDescriptionKey: "audio playback failed",
])
}
if !result.finished, let interruptedAt = result.interruptedAt, self.phase == .speaking {
if self.interruptOnSpeech {
self.lastInterruptedAtSeconds = interruptedAt
}
}
} else {
self.ttsLogger.info("talk system voice start chars=\(cleaned.count, privacy: .public)")
if self.interruptOnSpeech {
await self.startRecognition()
guard self.isCurrent(gen) else { return }
}
await MainActor.run { TalkModeController.shared.updatePhase(.speaking) }
self.phase = .speaking
await TalkSystemSpeechSynthesizer.shared.stop()
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
self.ttsLogger.info("talk system voice done")
}
} catch {
self.ttsLogger
.error("talk TTS failed: \(error.localizedDescription, privacy: .public); falling back to system voice")
do {
if self.interruptOnSpeech {
await self.startRecognition()
guard self.isCurrent(gen) else { return }
}
await MainActor.run { TalkModeController.shared.updatePhase(.speaking) }
self.phase = .speaking
await TalkSystemSpeechSynthesizer.shared.stop()
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
} catch {
self.ttsLogger.error("talk system voice failed: \(error.localizedDescription, privacy: .public)")
}
}
if self.phase == .speaking {
self.phase = .thinking
await MainActor.run { TalkModeController.shared.updatePhase(.thinking) }
}
}
private func resolveVoiceId(preferred: String?, apiKey: String) async -> String? {
let trimmed = preferred?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if !trimmed.isEmpty {
if let resolved = self.resolveVoiceAlias(trimmed) { return resolved }
self.ttsLogger.warning("talk unknown voice alias \(trimmed, privacy: .public)")
}
if let fallbackVoiceId { return fallbackVoiceId }
do {
let voices = try await ElevenLabsTTSClient(apiKey: apiKey).listVoices()
guard let first = voices.first else {
self.ttsLogger.error("elevenlabs voices list empty")
return nil
}
self.fallbackVoiceId = first.voiceId
if self.defaultVoiceId == nil {
self.defaultVoiceId = first.voiceId
}
if !self.voiceOverrideActive {
self.currentVoiceId = first.voiceId
}
let name = first.name ?? "unknown"
self.ttsLogger
.info("talk default voice selected \(name, privacy: .public) (\(first.voiceId, privacy: .public))")
return first.voiceId
} catch {
self.ttsLogger.error("elevenlabs list voices failed: \(error.localizedDescription, privacy: .public)")
return nil
}
}
private func resolveVoiceAlias(_ value: String?) -> String? {
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let normalized = trimmed.lowercased()
if let mapped = self.voiceAliases[normalized] { return mapped }
if self.voiceAliases.values.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) {
return trimmed
}
return Self.isLikelyVoiceId(trimmed) ? trimmed : nil
}
private static func isLikelyVoiceId(_ value: String) -> Bool {
guard value.count >= 10 else { return false }
return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" }
}
func stopSpeaking(reason: TalkStopReason) async {
let usePCM = self.lastPlaybackWasPCM
let interruptedAt = usePCM ? await self.stopPCM() : await self.stopMP3()
_ = usePCM ? await self.stopMP3() : await self.stopPCM()
await TalkSystemSpeechSynthesizer.shared.stop()
guard self.phase == .speaking else { return }
if reason == .speech, let interruptedAt {
self.lastInterruptedAtSeconds = interruptedAt
}
if reason == .manual {
return
}
if reason == .speech || reason == .userTap {
await self.startListening()
return
}
self.phase = .thinking
await MainActor.run { TalkModeController.shared.updatePhase(.thinking) }
}
// MARK: - Audio playback (MainActor helpers)
@MainActor
private func playPCM(
stream: AsyncThrowingStream<Data, Error>,
sampleRate: Double) async -> StreamingPlaybackResult
{
await PCMStreamingAudioPlayer.shared.play(stream: stream, sampleRate: sampleRate)
}
@MainActor
private func playMP3(stream: AsyncThrowingStream<Data, Error>) async -> StreamingPlaybackResult {
await StreamingAudioPlayer.shared.play(stream: stream)
}
@MainActor
private func stopPCM() -> Double? {
PCMStreamingAudioPlayer.shared.stop()
}
@MainActor
private func stopMP3() -> Double? {
StreamingAudioPlayer.shared.stop()
}
// MARK: - Config
private func reloadConfig() async {
let cfg = await self.fetchTalkConfig()
self.defaultVoiceId = cfg.voiceId
self.voiceAliases = cfg.voiceAliases
if !self.voiceOverrideActive {
self.currentVoiceId = cfg.voiceId
}
self.defaultModelId = cfg.modelId
if !self.modelOverrideActive {
self.currentModelId = cfg.modelId
}
self.defaultOutputFormat = cfg.outputFormat
self.interruptOnSpeech = cfg.interruptOnSpeech
self.apiKey = cfg.apiKey
let hasApiKey = (cfg.apiKey?.isEmpty == false)
let voiceLabel = (cfg.voiceId?.isEmpty == false) ? cfg.voiceId! : "none"
let modelLabel = (cfg.modelId?.isEmpty == false) ? cfg.modelId! : "none"
self.logger
.info(
"talk config voiceId=\(voiceLabel, privacy: .public) modelId=\(modelLabel, privacy: .public) apiKey=\(hasApiKey, privacy: .public) interrupt=\(cfg.interruptOnSpeech, privacy: .public)")
}
private struct TalkRuntimeConfig {
let voiceId: String?
let voiceAliases: [String: String]
let modelId: String?
let outputFormat: String?
let interruptOnSpeech: Bool
let apiKey: String?
}
private func fetchTalkConfig() async -> TalkRuntimeConfig {
let env = ProcessInfo.processInfo.environment
let envVoice = env["ELEVENLABS_VOICE_ID"]?.trimmingCharacters(in: .whitespacesAndNewlines)
let sagVoice = env["SAG_VOICE_ID"]?.trimmingCharacters(in: .whitespacesAndNewlines)
let envApiKey = env["ELEVENLABS_API_KEY"]?.trimmingCharacters(in: .whitespacesAndNewlines)
do {
let snap: ConfigSnapshot = try await GatewayConnection.shared.requestDecoded(
method: .configGet,
params: nil,
timeoutMs: 8000)
let talk = snap.config?["talk"]?.dictionaryValue
let ui = snap.config?["ui"]?.dictionaryValue
let rawSeam = ui?["seamColor"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
await MainActor.run {
AppStateStore.shared.seamColorHex = rawSeam.isEmpty ? nil : rawSeam
}
let voice = talk?["voiceId"]?.stringValue
let rawAliases = talk?["voiceAliases"]?.dictionaryValue
let resolvedAliases: [String: String] =
rawAliases?.reduce(into: [:]) { acc, entry in
let key = entry.key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
let value = entry.value.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !key.isEmpty, !value.isEmpty else { return }
acc[key] = value
} ?? [:]
let model = talk?["modelId"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedModel = (model?.isEmpty == false) ? model! : Self.defaultModelIdFallback
let outputFormat = talk?["outputFormat"]?.stringValue
let interrupt = talk?["interruptOnSpeech"]?.boolValue
let apiKey = talk?["apiKey"]?.stringValue
let resolvedVoice =
(voice?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? voice : nil) ??
(envVoice?.isEmpty == false ? envVoice : nil) ??
(sagVoice?.isEmpty == false ? sagVoice : nil)
let resolvedApiKey =
(envApiKey?.isEmpty == false ? envApiKey : nil) ??
(apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? apiKey : nil)
return TalkRuntimeConfig(
voiceId: resolvedVoice,
voiceAliases: resolvedAliases,
modelId: resolvedModel,
outputFormat: outputFormat,
interruptOnSpeech: interrupt ?? true,
apiKey: resolvedApiKey)
} catch {
let resolvedVoice =
(envVoice?.isEmpty == false ? envVoice : nil) ??
(sagVoice?.isEmpty == false ? sagVoice : nil)
let resolvedApiKey = envApiKey?.isEmpty == false ? envApiKey : nil
return TalkRuntimeConfig(
voiceId: resolvedVoice,
voiceAliases: [:],
modelId: Self.defaultModelIdFallback,
outputFormat: nil,
interruptOnSpeech: true,
apiKey: resolvedApiKey)
}
}
// MARK: - Audio level handling
private func noteAudioLevel(rms: Double) async {
if self.phase != .listening, self.phase != .speaking { return }
let alpha: Double = rms < self.noiseFloorRMS ? 0.08 : 0.01
self.noiseFloorRMS = max(1e-7, self.noiseFloorRMS + (rms - self.noiseFloorRMS) * alpha)
let threshold = max(self.minSpeechRMS, self.noiseFloorRMS * self.speechBoostFactor)
if rms >= threshold {
let now = Date()
self.lastHeard = now
self.lastSpeechEnergyAt = now
}
if self.phase == .listening {
let clamped = min(1.0, max(0.0, rms / max(self.minSpeechRMS, threshold)))
await MainActor.run { TalkModeController.shared.updateLevel(clamped) }
}
}
private static func rmsLevel(buffer: AVAudioPCMBuffer) -> Double? {
guard let channelData = buffer.floatChannelData?.pointee else { return nil }
let frameCount = Int(buffer.frameLength)
guard frameCount > 0 else { return nil }
var sum: Double = 0
for i in 0..<frameCount {
let sample = Double(channelData[i])
sum += sample * sample
}
return sqrt(sum / Double(frameCount))
}
private func shouldInterrupt(transcript: String, hasConfidence: Bool) async -> Bool {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
guard trimmed.count >= 3 else { return false }
if self.isLikelyEcho(of: trimmed) { return false }
let now = Date()
if let lastSpeechEnergyAt, now.timeIntervalSince(lastSpeechEnergyAt) > 0.35 {
return false
}
return hasConfidence
}
private func isLikelyEcho(of transcript: String) -> Bool {
guard let spoken = self.lastSpokenText?.lowercased(), !spoken.isEmpty else { return false }
let probe = transcript.lowercased()
if probe.count < 6 {
return spoken.contains(probe)
}
return spoken.contains(probe)
}
private static func resolveSpeed(speed: Double?, rateWPM: Int?, logger: Logger) -> Double? {
if let rateWPM, rateWPM > 0 {
let resolved = Double(rateWPM) / 175.0
if resolved <= 0.5 || resolved >= 2.0 {
logger.warning("talk rateWPM out of range: \(rateWPM, privacy: .public)")
return nil
}
return resolved
}
if let speed {
if speed <= 0.5 || speed >= 2.0 {
logger.warning("talk speed out of range: \(speed, privacy: .public)")
return nil
}
return speed
}
return nil
}
private static func validatedUnit(_ value: Double?, name: String, logger: Logger) -> Double? {
guard let value else { return nil }
if value < 0 || value > 1 {
logger.warning("talk \(name, privacy: .public) out of range: \(value, privacy: .public)")
return nil
}
return value
}
private static func validatedSeed(_ value: Int?, logger: Logger) -> UInt32? {
guard let value else { return nil }
if value < 0 || value > 4_294_967_295 {
logger.warning("talk seed out of range: \(value, privacy: .public)")
return nil
}
return UInt32(value)
}
private static func validatedNormalize(_ value: String?, logger: Logger) -> String? {
guard let value else { return nil }
let normalized = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
guard ["auto", "on", "off"].contains(normalized) else {
logger.warning("talk normalize invalid: \(normalized, privacy: .public)")
return nil
}
return normalized
}
}

View File

@@ -0,0 +1,8 @@
import Foundation
enum TalkModePhase: String {
case idle
case listening
case thinking
case speaking
}

View File

@@ -0,0 +1,146 @@
import AppKit
import Observation
import OSLog
import SwiftUI
@MainActor
@Observable
final class TalkOverlayController {
static let shared = TalkOverlayController()
static let overlaySize: CGFloat = 440
static let orbSize: CGFloat = 96
static let orbPadding: CGFloat = 12
static let orbHitSlop: CGFloat = 10
private let logger = Logger(subsystem: "com.steipete.clawdis", category: "talk.overlay")
struct Model {
var isVisible: Bool = false
var phase: TalkModePhase = .idle
var isPaused: Bool = false
var level: Double = 0
}
var model = Model()
private var window: NSPanel?
private var hostingView: NSHostingView<TalkOverlayView>?
private let screenInset: CGFloat = 0
func present() {
self.ensureWindow()
self.hostingView?.rootView = TalkOverlayView(controller: self)
let target = self.targetFrame()
guard let window else { return }
if !self.model.isVisible {
self.model.isVisible = true
let start = target.offsetBy(dx: 0, dy: -6)
window.setFrame(start, display: true)
window.alphaValue = 0
window.orderFrontRegardless()
NSAnimationContext.runAnimationGroup { context in
context.duration = 0.18
context.timingFunction = CAMediaTimingFunction(name: .easeOut)
window.animator().setFrame(target, display: true)
window.animator().alphaValue = 1
}
} else {
window.setFrame(target, display: true)
window.orderFrontRegardless()
}
}
func dismiss() {
guard let window else {
self.model.isVisible = false
return
}
let target = window.frame.offsetBy(dx: 6, dy: 6)
NSAnimationContext.runAnimationGroup { context in
context.duration = 0.16
context.timingFunction = CAMediaTimingFunction(name: .easeOut)
window.animator().setFrame(target, display: true)
window.animator().alphaValue = 0
} completionHandler: {
Task { @MainActor in
window.orderOut(nil)
self.model.isVisible = false
}
}
}
func updatePhase(_ phase: TalkModePhase) {
guard self.model.phase != phase else { return }
self.logger.info("talk overlay phase=\(phase.rawValue, privacy: .public)")
self.model.phase = phase
}
func updatePaused(_ paused: Bool) {
guard self.model.isPaused != paused else { return }
self.logger.info("talk overlay paused=\(paused)")
self.model.isPaused = paused
}
func updateLevel(_ level: Double) {
guard self.model.isVisible else { return }
self.model.level = max(0, min(1, level))
}
func currentWindowOrigin() -> CGPoint? {
self.window?.frame.origin
}
func setWindowOrigin(_ origin: CGPoint) {
guard let window else { return }
window.setFrameOrigin(origin)
}
// MARK: - Private
private func ensureWindow() {
if self.window != nil { return }
let panel = NSPanel(
contentRect: NSRect(x: 0, y: 0, width: Self.overlaySize, height: Self.overlaySize),
styleMask: [.nonactivatingPanel, .borderless],
backing: .buffered,
defer: false)
panel.isOpaque = false
panel.backgroundColor = .clear
panel.hasShadow = false
panel.level = NSWindow.Level(rawValue: NSWindow.Level.popUpMenu.rawValue - 4)
panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient]
panel.hidesOnDeactivate = false
panel.isMovable = false
panel.acceptsMouseMovedEvents = true
panel.isFloatingPanel = true
panel.becomesKeyOnlyIfNeeded = true
panel.titleVisibility = .hidden
panel.titlebarAppearsTransparent = true
let host = TalkOverlayHostingView(rootView: TalkOverlayView(controller: self))
host.translatesAutoresizingMaskIntoConstraints = false
panel.contentView = host
self.hostingView = host
self.window = panel
}
private func targetFrame() -> NSRect {
let screen = self.window?.screen
?? NSScreen.main
?? NSScreen.screens.first
guard let screen else { return .zero }
let size = NSSize(width: Self.overlaySize, height: Self.overlaySize)
let visible = screen.visibleFrame
let origin = CGPoint(
x: visible.maxX - size.width - self.screenInset,
y: visible.maxY - size.height - self.screenInset)
return NSRect(origin: origin, size: size)
}
}
private final class TalkOverlayHostingView: NSHostingView<TalkOverlayView> {
override func acceptsFirstMouse(for event: NSEvent?) -> Bool {
true
}
}

View File

@@ -0,0 +1,219 @@
import AppKit
import SwiftUI
struct TalkOverlayView: View {
var controller: TalkOverlayController
@State private var appState = AppStateStore.shared
@State private var hoveringWindow = false
var body: some View {
ZStack(alignment: .topTrailing) {
let isPaused = self.controller.model.isPaused
Color.clear
TalkOrbView(
phase: self.controller.model.phase,
level: self.controller.model.level,
accent: self.seamColor,
isPaused: isPaused)
.frame(width: TalkOverlayController.orbSize, height: TalkOverlayController.orbSize)
.padding(.top, TalkOverlayController.orbPadding)
.padding(.trailing, TalkOverlayController.orbPadding)
.contentShape(Circle())
.opacity(isPaused ? 0.55 : 1)
.background(
TalkOrbInteractionView(
onSingleClick: { TalkModeController.shared.togglePaused() },
onDoubleClick: { TalkModeController.shared.stopSpeaking(reason: .userTap) },
onDragStart: { TalkModeController.shared.setPaused(true) }))
.overlay(alignment: .topLeading) {
Button {
TalkModeController.shared.exitTalkMode()
} label: {
Image(systemName: "xmark")
.font(.system(size: 10, weight: .bold))
.foregroundStyle(Color.white.opacity(0.95))
.frame(width: 18, height: 18)
.background(Color.black.opacity(0.4))
.clipShape(Circle())
}
.buttonStyle(.plain)
.contentShape(Circle())
.offset(x: -2, y: -2)
.opacity(self.hoveringWindow ? 1 : 0)
.animation(.easeOut(duration: 0.12), value: self.hoveringWindow)
}
.onHover { self.hoveringWindow = $0 }
}
.frame(
width: TalkOverlayController.overlaySize,
height: TalkOverlayController.overlaySize,
alignment: .topTrailing)
}
private static let defaultSeamColor = Color(red: 79 / 255.0, green: 122 / 255.0, blue: 154 / 255.0)
private var seamColor: Color {
Self.color(fromHex: self.appState.seamColorHex) ?? Self.defaultSeamColor
}
private static func color(fromHex raw: String?) -> Color? {
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed
guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil }
let r = Double((value >> 16) & 0xFF) / 255.0
let g = Double((value >> 8) & 0xFF) / 255.0
let b = Double(value & 0xFF) / 255.0
return Color(red: r, green: g, blue: b)
}
}
private struct TalkOrbInteractionView: NSViewRepresentable {
let onSingleClick: () -> Void
let onDoubleClick: () -> Void
let onDragStart: () -> Void
func makeNSView(context: Context) -> NSView {
let view = OrbInteractionNSView()
view.onSingleClick = self.onSingleClick
view.onDoubleClick = self.onDoubleClick
view.onDragStart = self.onDragStart
view.wantsLayer = true
view.layer?.backgroundColor = NSColor.clear.cgColor
return view
}
func updateNSView(_ nsView: NSView, context: Context) {
guard let view = nsView as? OrbInteractionNSView else { return }
view.onSingleClick = self.onSingleClick
view.onDoubleClick = self.onDoubleClick
view.onDragStart = self.onDragStart
}
}
private final class OrbInteractionNSView: NSView {
var onSingleClick: (() -> Void)?
var onDoubleClick: (() -> Void)?
var onDragStart: (() -> Void)?
private var mouseDownEvent: NSEvent?
private var didDrag = false
private var suppressSingleClick = false
override var acceptsFirstResponder: Bool { true }
override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
override func mouseDown(with event: NSEvent) {
self.mouseDownEvent = event
self.didDrag = false
self.suppressSingleClick = event.clickCount > 1
if event.clickCount == 2 {
self.onDoubleClick?()
}
}
override func mouseDragged(with event: NSEvent) {
guard let startEvent = self.mouseDownEvent else { return }
if !self.didDrag {
let dx = event.locationInWindow.x - startEvent.locationInWindow.x
let dy = event.locationInWindow.y - startEvent.locationInWindow.y
if abs(dx) + abs(dy) < 2 { return }
self.didDrag = true
self.onDragStart?()
self.window?.performDrag(with: startEvent)
}
}
override func mouseUp(with event: NSEvent) {
if !self.didDrag && !self.suppressSingleClick {
self.onSingleClick?()
}
self.mouseDownEvent = nil
self.didDrag = false
self.suppressSingleClick = false
}
}
private struct TalkOrbView: View {
let phase: TalkModePhase
let level: Double
let accent: Color
let isPaused: Bool
var body: some View {
if self.isPaused {
Circle()
.fill(self.orbGradient)
.overlay(Circle().stroke(Color.white.opacity(0.35), lineWidth: 1))
.shadow(color: Color.black.opacity(0.18), radius: 10, x: 0, y: 5)
} else {
TimelineView(.animation) { context in
let t = context.date.timeIntervalSinceReferenceDate
let listenScale = phase == .listening ? (1 + CGFloat(self.level) * 0.12) : 1
let pulse = phase == .speaking ? (1 + 0.06 * sin(t * 6)) : 1
ZStack {
Circle()
.fill(self.orbGradient)
.overlay(Circle().stroke(Color.white.opacity(0.45), lineWidth: 1))
.shadow(color: Color.black.opacity(0.22), radius: 10, x: 0, y: 5)
.scaleEffect(pulse * listenScale)
TalkWaveRings(phase: phase, level: level, time: t, accent: self.accent)
if phase == .thinking {
TalkOrbitArcs(time: t)
}
}
}
}
}
private var orbGradient: RadialGradient {
RadialGradient(
colors: [Color.white, self.accent],
center: .topLeading,
startRadius: 4,
endRadius: 52)
}
}
private struct TalkWaveRings: View {
let phase: TalkModePhase
let level: Double
let time: TimeInterval
let accent: Color
var body: some View {
ZStack {
ForEach(0..<3, id: \.self) { idx in
let speed = phase == .speaking ? 1.4 : phase == .listening ? 0.9 : 0.6
let progress = (time * speed + Double(idx) * 0.28).truncatingRemainder(dividingBy: 1)
let amplitude = phase == .speaking ? 0.95 : phase == .listening ? 0.5 + level * 0.7 : 0.35
let scale = 0.75 + progress * amplitude + (phase == .listening ? level * 0.15 : 0)
let alpha = phase == .speaking ? 0.72 : phase == .listening ? 0.58 + level * 0.28 : 0.4
Circle()
.stroke(self.accent.opacity(alpha - progress * 0.3), lineWidth: 1.6)
.scaleEffect(scale)
.opacity(alpha - progress * 0.6)
}
}
}
}
private struct TalkOrbitArcs: View {
let time: TimeInterval
var body: some View {
ZStack {
Circle()
.trim(from: 0.08, to: 0.26)
.stroke(Color.white.opacity(0.88), style: StrokeStyle(lineWidth: 1.6, lineCap: .round))
.rotationEffect(.degrees(time * 42))
Circle()
.trim(from: 0.62, to: 0.86)
.stroke(Color.white.opacity(0.7), style: StrokeStyle(lineWidth: 1.4, lineCap: .round))
.rotationEffect(.degrees(-time * 35))
}
.scaleEffect(1.08)
}
}

View File

@@ -1,7 +1,6 @@
import AppKit
import Foundation
import Observation
import OSLog
@MainActor
@Observable

View File

@@ -1,6 +1,5 @@
import AppKit
import Observation
import OSLog
import SwiftUI
/// Lightweight, borderless panel that shows the current voice wake transcript near the menu bar.

View File

@@ -1,6 +1,5 @@
import AVFoundation
import Foundation
import OSLog
import Speech
import SwabbleKit

View File

@@ -29,6 +29,10 @@ final class WebChatManager {
var onPanelVisibilityChanged: ((Bool) -> Void)?
var activeSessionKey: String? {
self.panelSessionKey ?? self.windowSessionKey
}
func show(sessionKey: String) {
self.closePanel()
if let controller = self.windowController {

View File

@@ -155,7 +155,8 @@ final class WebChatSwiftUIWindowController {
self.sessionKey = sessionKey
self.presentation = presentation
let vm = ClawdisChatViewModel(sessionKey: sessionKey, transport: transport)
self.hosting = NSHostingController(rootView: ClawdisChatView(viewModel: vm))
let accent = Self.color(fromHex: AppStateStore.shared.seamColorHex)
self.hosting = NSHostingController(rootView: ClawdisChatView(viewModel: vm, userAccent: accent))
self.contentController = Self.makeContentController(for: presentation, hosting: self.hosting)
self.window = Self.makeWindow(for: presentation, contentViewController: self.contentController)
}
@@ -355,4 +356,15 @@ final class WebChatSwiftUIWindowController {
window.setFrame(frame, display: false)
}
}
private static func color(fromHex raw: String?) -> Color? {
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed
guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil }
let r = Double((value >> 16) & 0xFF) / 255.0
let g = Double((value >> 8) & 0xFF) / 255.0
let b = Double(value & 0xFF) / 255.0
return Color(red: r, green: g, blue: b)
}
}

View File

@@ -689,6 +689,23 @@ public struct ConfigSetParams: Codable {
}
}
public struct TalkModeParams: Codable {
public let enabled: Bool
public let phase: String?
public init(
enabled: Bool,
phase: String?
) {
self.enabled = enabled
self.phase = phase
}
private enum CodingKeys: String, CodingKey {
case enabled
case phase
}
}
public struct ProvidersStatusParams: Codable {
public let probe: Bool?
public let timeoutms: Int?

View File

@@ -52,12 +52,17 @@ import Testing
try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: nodePath.path)
try self.makeExec(at: scriptPath)
let cmd = CommandResolver.clawdisCommand(subcommand: "rpc", defaults: defaults)
let cmd = CommandResolver.clawdisCommand(
subcommand: "rpc",
defaults: defaults,
searchPaths: [tmp.appendingPathComponent("node_modules/.bin").path])
#expect(cmd.count >= 3)
#expect(cmd[0] == nodePath.path)
#expect(cmd[1] == scriptPath.path)
#expect(cmd[2] == "rpc")
if cmd.count >= 3 {
#expect(cmd[0] == nodePath.path)
#expect(cmd[1] == scriptPath.path)
#expect(cmd[2] == "rpc")
}
}
@Test func fallsBackToPnpm() async throws {

View File

@@ -43,7 +43,8 @@ struct ConnectionsSettingsSmokeTests {
elapsedMs: 120,
bot: ProvidersStatusSnapshot.TelegramBot(id: 123, username: "clawdisbot"),
webhook: ProvidersStatusSnapshot.TelegramWebhook(url: "https://example.com/hook", hasCustomCert: false)),
lastProbeAt: 1_700_000_050_000))
lastProbeAt: 1_700_000_050_000),
discord: nil)
store.whatsappLoginMessage = "Scan QR"
store.whatsappLoginQrDataUrl =
@@ -92,7 +93,8 @@ struct ConnectionsSettingsSmokeTests {
elapsedMs: 120,
bot: nil,
webhook: nil),
lastProbeAt: 1_700_000_100_000))
lastProbeAt: 1_700_000_100_000),
discord: nil)
let view = ConnectionsSettings(store: store)
_ = view.body

View File

@@ -0,0 +1,97 @@
import Foundation
import Testing
@testable import Clawdis
@Suite(.serialized) struct TalkAudioPlayerTests {
@MainActor
@Test func playDoesNotHangWhenPlaybackEndsOrFails() async throws {
let wav = makeWav16Mono(sampleRate: 8000, samples: 80)
defer { _ = TalkAudioPlayer.shared.stop() }
_ = try await withTimeout(seconds: 2.0) {
await TalkAudioPlayer.shared.play(data: wav)
}
#expect(true)
}
@MainActor
@Test func playDoesNotHangWhenPlayIsCalledTwice() async throws {
let wav = makeWav16Mono(sampleRate: 8000, samples: 800)
defer { _ = TalkAudioPlayer.shared.stop() }
let first = Task { @MainActor in
await TalkAudioPlayer.shared.play(data: wav)
}
await Task.yield()
_ = await TalkAudioPlayer.shared.play(data: wav)
_ = try await withTimeout(seconds: 2.0) {
await first.value
}
#expect(true)
}
}
private struct TimeoutError: Error {}
private func withTimeout<T: Sendable>(
seconds: Double,
_ work: @escaping @Sendable () async throws -> T) async throws -> T
{
try await withThrowingTaskGroup(of: T.self) { group in
group.addTask {
try await work()
}
group.addTask {
try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000))
throw TimeoutError()
}
let result = try await group.next()
group.cancelAll()
guard let result else { throw TimeoutError() }
return result
}
}
private func makeWav16Mono(sampleRate: UInt32, samples: Int) -> Data {
let channels: UInt16 = 1
let bitsPerSample: UInt16 = 16
let blockAlign = channels * (bitsPerSample / 8)
let byteRate = sampleRate * UInt32(blockAlign)
let dataSize = UInt32(samples) * UInt32(blockAlign)
var data = Data()
data.append(contentsOf: [0x52, 0x49, 0x46, 0x46]) // RIFF
data.appendLEUInt32(36 + dataSize)
data.append(contentsOf: [0x57, 0x41, 0x56, 0x45]) // WAVE
data.append(contentsOf: [0x66, 0x6D, 0x74, 0x20]) // fmt
data.appendLEUInt32(16) // PCM
data.appendLEUInt16(1) // audioFormat
data.appendLEUInt16(channels)
data.appendLEUInt32(sampleRate)
data.appendLEUInt32(byteRate)
data.appendLEUInt16(blockAlign)
data.appendLEUInt16(bitsPerSample)
data.append(contentsOf: [0x64, 0x61, 0x74, 0x61]) // data
data.appendLEUInt32(dataSize)
// Silence samples.
data.append(Data(repeating: 0, count: Int(dataSize)))
return data
}
private extension Data {
mutating func appendLEUInt16(_ value: UInt16) {
var v = value.littleEndian
Swift.withUnsafeBytes(of: &v) { append(contentsOf: $0) }
}
mutating func appendLEUInt32(_ value: UInt32) {
var v = value.littleEndian
Swift.withUnsafeBytes(of: &v) { append(contentsOf: $0) }
}
}