531 lines
21 KiB
Swift
531 lines
21 KiB
Swift
import AVFoundation
|
|
import Foundation
|
|
import OSLog
|
|
import Speech
|
|
import SwabbleKit
|
|
#if canImport(AppKit)
|
|
import AppKit
|
|
#endif
|
|
|
|
/// Background listener that keeps the voice-wake pipeline alive outside the settings test view.
|
|
actor VoiceWakeRuntime {
|
|
static let shared = VoiceWakeRuntime()
|
|
|
|
enum ListeningState { case idle, voiceWake, pushToTalk }
|
|
|
|
private let logger = Logger(subsystem: "com.clawdbot", category: "voicewake.runtime")
|
|
|
|
private var recognizer: SFSpeechRecognizer?
|
|
// Lazily created on start to avoid creating an AVAudioEngine at app launch, which can switch Bluetooth
|
|
// headphones into the low-quality headset profile even if Voice Wake is disabled.
|
|
private var audioEngine: AVAudioEngine?
|
|
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
|
|
private var recognitionTask: SFSpeechRecognitionTask?
|
|
private var recognitionGeneration: Int = 0 // drop stale callbacks after restarts
|
|
private var lastHeard: Date?
|
|
private var noiseFloorRMS: Double = 1e-4
|
|
private var captureStartedAt: Date?
|
|
private var captureTask: Task<Void, Never>?
|
|
private var capturedTranscript: String = ""
|
|
private var isCapturing: Bool = false
|
|
private var heardBeyondTrigger: Bool = false
|
|
private var triggerChimePlayed: Bool = false
|
|
private var committedTranscript: String = ""
|
|
private var volatileTranscript: String = ""
|
|
private var cooldownUntil: Date?
|
|
private var currentConfig: RuntimeConfig?
|
|
private var listeningState: ListeningState = .idle
|
|
private var overlayToken: UUID?
|
|
private var activeTriggerEndTime: TimeInterval?
|
|
private var scheduledRestartTask: Task<Void, Never>?
|
|
|
|
// Tunables
|
|
// Silence threshold once we've captured user speech (post-trigger).
|
|
private let silenceWindow: TimeInterval = 2.0
|
|
// Silence threshold when we only heard the trigger but no post-trigger speech yet.
|
|
private let triggerOnlySilenceWindow: TimeInterval = 5.0
|
|
// Maximum capture duration from trigger until we force-send, to avoid runaway sessions.
|
|
private let captureHardStop: TimeInterval = 120.0
|
|
private let debounceAfterSend: TimeInterval = 0.35
|
|
// Voice activity detection parameters (RMS-based).
|
|
private let minSpeechRMS: Double = 1e-3
|
|
private let speechBoostFactor: Double = 6.0 // how far above noise floor we require to mark speech
|
|
|
|
/// Stops the active Speech pipeline without clearing the stored config, so we can restart cleanly.
|
|
private func haltRecognitionPipeline() {
|
|
// Bump generation first so any in-flight callbacks from the cancelled task get dropped.
|
|
self.recognitionGeneration &+= 1
|
|
self.recognitionTask?.cancel()
|
|
self.recognitionTask = nil
|
|
self.recognitionRequest?.endAudio()
|
|
self.recognitionRequest = nil
|
|
self.audioEngine?.inputNode.removeTap(onBus: 0)
|
|
self.audioEngine?.stop()
|
|
// Release the engine so we also release any audio session/resources when Voice Wake is idle.
|
|
self.audioEngine = nil
|
|
}
|
|
|
|
struct RuntimeConfig: Equatable {
|
|
let triggers: [String]
|
|
let micID: String?
|
|
let localeID: String?
|
|
let triggerChime: VoiceWakeChime
|
|
let sendChime: VoiceWakeChime
|
|
}
|
|
|
|
private struct RecognitionUpdate {
|
|
let transcript: String?
|
|
let segments: [WakeWordSegment]
|
|
let isFinal: Bool
|
|
let error: Error?
|
|
let generation: Int
|
|
}
|
|
|
|
func refresh(state: AppState) async {
|
|
let snapshot = await MainActor.run { () -> (Bool, RuntimeConfig) in
|
|
let enabled = state.swabbleEnabled
|
|
let config = RuntimeConfig(
|
|
triggers: sanitizeVoiceWakeTriggers(state.swabbleTriggerWords),
|
|
micID: state.voiceWakeMicID.isEmpty ? nil : state.voiceWakeMicID,
|
|
localeID: state.voiceWakeLocaleID.isEmpty ? nil : state.voiceWakeLocaleID,
|
|
triggerChime: state.voiceWakeTriggerChime,
|
|
sendChime: state.voiceWakeSendChime)
|
|
return (enabled, config)
|
|
}
|
|
|
|
guard voiceWakeSupported, snapshot.0 else {
|
|
self.stop()
|
|
return
|
|
}
|
|
|
|
guard PermissionManager.voiceWakePermissionsGranted() else {
|
|
self.logger.debug("voicewake runtime not starting: permissions missing")
|
|
self.stop()
|
|
return
|
|
}
|
|
|
|
let config = snapshot.1
|
|
|
|
if config == self.currentConfig, self.recognitionTask != nil {
|
|
return
|
|
}
|
|
|
|
self.stop()
|
|
await self.start(with: config)
|
|
}
|
|
|
|
private func start(with config: RuntimeConfig) async {
|
|
do {
|
|
self.recognitionGeneration &+= 1
|
|
let generation = self.recognitionGeneration
|
|
|
|
self.configureSession(localeID: config.localeID)
|
|
|
|
guard let recognizer, recognizer.isAvailable else {
|
|
self.logger.error("voicewake runtime: speech recognizer unavailable")
|
|
return
|
|
}
|
|
|
|
self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
|
|
self.recognitionRequest?.shouldReportPartialResults = true
|
|
guard let request = self.recognitionRequest else { return }
|
|
|
|
// Lazily create the engine here so app launch doesn't grab audio resources / trigger Bluetooth HFP.
|
|
if self.audioEngine == nil {
|
|
self.audioEngine = AVAudioEngine()
|
|
}
|
|
guard let audioEngine = self.audioEngine else { return }
|
|
|
|
let input = audioEngine.inputNode
|
|
let format = input.outputFormat(forBus: 0)
|
|
input.removeTap(onBus: 0)
|
|
input.installTap(onBus: 0, bufferSize: 2048, format: format) { [weak self, weak request] buffer, _ in
|
|
request?.append(buffer)
|
|
if let rms = Self.rmsLevel(buffer: buffer) {
|
|
Task.detached { [weak self] in
|
|
await self?.noteAudioLevel(rms: rms)
|
|
}
|
|
}
|
|
}
|
|
|
|
audioEngine.prepare()
|
|
try audioEngine.start()
|
|
|
|
self.currentConfig = config
|
|
self.lastHeard = Date()
|
|
// Preserve any existing cooldownUntil so the debounce after send isn't wiped by a restart.
|
|
|
|
self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self, generation] result, error in
|
|
guard let self else { return }
|
|
let transcript = result?.bestTranscription.formattedString
|
|
let segments = result.flatMap { result in
|
|
transcript
|
|
.map { WakeWordSpeechSegments.from(transcription: result.bestTranscription, transcript: $0) }
|
|
} ?? []
|
|
let isFinal = result?.isFinal ?? false
|
|
let update = RecognitionUpdate(
|
|
transcript: transcript,
|
|
segments: segments,
|
|
isFinal: isFinal,
|
|
error: error,
|
|
generation: generation)
|
|
Task { await self.handleRecognition(update, config: config) }
|
|
}
|
|
|
|
self.logger.info("voicewake runtime started")
|
|
DiagnosticsFileLog.shared.log(category: "voicewake.runtime", event: "started", fields: [
|
|
"locale": config.localeID ?? "",
|
|
"micID": config.micID ?? "",
|
|
])
|
|
} catch {
|
|
self.logger.error("voicewake runtime failed to start: \(error.localizedDescription, privacy: .public)")
|
|
self.stop()
|
|
}
|
|
}
|
|
|
|
private func stop(dismissOverlay: Bool = true, cancelScheduledRestart: Bool = true) {
|
|
if cancelScheduledRestart {
|
|
self.scheduledRestartTask?.cancel()
|
|
self.scheduledRestartTask = nil
|
|
}
|
|
self.captureTask?.cancel()
|
|
self.captureTask = nil
|
|
self.isCapturing = false
|
|
self.capturedTranscript = ""
|
|
self.captureStartedAt = nil
|
|
self.triggerChimePlayed = false
|
|
self.haltRecognitionPipeline()
|
|
self.recognizer = nil
|
|
self.currentConfig = nil
|
|
self.listeningState = .idle
|
|
self.activeTriggerEndTime = nil
|
|
self.logger.debug("voicewake runtime stopped")
|
|
DiagnosticsFileLog.shared.log(category: "voicewake.runtime", event: "stopped")
|
|
|
|
let token = self.overlayToken
|
|
self.overlayToken = nil
|
|
guard dismissOverlay else { return }
|
|
Task { @MainActor in
|
|
if let token {
|
|
VoiceSessionCoordinator.shared.dismiss(token: token, reason: .explicit, outcome: .empty)
|
|
} else {
|
|
VoiceWakeOverlayController.shared.dismiss()
|
|
}
|
|
}
|
|
}
|
|
|
|
private func configureSession(localeID: String?) {
|
|
let locale = localeID.flatMap { Locale(identifier: $0) } ?? Locale(identifier: Locale.current.identifier)
|
|
self.recognizer = SFSpeechRecognizer(locale: locale)
|
|
}
|
|
|
|
private func handleRecognition(_ update: RecognitionUpdate, config: RuntimeConfig) async {
|
|
if update.generation != self.recognitionGeneration {
|
|
return // stale callback from a superseded recognizer session
|
|
}
|
|
if let error = update.error {
|
|
self.logger.debug("voicewake recognition error: \(error.localizedDescription, privacy: .public)")
|
|
}
|
|
|
|
guard let transcript = update.transcript else { return }
|
|
|
|
let now = Date()
|
|
if !transcript.isEmpty {
|
|
self.lastHeard = now
|
|
if self.isCapturing {
|
|
let trimmed = Self.commandAfterTrigger(
|
|
transcript: transcript,
|
|
segments: update.segments,
|
|
triggerEndTime: self.activeTriggerEndTime,
|
|
triggers: config.triggers)
|
|
self.capturedTranscript = trimmed
|
|
self.updateHeardBeyondTrigger(withTrimmed: trimmed)
|
|
if update.isFinal {
|
|
self.committedTranscript = trimmed
|
|
self.volatileTranscript = ""
|
|
} else {
|
|
self.volatileTranscript = Self.delta(after: self.committedTranscript, current: trimmed)
|
|
}
|
|
|
|
let attributed = Self.makeAttributed(
|
|
committed: self.committedTranscript,
|
|
volatile: self.volatileTranscript,
|
|
isFinal: update.isFinal)
|
|
let snapshot = self.committedTranscript + self.volatileTranscript
|
|
if let token = self.overlayToken {
|
|
await MainActor.run {
|
|
VoiceSessionCoordinator.shared.updatePartial(
|
|
token: token,
|
|
text: snapshot,
|
|
attributed: attributed)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if self.isCapturing { return }
|
|
|
|
let gateConfig = WakeWordGateConfig(triggers: config.triggers)
|
|
if let match = WakeWordGate.match(transcript: transcript, segments: update.segments, config: gateConfig) {
|
|
if let cooldown = cooldownUntil, now < cooldown {
|
|
return
|
|
}
|
|
await self.beginCapture(command: match.command, triggerEndTime: match.triggerEndTime, config: config)
|
|
}
|
|
}
|
|
|
|
private func beginCapture(command: String, triggerEndTime: TimeInterval, config: RuntimeConfig) async {
|
|
self.listeningState = .voiceWake
|
|
self.isCapturing = true
|
|
DiagnosticsFileLog.shared.log(category: "voicewake.runtime", event: "beginCapture")
|
|
self.capturedTranscript = command
|
|
self.committedTranscript = ""
|
|
self.volatileTranscript = command
|
|
self.captureStartedAt = Date()
|
|
self.cooldownUntil = nil
|
|
self.heardBeyondTrigger = !command.isEmpty
|
|
self.triggerChimePlayed = false
|
|
self.activeTriggerEndTime = triggerEndTime
|
|
|
|
if config.triggerChime != .none, !self.triggerChimePlayed {
|
|
self.triggerChimePlayed = true
|
|
await MainActor.run { VoiceWakeChimePlayer.play(config.triggerChime, reason: "voicewake.trigger") }
|
|
}
|
|
|
|
let snapshot = self.committedTranscript + self.volatileTranscript
|
|
let attributed = Self.makeAttributed(
|
|
committed: self.committedTranscript,
|
|
volatile: self.volatileTranscript,
|
|
isFinal: false)
|
|
self.overlayToken = await MainActor.run {
|
|
VoiceSessionCoordinator.shared.startSession(
|
|
source: .wakeWord,
|
|
text: snapshot,
|
|
attributed: attributed,
|
|
forwardEnabled: true)
|
|
}
|
|
|
|
// Keep the "ears" boosted for the capture window so the status icon animates while recording.
|
|
await MainActor.run { AppStateStore.shared.triggerVoiceEars(ttl: nil) }
|
|
|
|
self.captureTask?.cancel()
|
|
self.captureTask = Task { [weak self] in
|
|
guard let self else { return }
|
|
await self.monitorCapture(config: config)
|
|
}
|
|
}
|
|
|
|
private func monitorCapture(config: RuntimeConfig) async {
|
|
let start = self.captureStartedAt ?? Date()
|
|
let hardStop = start.addingTimeInterval(self.captureHardStop)
|
|
|
|
while self.isCapturing {
|
|
let now = Date()
|
|
if now >= hardStop {
|
|
// Hard-stop after a maximum duration so we never leave the recognizer pinned open.
|
|
await self.finalizeCapture(config: config)
|
|
return
|
|
}
|
|
|
|
let silenceThreshold = self.heardBeyondTrigger ? self.silenceWindow : self.triggerOnlySilenceWindow
|
|
if let last = self.lastHeard, now.timeIntervalSince(last) >= silenceThreshold {
|
|
await self.finalizeCapture(config: config)
|
|
return
|
|
}
|
|
|
|
try? await Task.sleep(nanoseconds: 200_000_000)
|
|
}
|
|
}
|
|
|
|
private func finalizeCapture(config: RuntimeConfig) async {
|
|
guard self.isCapturing else { return }
|
|
self.isCapturing = false
|
|
// Disarm trigger matching immediately (before halting recognition) to avoid double-trigger
|
|
// races from late callbacks that arrive after isCapturing is cleared.
|
|
self.cooldownUntil = Date().addingTimeInterval(self.debounceAfterSend)
|
|
self.captureTask?.cancel()
|
|
self.captureTask = nil
|
|
|
|
let finalTranscript = self.capturedTranscript.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
DiagnosticsFileLog.shared.log(category: "voicewake.runtime", event: "finalizeCapture", fields: [
|
|
"finalLen": "\(finalTranscript.count)",
|
|
])
|
|
// Stop further recognition events so we don't retrigger immediately with buffered audio.
|
|
self.haltRecognitionPipeline()
|
|
self.capturedTranscript = ""
|
|
self.captureStartedAt = nil
|
|
self.lastHeard = nil
|
|
self.heardBeyondTrigger = false
|
|
self.triggerChimePlayed = false
|
|
self.activeTriggerEndTime = nil
|
|
|
|
await MainActor.run { AppStateStore.shared.stopVoiceEars() }
|
|
if let token = self.overlayToken {
|
|
await MainActor.run { VoiceSessionCoordinator.shared.updateLevel(token: token, 0) }
|
|
}
|
|
|
|
let delay: TimeInterval = 0.0
|
|
let sendChime = finalTranscript.isEmpty ? .none : config.sendChime
|
|
if let token = self.overlayToken {
|
|
await MainActor.run {
|
|
VoiceSessionCoordinator.shared.finalize(
|
|
token: token,
|
|
text: finalTranscript,
|
|
sendChime: sendChime,
|
|
autoSendAfter: delay)
|
|
}
|
|
} else if !finalTranscript.isEmpty {
|
|
if sendChime != .none {
|
|
await MainActor.run { VoiceWakeChimePlayer.play(sendChime, reason: "voicewake.send") }
|
|
}
|
|
Task.detached {
|
|
await VoiceWakeForwarder.forward(transcript: finalTranscript)
|
|
}
|
|
}
|
|
self.overlayToken = nil
|
|
self.scheduleRestartRecognizer()
|
|
}
|
|
|
|
// MARK: - Audio level handling
|
|
|
|
private func noteAudioLevel(rms: Double) {
|
|
guard self.isCapturing else { return }
|
|
|
|
// Update adaptive noise floor: faster when lower energy (quiet), slower when loud.
|
|
let alpha: Double = rms < self.noiseFloorRMS ? 0.08 : 0.01
|
|
self.noiseFloorRMS = max(1e-7, self.noiseFloorRMS + (rms - self.noiseFloorRMS) * alpha)
|
|
|
|
let threshold = max(self.minSpeechRMS, self.noiseFloorRMS * self.speechBoostFactor)
|
|
if rms >= threshold {
|
|
self.lastHeard = Date()
|
|
}
|
|
|
|
// Normalize against the adaptive threshold so the UI meter stays roughly 0...1 across devices.
|
|
let clamped = min(1.0, max(0.0, rms / max(self.minSpeechRMS, threshold)))
|
|
if let token = self.overlayToken {
|
|
Task { @MainActor in
|
|
VoiceSessionCoordinator.shared.updateLevel(token: token, clamped)
|
|
}
|
|
}
|
|
}
|
|
|
|
private static func rmsLevel(buffer: AVAudioPCMBuffer) -> Double? {
|
|
guard let channelData = buffer.floatChannelData?.pointee else { return nil }
|
|
let frameCount = Int(buffer.frameLength)
|
|
guard frameCount > 0 else { return nil }
|
|
var sum: Double = 0
|
|
for i in 0..<frameCount {
|
|
let sample = Double(channelData[i])
|
|
sum += sample * sample
|
|
}
|
|
return sqrt(sum / Double(frameCount))
|
|
}
|
|
|
|
private func restartRecognizer() {
|
|
// Restart the recognizer so we listen for the next trigger with a clean buffer.
|
|
let current = self.currentConfig
|
|
self.stop(dismissOverlay: false, cancelScheduledRestart: false)
|
|
if let current {
|
|
Task { await self.start(with: current) }
|
|
}
|
|
}
|
|
|
|
private func restartRecognizerIfIdleAndOverlayHidden() async {
|
|
if self.isCapturing { return }
|
|
self.restartRecognizer()
|
|
}
|
|
|
|
private func scheduleRestartRecognizer(delay: TimeInterval = 0.7) {
|
|
self.scheduledRestartTask?.cancel()
|
|
self.scheduledRestartTask = Task { [weak self] in
|
|
let nanos = UInt64(max(0, delay) * 1_000_000_000)
|
|
try? await Task.sleep(nanoseconds: nanos)
|
|
guard let self else { return }
|
|
await self.restartRecognizerIfIdleAndOverlayHidden()
|
|
}
|
|
}
|
|
|
|
func applyPushToTalkCooldown() {
|
|
self.cooldownUntil = Date().addingTimeInterval(self.debounceAfterSend)
|
|
}
|
|
|
|
func pauseForPushToTalk() {
|
|
self.listeningState = .pushToTalk
|
|
self.stop(dismissOverlay: false)
|
|
}
|
|
|
|
private func updateHeardBeyondTrigger(withTrimmed trimmed: String) {
|
|
if !self.heardBeyondTrigger, !trimmed.isEmpty {
|
|
self.heardBeyondTrigger = true
|
|
}
|
|
}
|
|
|
|
private static func trimmedAfterTrigger(_ text: String, triggers: [String]) -> String {
|
|
let lower = text.lowercased()
|
|
for trigger in triggers {
|
|
let token = trigger.lowercased().trimmingCharacters(in: .whitespacesAndNewlines)
|
|
guard !token.isEmpty, let range = lower.range(of: token) else { continue }
|
|
let after = range.upperBound
|
|
let trimmed = text[after...].trimmingCharacters(in: .whitespacesAndNewlines)
|
|
return String(trimmed)
|
|
}
|
|
return text
|
|
}
|
|
|
|
private static func commandAfterTrigger(
|
|
transcript: String,
|
|
segments: [WakeWordSegment],
|
|
triggerEndTime: TimeInterval?,
|
|
triggers: [String]) -> String
|
|
{
|
|
guard let triggerEndTime else {
|
|
return self.trimmedAfterTrigger(transcript, triggers: triggers)
|
|
}
|
|
let trimmed = WakeWordGate.commandText(
|
|
transcript: transcript,
|
|
segments: segments,
|
|
triggerEndTime: triggerEndTime)
|
|
return trimmed.isEmpty ? self.trimmedAfterTrigger(transcript, triggers: triggers) : trimmed
|
|
}
|
|
|
|
#if DEBUG
|
|
static func _testTrimmedAfterTrigger(_ text: String, triggers: [String]) -> String {
|
|
self.trimmedAfterTrigger(text, triggers: triggers)
|
|
}
|
|
|
|
static func _testHasContentAfterTrigger(_ text: String, triggers: [String]) -> Bool {
|
|
!self.trimmedAfterTrigger(text, triggers: triggers).isEmpty
|
|
}
|
|
|
|
static func _testAttributedColor(isFinal: Bool) -> NSColor {
|
|
self.makeAttributed(committed: "sample", volatile: "", isFinal: isFinal)
|
|
.attribute(.foregroundColor, at: 0, effectiveRange: nil) as? NSColor ?? .clear
|
|
}
|
|
|
|
#endif
|
|
|
|
private static func delta(after committed: String, current: String) -> String {
|
|
if current.hasPrefix(committed) {
|
|
let start = current.index(current.startIndex, offsetBy: committed.count)
|
|
return String(current[start...])
|
|
}
|
|
return current
|
|
}
|
|
|
|
private static func makeAttributed(committed: String, volatile: String, isFinal: Bool) -> NSAttributedString {
|
|
let full = NSMutableAttributedString()
|
|
let committedAttr: [NSAttributedString.Key: Any] = [
|
|
.foregroundColor: NSColor.labelColor,
|
|
.font: NSFont.systemFont(ofSize: 13, weight: .regular),
|
|
]
|
|
full.append(NSAttributedString(string: committed, attributes: committedAttr))
|
|
let volatileColor: NSColor = isFinal ? .labelColor : NSColor.tertiaryLabelColor
|
|
let volatileAttr: [NSAttributedString.Key: Any] = [
|
|
.foregroundColor: volatileColor,
|
|
.font: NSFont.systemFont(ofSize: 13, weight: .regular),
|
|
]
|
|
full.append(NSAttributedString(string: volatile, attributes: volatileAttr))
|
|
return full
|
|
}
|
|
}
|