let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fallback mp3 when pcm blocked

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-30 14:32:47 +01:00
parent 3aefe375c1
commit d7b267843e
4 changed files with 60 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
apps/ios/Sources/Voice/TalkModeManager.swift
View File

@@ -473,19 +473,23 @@ final class TalkModeManager: NSObject {
} }
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
let request = ElevenLabsTTSRequest( func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
text: cleaned, ElevenLabsTTSRequest(
modelId: modelId, text: cleaned,
outputFormat: outputFormat, modelId: modelId,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM), outputFormat: outputFormat,
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId), speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity), stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
style: TalkTTSValidation.validatedUnit(directive?.style), similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
speakerBoost: directive?.speakerBoost, style: TalkTTSValidation.validatedUnit(directive?.style),
seed: TalkTTSValidation.validatedSeed(directive?.seed), speakerBoost: directive?.speakerBoost,
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize), seed: TalkTTSValidation.validatedSeed(directive?.seed),
language: language, normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier)) language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
}
let request = makeRequest(outputFormat: outputFormat)
let client = ElevenLabsTTSClient(apiKey: apiKey) let client = ElevenLabsTTSClient(apiKey: apiKey)
let stream = client.streamSynthesize(voiceId: voiceId, request: request) let stream = client.streamSynthesize(voiceId: voiceId, request: request)
@@ -504,7 +508,17 @@ final class TalkModeManager: NSObject {
let result: StreamingPlaybackResult let result: StreamingPlaybackResult
if let sampleRate { if let sampleRate {
self.lastPlaybackWasPCM = true self.lastPlaybackWasPCM = true
result = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate) var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
if !playback.finished, playback.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
self.logger.warning("pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: makeRequest(outputFormat: mp3Format))
playback = await self.mp3Player.play(stream: mp3Stream)
}
result = playback
} else { } else {
self.lastPlaybackWasPCM = false self.lastPlaybackWasPCM = false
result = await self.mp3Player.play(stream: stream) result = await self.mp3Player.play(stream: stream)

39
apps/macos/Sources/Clawdis/TalkModeRuntime.swift
View File

@@ -507,19 +507,23 @@ actor TalkModeRuntime {
} }
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
let request = ElevenLabsTTSRequest( func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
text: cleaned, ElevenLabsTTSRequest(
modelId: modelId, text: cleaned,
outputFormat: outputFormat, modelId: modelId,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM), outputFormat: outputFormat,
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId), speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity), stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
style: TalkTTSValidation.validatedUnit(directive?.style), similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
speakerBoost: directive?.speakerBoost, style: TalkTTSValidation.validatedUnit(directive?.style),
seed: TalkTTSValidation.validatedSeed(directive?.seed), speakerBoost: directive?.speakerBoost,
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize), seed: TalkTTSValidation.validatedSeed(directive?.seed),
language: language, normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier)) language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
}
let request = makeRequest(outputFormat: outputFormat)
self.ttsLogger.info("talk TTS synth timeout=\(synthTimeoutSeconds, privacy: .public)s") self.ttsLogger.info("talk TTS synth timeout=\(synthTimeoutSeconds, privacy: .public)s")
let client = ElevenLabsTTSClient(apiKey: apiKey) let client = ElevenLabsTTSClient(apiKey: apiKey)
@@ -539,6 +543,15 @@ actor TalkModeRuntime {
if let sampleRate { if let sampleRate {
self.lastPlaybackWasPCM = true self.lastPlaybackWasPCM = true
result = await self.playPCM(stream: stream, sampleRate: sampleRate) result = await self.playPCM(stream: stream, sampleRate: sampleRate)
if !result.finished, result.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
self.ttsLogger.warning("talk pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: makeRequest(outputFormat: mp3Format))
result = await self.playMP3(stream: mp3Stream)
}
} else { } else {
self.lastPlaybackWasPCM = false self.lastPlaybackWasPCM = false
result = await self.playMP3(stream: stream) result = await self.playMP3(stream: stream)

5
apps/macos/Sources/Clawdis/TalkOverlay.swift
View File

@@ -112,6 +112,7 @@ final class TalkOverlayController {
panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient] panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient]
panel.hidesOnDeactivate = false panel.hidesOnDeactivate = false
panel.isMovable = false panel.isMovable = false
panel.acceptsMouseMovedEvents = true
panel.isFloatingPanel = true panel.isFloatingPanel = true
panel.becomesKeyOnlyIfNeeded = true panel.becomesKeyOnlyIfNeeded = true
panel.titleVisibility = .hidden panel.titleVisibility = .hidden
@@ -136,6 +137,10 @@ final class TalkOverlayController {
} }
private final class TalkOverlayHostingView: NSHostingView<TalkOverlayView> { private final class TalkOverlayHostingView: NSHostingView<TalkOverlayView> {
override func acceptsFirstMouse(for event: NSEvent?) -> Bool {
true
}
override func hitTest(_ point: NSPoint) -> NSView? { override func hitTest(_ point: NSPoint) -> NSView? {
let center = CGPoint( let center = CGPoint(
x: self.bounds.maxX - TalkOverlayController.orbPadding - (TalkOverlayController.orbSize / 2), x: self.bounds.maxX - TalkOverlayController.orbPadding - (TalkOverlayController.orbSize / 2),

1
apps/macos/Sources/Clawdis/TalkOverlayView.swift
View File

@@ -97,6 +97,7 @@ private final class OrbInteractionNSView: NSView {
private var suppressSingleClick = false private var suppressSingleClick = false
override var acceptsFirstResponder: Bool { true } override var acceptsFirstResponder: Bool { true }
override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
override func mouseDown(with event: NSEvent) { override func mouseDown(with event: NSEvent) {
self.mouseDownEvent = event self.mouseDownEvent = event