let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fallback mp3 when pcm blocked

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-30 14:32:47 +01:00
parent 3aefe375c1
commit d7b267843e
4 changed files with 60 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
apps/ios/Sources/Voice/TalkModeManager.swift
View File

@@ -473,19 +473,23 @@ final class TalkModeManager: NSObject {
}
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
let request = ElevenLabsTTSRequest(
text: cleaned,
modelId: modelId,
outputFormat: outputFormat,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
style: TalkTTSValidation.validatedUnit(directive?.style),
speakerBoost: directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
ElevenLabsTTSRequest(
text: cleaned,
modelId: modelId,
outputFormat: outputFormat,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
style: TalkTTSValidation.validatedUnit(directive?.style),
speakerBoost: directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
}
let request = makeRequest(outputFormat: outputFormat)
let client = ElevenLabsTTSClient(apiKey: apiKey)
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
@@ -504,7 +508,17 @@ final class TalkModeManager: NSObject {
let result: StreamingPlaybackResult
if let sampleRate {
self.lastPlaybackWasPCM = true
result = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
if !playback.finished, playback.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
self.logger.warning("pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: makeRequest(outputFormat: mp3Format))
playback = await self.mp3Player.play(stream: mp3Stream)
}
result = playback
} else {
self.lastPlaybackWasPCM = false
result = await self.mp3Player.play(stream: stream)

39
apps/macos/Sources/Clawdis/TalkModeRuntime.swift
View File

@@ -507,19 +507,23 @@ actor TalkModeRuntime {
}
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
let request = ElevenLabsTTSRequest(
text: cleaned,
modelId: modelId,
outputFormat: outputFormat,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
style: TalkTTSValidation.validatedUnit(directive?.style),
speakerBoost: directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
ElevenLabsTTSRequest(
text: cleaned,
modelId: modelId,
outputFormat: outputFormat,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
style: TalkTTSValidation.validatedUnit(directive?.style),
speakerBoost: directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
}
let request = makeRequest(outputFormat: outputFormat)
self.ttsLogger.info("talk TTS synth timeout=\(synthTimeoutSeconds, privacy: .public)s")
let client = ElevenLabsTTSClient(apiKey: apiKey)
@@ -539,6 +543,15 @@ actor TalkModeRuntime {
if let sampleRate {
self.lastPlaybackWasPCM = true
result = await self.playPCM(stream: stream, sampleRate: sampleRate)
if !result.finished, result.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
self.ttsLogger.warning("talk pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: makeRequest(outputFormat: mp3Format))
result = await self.playMP3(stream: mp3Stream)
}
} else {
self.lastPlaybackWasPCM = false
result = await self.playMP3(stream: stream)

5
apps/macos/Sources/Clawdis/TalkOverlay.swift
View File

@@ -112,6 +112,7 @@ final class TalkOverlayController {
panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient]
panel.hidesOnDeactivate = false
panel.isMovable = false
panel.acceptsMouseMovedEvents = true
panel.isFloatingPanel = true
panel.becomesKeyOnlyIfNeeded = true
panel.titleVisibility = .hidden
@@ -136,6 +137,10 @@ final class TalkOverlayController {
}
private final class TalkOverlayHostingView: NSHostingView<TalkOverlayView> {
override func acceptsFirstMouse(for event: NSEvent?) -> Bool {
true
}
override func hitTest(_ point: NSPoint) -> NSView? {
let center = CGPoint(
x: self.bounds.maxX - TalkOverlayController.orbPadding - (TalkOverlayController.orbSize / 2),

1
apps/macos/Sources/Clawdis/TalkOverlayView.swift
View File

@@ -97,6 +97,7 @@ private final class OrbInteractionNSView: NSView {
private var suppressSingleClick = false
override var acceptsFirstResponder: Bool { true }
override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
override func mouseDown(with event: NSEvent) {
self.mouseDownEvent = event