diff --git a/.cargo/config.toml b/.cargo/config.toml index d42c757d31..594c7b357f 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -9,3 +9,10 @@ debug = "line-tables-only" # cmake_minimum_required < 3.5; audiopus_sys's vendored opus declares 3.1. # Same workaround CI uses. A value already set in the environment wins. CMAKE_POLICY_VERSION_MINIMUM = "3.5" + +# MSVC otherwise reports the legacy C++98 value for `__cplusplus`, even when +# cc-rs selects C++14. sstretch uses that macro to gate its C++11 +# `std::make_unique` polyfill, which collides with the MSVC standard library. +# Match sherpa-onnx's static MSVC runtime so both native audio libraries can +# link into the same Tauri binary without LNK2038 RuntimeLibrary mismatches. +CXXFLAGS_x86_64_pc_windows_msvc = "/Zc:__cplusplus /MT" diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 00d3fba3b5..5c2f965c20 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1093,6 +1093,7 @@ dependencies = [ "serde_yaml", "sha2 0.11.0", "sherpa-onnx", + "ssstretch", "strip-ansi-escapes", "tar", "tauri", @@ -1539,6 +1540,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width 0.1.14", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -2111,6 +2123,68 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "cxx" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe442a792c7c736eea18b32a7f8a3b63cf8aafabda6760042dc2fdeda456291" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash 0.2.0", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3184a94384c663718698311a78a51ac00c484c10b4eeac06fb0a068c5f64fa2" +dependencies = [ + "cc", + "codespan-reporting", + "indexmap 2.14.0", + "proc-macro2", + "quote", + "scratch", + "syn 3.0.3", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0148d8fd1199329ddf1d157a5e134e51ceff37c6a7ddd38615c399d81cb05d8d" +dependencies = [ + "clap", + "codespan-reporting", + "indexmap 2.14.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52850339faed2eaadd24e286dc1d8268cc6f8a7bd9524d713adc9099566b4c89" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c77c856545d886c9bd5215409ebb63b925e262135248b50c79e5a5f194ee47c" +dependencies = [ + "indexmap 2.14.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "darling" version = "0.20.11" @@ -4781,6 +4855,15 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] + [[package]] name = "link-section" version = "0.19.0" @@ -8879,6 +8962,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + [[package]] name = "scrypt" version = "0.11.0" @@ -9661,6 +9750,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ssstretch" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ee31a0a494b76c8d3047aac804c5f4eb4b6e96c75414e3043b2212301bb8c6" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -9952,6 +10051,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -10572,6 +10682,15 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "termina" version = "0.3.3" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index b80684f955..a33716da42 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -130,6 +130,7 @@ regex = "1" rusqlite = { version = "0.37", features = ["bundled"] } axum = "0.8" rodio = "0.22" +ssstretch = "0.1.0" earshot = "1.0" rubato = "3.0" audioadapter-buffers = "3.0" diff --git a/desktop/src-tauri/examples/tts_speed_probe.rs b/desktop/src-tauri/examples/tts_speed_probe.rs new file mode 100644 index 0000000000..b573c1d04a --- /dev/null +++ b/desktop/src-tauri/examples/tts_speed_probe.rs @@ -0,0 +1,32 @@ +use std::time::Instant; + +#[path = "../src/huddle/playback_speed_dsp.rs"] +mod playback_speed_dsp; + +const SAMPLE_RATE: u32 = 24_000; +const TONE_SECONDS: f64 = 10.0; + +fn main() { + let tone: Vec = (0..SAMPLE_RATE * TONE_SECONDS as u32) + .map(|index| { + (2.0 * std::f32::consts::PI * 220.0 * index as f32 / SAMPLE_RATE as f32).sin() * 0.25 + }) + .collect(); + + let compensated_lookahead = playback_speed_dsp::compensated_output_latency_samples(SAMPLE_RATE); + let compensated_lookahead_ms = compensated_lookahead as f64 * 1_000.0 / SAMPLE_RATE as f64; + + for speed in [0.75_f32, 1.25, 1.5] { + let started = Instant::now(); + let output = playback_speed_dsp::process_complete_chunk(&tone, speed, SAMPLE_RATE) + .expect("process synthesized model audio"); + let elapsed = started.elapsed(); + println!( + "{speed:.2}x: processing={:.2}ms, {:.3}% realtime, output={} samples, compensated \ + DSP lookahead={compensated_lookahead_ms:.1}ms", + elapsed.as_secs_f64() * 1_000.0, + elapsed.as_secs_f64() / TONE_SECONDS * 100.0, + output.len(), + ); + } +} diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index fc90e6ab14..cd7ce1eb8a 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -54,12 +54,11 @@ pub struct AppState { pub managed_agent_processes: Mutex>, pub huddle_state: Mutex, pub huddle_audio: crate::huddle::tts_settings::HuddleAudioSettingsState, - /// Tauri app handle — stored after setup so huddle commands can emit - /// `huddle-state-changed` events without needing the handle threaded - /// through every call site. - /// + /// Tauri app handle, stored after setup so huddle commands can emit + /// `huddle-state-changed` without threading it through every call site. /// Set once during `setup()` in `lib.rs`; never cleared. pub app_handle: Mutex>, + pub tts_playback_speed: crate::huddle::playback_speed::PlaybackSpeedControl, /// Port of the localhost media streaming proxy (set during setup). pub media_proxy_port: AtomicU16, /// Set when identity resolution detected a "keyring-locked" state: the @@ -219,6 +218,7 @@ pub fn build_app_state() -> AppState { huddle_state: Mutex::new(HuddleState::default()), huddle_audio: Default::default(), app_handle: Mutex::new(None), + tts_playback_speed: crate::huddle::playback_speed::PlaybackSpeedControl::default(), media_proxy_port: AtomicU16::new(0), prevent_sleep: Arc::new(Mutex::new( crate::prevent_sleep::PreventSleepState::default(), diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 03264f80f4..be0524ee24 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -29,6 +29,7 @@ pub mod audio_output; pub mod jitter; pub mod models; pub mod pipeline; +pub mod playback_speed; pub mod playout; pub mod pocket; pub mod preprocessing; diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs index fba5464a69..f2b91c5558 100644 --- a/desktop/src-tauri/src/huddle/pipeline.rs +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -451,6 +451,7 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result Result, + transition: Arc>, +} + +impl Default for PlaybackSpeedControl { + fn default() -> Self { + Self { + speed: Arc::new(AtomicU32::new(DEFAULT_PLAYBACK_SPEED.to_bits())), + transition: Arc::new(tokio::sync::Mutex::new(())), + } + } +} + +impl PlaybackSpeedControl { + /// Return the current generated-speech playback speed. + pub fn get(&self) -> f32 { + f32::from_bits(self.speed.load(Ordering::Acquire)) + } + + /// Update the in-memory speed after validation. + pub fn set(&self, speed: f32) -> Result<(), String> { + validate_speed(speed)?; + self.speed.store(speed.to_bits(), Ordering::Release); + Ok(()) + } + + async fn persist_to_path(&self, path: &Path, speed: f32) -> Result<(), String> { + validate_speed(speed)?; + let _transition = self.transition.lock().await; + save_to_path(path, speed)?; + self.speed.store(speed.to_bits(), Ordering::Release); + Ok(()) + } +} + +#[derive(Debug, Deserialize, Serialize)] +struct PersistedPlaybackSettings { + speed: f32, +} + +/// Load the global playback speed during app setup. Logs and keeps the 1x +/// default if no valid persisted setting exists. +pub fn load_playback_speed(app: &AppHandle, control: &PlaybackSpeedControl) { + let result = settings_path(app) + .and_then(|path| load_from_path(&path)) + .and_then(|speed| control.set(speed)); + if let Err(error) = result { + eprintln!("buzz-desktop: failed to load TTS playback speed; using 1x: {error}"); + } +} + +/// Return the globally configured generated-speech playback speed. +#[tauri::command] +pub fn get_tts_playback_speed(state: State<'_, AppState>) -> f32 { + state.tts_playback_speed.get() +} + +/// Persist and apply the global generated-speech playback speed. +#[tauri::command] +pub async fn set_tts_playback_speed( + speed: f32, + app: AppHandle, + state: State<'_, AppState>, +) -> Result<(), String> { + state + .tts_playback_speed + .persist_to_path(&settings_path(&app)?, speed) + .await +} + +fn settings_path(app: &AppHandle) -> Result { + app.path() + .app_data_dir() + .map(|directory| directory.join(SETTINGS_FILE)) + .map_err(|error| format!("resolve TTS playback settings directory: {error}")) +} + +fn load_from_path(path: &Path) -> Result { + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(DEFAULT_PLAYBACK_SPEED); + } + Err(error) => return Err(format!("read {}: {error}", path.display())), + }; + let settings: PersistedPlaybackSettings = serde_json::from_slice(&bytes) + .map_err(|error| format!("parse {}: {error}", path.display()))?; + validate_speed(settings.speed)?; + Ok(settings.speed) +} + +fn save_to_path(path: &Path, speed: f32) -> Result<(), String> { + validate_speed(speed)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("create {}: {error}", parent.display()))?; + } + let payload = serde_json::to_vec_pretty(&PersistedPlaybackSettings { speed }) + .map_err(|error| format!("serialize TTS playback settings: {error}"))?; + let resolved = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let mut file = AtomicWriteFile::open(&resolved) + .map_err(|error| format!("open {} for atomic write: {error}", resolved.display()))?; + file.write_all(&payload) + .map_err(|error| format!("write {}: {error}", resolved.display()))?; + file.commit() + .map_err(|error| format!("commit {}: {error}", resolved.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_RATE: u32 = 24_000; + + #[test] + fn unity_processing_is_a_bit_exact_bypass() { + let input = vec![0.0, 0.125, -0.5, 1.0]; + let output = process_complete_chunk(&input, 1.0, SAMPLE_RATE).expect("unity output"); + assert_eq!(output, input); + } + + #[test] + fn complete_processing_preserves_length_pitch_and_order() { + let input: Vec = (0..24_000) + .map(|sample| { + let frequency = if sample < 8_000 { + 220.0 + } else if sample < 16_000 { + 440.0 + } else { + 660.0 + }; + (2.0 * std::f32::consts::PI * frequency * sample as f32 / SAMPLE_RATE as f32).sin() + }) + .collect(); + let output = process_complete_chunk(&input, 1.25, SAMPLE_RATE).expect("complete output"); + + assert_eq!(output.len(), 19_200); + let first_frequency = zero_crossing_frequency(&output[1_500..5_000], SAMPLE_RATE); + let second_frequency = zero_crossing_frequency(&output[7_500..11_000], SAMPLE_RATE); + let third_frequency = zero_crossing_frequency(&output[14_000..18_000], SAMPLE_RATE); + assert!( + (first_frequency - 220.0).abs() < 8.0, + "first segment measured {first_frequency} Hz" + ); + assert!( + (second_frequency - 440.0).abs() <= 10.0, + "second segment measured {second_frequency} Hz" + ); + assert!( + (third_frequency - 660.0).abs() <= 12.0, + "third segment measured {third_frequency} Hz" + ); + } + + #[test] + fn processor_preserves_sine_pitch() { + let frequency = 220.0_f32; + let input: Vec = (0..SAMPLE_RATE * 2) + .map(|sample| { + (2.0 * std::f32::consts::PI * frequency * sample as f32 / SAMPLE_RATE as f32).sin() + }) + .collect(); + let output = process_complete_chunk(&input, 1.5, SAMPLE_RATE).expect("complete output"); + assert!( + root_mean_square(&output[..480]) > 0.2, + "full reset pre-roll was not removed" + ); + assert!( + root_mean_square(&output[output.len() - 480..]) > 0.2, + "speech tail was truncated" + ); + let measured = zero_crossing_frequency(&output[2_000..], SAMPLE_RATE); + assert!( + (measured - frequency).abs() < 3.0, + "expected {frequency} Hz, measured {measured} Hz" + ); + } + + #[test] + fn complete_processing_preserves_onset_timing() { + let input: Vec = (0..SAMPLE_RATE) + .map(|sample| { + if (4_800..12_000).contains(&sample) || sample >= 16_800 { + (2.0 * std::f32::consts::PI * 220.0 * sample as f32 / SAMPLE_RATE as f32).sin() + } else { + 0.0 + } + }) + .collect(); + let output = process_complete_chunk(&input, 1.25, SAMPLE_RATE).expect("complete output"); + let active_windows = active_window_starts(&output, 240, 0.15); + + let first_onset = *active_windows.first().expect("first tone onset"); + let second_onset = active_windows + .windows(2) + .find_map(|pair| (pair[1] > pair[0] + 480).then_some(pair[1])) + .expect("second tone onset"); + assert!( + (3_200..=4_400).contains(&first_onset), + "first onset shifted to sample {first_onset}" + ); + assert!( + (12_800..=14_000).contains(&second_onset), + "second onset shifted to sample {second_onset}" + ); + } + + #[test] + fn non_unity_latency_stays_below_75_ms() { + let mut stretch = ssstretch::Stretch::new(); + stretch.preset_default(1, SAMPLE_RATE as f32); + let latency_ms = stretch.output_latency().max(0) as f64 * 1_000.0 / SAMPLE_RATE as f64; + assert!(latency_ms <= 75.0, "algorithmic latency was {latency_ms}ms"); + } + + #[test] + fn persisted_speed_round_trips_and_rejects_invalid_values() { + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join(SETTINGS_FILE); + save_to_path(&path, playback_speed_dsp::MIN_PLAYBACK_SPEED).expect("save minimum"); + assert_eq!( + load_from_path(&path).expect("load minimum"), + playback_speed_dsp::MIN_PLAYBACK_SPEED + ); + save_to_path(&path, playback_speed_dsp::MAX_PLAYBACK_SPEED).expect("save maximum"); + assert_eq!( + load_from_path(&path).expect("load maximum"), + playback_speed_dsp::MAX_PLAYBACK_SPEED + ); + + std::fs::write(&path, br#"{"speed":4.1}"#).expect("invalid fixture"); + assert!(load_from_path(&path).is_err()); + } + + #[tokio::test] + async fn serialized_persistence_finishes_with_the_latest_speed() { + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join(SETTINGS_FILE); + let control = PlaybackSpeedControl::default(); + let transition = control.transition.lock().await; + + let (first_enqueued_tx, first_enqueued_rx) = tokio::sync::oneshot::channel(); + let first_control = control.clone(); + let first_path = path.clone(); + let first = tokio::spawn(async move { + first_enqueued_tx.send(()).expect("signal first waiter"); + first_control.persist_to_path(&first_path, 1.25).await + }); + first_enqueued_rx.await.expect("first waiter started"); + tokio::task::yield_now().await; + + let (second_enqueued_tx, second_enqueued_rx) = tokio::sync::oneshot::channel(); + let second_control = control.clone(); + let second_path = path.clone(); + let second = tokio::spawn(async move { + second_enqueued_tx.send(()).expect("signal second waiter"); + second_control.persist_to_path(&second_path, 1.5).await + }); + second_enqueued_rx.await.expect("second waiter started"); + tokio::task::yield_now().await; + + drop(transition); + first.await.expect("join first save").expect("first save"); + second + .await + .expect("join second save") + .expect("second save"); + + assert_eq!(load_from_path(&path).expect("load final speed"), 1.5); + assert_eq!(control.get(), 1.5); + } + + fn zero_crossing_frequency(samples: &[f32], sample_rate: u32) -> f32 { + let crossings = samples + .windows(2) + .filter(|pair| pair[0] <= 0.0 && pair[1] > 0.0) + .count(); + crossings as f32 * sample_rate as f32 / samples.len() as f32 + } + + fn root_mean_square(samples: &[f32]) -> f32 { + (samples.iter().map(|sample| sample * sample).sum::() / samples.len() as f32).sqrt() + } + + fn active_window_starts(samples: &[f32], window: usize, threshold: f32) -> Vec { + samples + .chunks_exact(window) + .enumerate() + .filter_map(|(index, chunk)| { + (root_mean_square(chunk) > threshold).then_some(index * window) + }) + .collect() + } +} diff --git a/desktop/src-tauri/src/huddle/playback_speed_dsp.rs b/desktop/src-tauri/src/huddle/playback_speed_dsp.rs new file mode 100644 index 0000000000..7a496ad501 --- /dev/null +++ b/desktop/src-tauri/src/huddle/playback_speed_dsp.rs @@ -0,0 +1,119 @@ +//! Pure pitch-preserving playback-speed processing. +//! +//! This module intentionally has no application, persistence, or filesystem +//! dependencies so production playback and diagnostic examples can compile +//! the exact same DSP boundary. + +/// Slowest supported generated-speech playback speed. +pub const MIN_PLAYBACK_SPEED: f32 = 0.25; +/// Fastest supported generated-speech playback speed. +pub const MAX_PLAYBACK_SPEED: f32 = 4.0; +/// Default generated-speech playback speed. +pub const DEFAULT_PLAYBACK_SPEED: f32 = 1.0; + +pub(super) const UNITY_EPSILON: f32 = 0.000_1; + +/// Pitch-preserve one already-buffered Pocket chunk. +/// +/// The returned buffer has exactly `input.len() / speed` samples. Signalsmith +/// starts a reset processor `input_latency` samples before the supplied audio +/// and emits another `output_latency` samples of pre-roll. Both components are +/// removed after draining, so the returned chunk retains its beginning and +/// tail without buffering any later Pocket chunk. +pub fn process_complete_chunk( + input: &[f32], + speed: f32, + sample_rate: u32, +) -> Result, String> { + validate_speed(speed)?; + if input.is_empty() || (speed - DEFAULT_PLAYBACK_SPEED).abs() <= UNITY_EPSILON { + return Ok(input.to_vec()); + } + + let expected = (input.len() as f64 / speed as f64).round() as usize; + let mut stretch = ssstretch::Stretch::new(); + stretch.preset_default(1, sample_rate as f32); + let input_latency = stretch.input_latency().max(0) as usize; + let output_latency = stretch.output_latency().max(0) as usize; + let reset_pre_roll = (input_latency as f64 / speed as f64).ceil() as usize; + + let inputs = [input.to_vec()]; + let mut output = [Vec::with_capacity(expected)]; + stretch.process_vec( + &inputs, + i32_len(input.len())?, + &mut output, + i32_len(expected)?, + ); + + let latency_input = [vec![0.0; input_latency]]; + let mut latency_output = [Vec::with_capacity(reset_pre_roll)]; + stretch.process_vec( + &latency_input, + i32_len(input_latency)?, + &mut latency_output, + i32_len(reset_pre_roll)?, + ); + output[0].extend_from_slice(&latency_output[0]); + + let mut flushed = [Vec::with_capacity(output_latency)]; + stretch.flush_vec(&mut flushed, i32_len(output_latency)?); + output[0].extend_from_slice(&flushed[0]); + + let start = reset_pre_roll.saturating_add(output_latency); + let end = start.saturating_add(expected); + if output[0].len() < end { + return Err(format!( + "time stretcher produced {} samples, need {end}", + output[0].len() + )); + } + Ok(output[0][start..end].to_vec()) +} + +/// Pitch-preserve one complete playback chunk across its model-unit splits. +/// +/// Pocket may divide a natural synthesis chunk into multiple model-valid +/// units. Joining those units before processing keeps one stretcher timeline +/// across the hidden boundaries instead of resetting the DSP at each unit. +// This file is also compiled directly by the single-buffer diagnostic example. +#[allow(dead_code)] +pub fn process_complete_playback_chunk( + model_units: &[Vec], + speed: f32, + sample_rate: u32, +) -> Result, String> { + let sample_count = model_units.iter().try_fold(0_usize, |total, unit| { + total + .checked_add(unit.len()) + .ok_or_else(|| "audio chunk is too large to process".to_string()) + })?; + let mut samples = Vec::with_capacity(sample_count); + for unit in model_units { + samples.extend_from_slice(unit); + } + process_complete_chunk(&samples, speed, sample_rate) +} + +/// Return Signalsmith's compensated output lookahead for descriptive reporting. +#[allow(dead_code)] +pub(crate) fn compensated_output_latency_samples(sample_rate: u32) -> usize { + let mut stretch = ssstretch::Stretch::new(); + stretch.preset_default(1, sample_rate as f32); + stretch.output_latency().max(0) as usize +} + +/// Validate a generated-speech playback speed. +pub fn validate_speed(speed: f32) -> Result<(), String> { + if speed.is_finite() && (MIN_PLAYBACK_SPEED..=MAX_PLAYBACK_SPEED).contains(&speed) { + Ok(()) + } else { + Err(format!( + "Speech playback speed must be between {MIN_PLAYBACK_SPEED} and {MAX_PLAYBACK_SPEED}" + )) + } +} + +fn i32_len(length: usize) -> Result { + i32::try_from(length).map_err(|_| "audio chunk is too large to process".to_string()) +} diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index c03589f9fe..c11b4e7756 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -47,6 +47,7 @@ use std::{ time::Duration, }; +use super::playback_speed::{process_complete_playback_chunk, PlaybackSpeedControl}; use super::pocket::{ load_text_to_speech, load_voice_style, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT, }; @@ -167,6 +168,7 @@ impl TtsPipeline { cancel: Arc, voice: &str, output_device: Option, + playback_speed: PlaybackSpeedControl, ) -> Result { let (text_tx, text_rx) = mpsc::sync_channel::(TEXT_QUEUE_DEPTH); let shutdown = Arc::new(AtomicBool::new(false)); @@ -204,6 +206,7 @@ impl TtsPipeline { ), output_device, startup_tx, + playback_speed, ) }) .map_err(|e| format!("failed to spawn tts-worker thread: {e}"))?; @@ -310,6 +313,7 @@ fn tts_worker( control_state: WorkerControlState, output_device: Option, startup_tx: mpsc::SyncSender>, + playback_speed: PlaybackSpeedControl, ) { let (selected_voice, voice_generation, voice_change_ack) = voice_state; let (tts_active, shutdown, cancel_signals) = control_state; @@ -710,7 +714,8 @@ fn tts_worker( ); continue; } - let mut playback_audio = PlaybackChunkAudio::new(); + let mut playback_units = Vec::with_capacity(model_chunks.len()); + let mut first_model_unit_index = None; for model_chunk in &model_chunks { let chunk_index = model_unit_index; model_unit_index += 1; @@ -754,21 +759,8 @@ fn tts_worker( } match synthesis { Ok(samples) if !samples.is_empty() => { - if let Some(prepared) = playback_audio.push( - samples, - chunk_index, - &mut first_append, - silence_buf_len, - player.empty(), - ) { - if !append_audio(prepared, route_id) { - first_append = true; - synthesis_outcome = "cancelled"; - break 'playback_chunks; - } - appended_audio = true; - last_route_id = route_id; - } + first_model_unit_index.get_or_insert(chunk_index); + playback_units.push(samples); } Ok(_) => { eprintln!( @@ -784,16 +776,39 @@ fn tts_worker( } } } - if let Some(prepared) = - playback_audio.finish(&mut first_append, silence_buf_len, player.empty()) - { - if !append_audio(prepared, route_id) { - first_append = true; - synthesis_outcome = "cancelled"; - break 'playback_chunks; + if let Some(chunk_index) = first_model_unit_index { + let speed = playback_speed.get(); + let samples = + match process_complete_playback_chunk(&playback_units, speed, SAMPLE_RATE) { + Ok(processed) => processed, + Err(error) => { + eprintln!( + "buzz-desktop: TTS playback-speed processing failed at \ + {speed:.2}x: {error}; using 1x playback" + ); + playback_units.into_iter().flatten().collect() + } + }; + let mut playback_audio = PlaybackChunkAudio::new(); + let completed_audio = playback_audio.push( + samples, + chunk_index, + &mut first_append, + silence_buf_len, + player.empty(), + ); + debug_assert!(completed_audio.is_none()); + if let Some(prepared) = + playback_audio.finish(&mut first_append, silence_buf_len, player.empty()) + { + if !append_audio(prepared, route_id) { + first_append = true; + synthesis_outcome = "cancelled"; + break 'playback_chunks; + } + appended_audio = true; + last_route_id = route_id; } - appended_audio = true; - last_route_id = route_id; } if synthesis_outcome == "failed" { break 'playback_chunks; @@ -905,6 +920,9 @@ fn lock_player_ops(ops: &Mutex<()>) -> MutexGuard<'_, ()> { // ── Tests ───────────────────────────────────────────────────────────────────── +#[cfg(test)] +#[path = "tts_playback_speed_tests.rs"] +mod playback_speed_tests; #[cfg(test)] #[path = "tts_tests.rs"] mod tests; diff --git a/desktop/src-tauri/src/huddle/tts_playback_speed_tests.rs b/desktop/src-tauri/src/huddle/tts_playback_speed_tests.rs new file mode 100644 index 0000000000..647c81b386 --- /dev/null +++ b/desktop/src-tauri/src/huddle/tts_playback_speed_tests.rs @@ -0,0 +1,53 @@ +use super::*; +use crate::huddle::playback_speed::process_complete_chunk; + +/// Regression guard: playback decoration happens after speed processing, so +/// the fixed device cushion is never time-stretched. +#[test] +fn playback_speed_preserves_production_sentence_lead_in() { + let processed = process_complete_chunk(&vec![0.5; 2_400], 1.5, SAMPLE_RATE) + .expect("process synthesized model audio"); + let mut first = true; + let processed = build_sentence_append_buffer(&mut first, processed, 2_400, true, true); + + assert_eq!(SENTENCE_LEAD_IN_SAMPLES, 480); + assert!( + processed[..SENTENCE_LEAD_IN_SAMPLES] + .iter() + .all(|&sample| sample == 0.0), + "the fixed device cushion must remain pure zero" + ); + assert!( + processed[SENTENCE_LEAD_IN_SAMPLES..] + .iter() + .any(|sample| sample.abs() > 0.1), + "speech energy must remain after the fixed device cushion" + ); +} + +#[test] +fn playback_speed_keeps_one_timeline_across_model_units() { + let frequency = 220.0_f32; + let input: Vec = (0..SAMPLE_RATE) + .map(|sample| { + (2.0 * std::f32::consts::PI * frequency * sample as f32 / SAMPLE_RATE as f32).sin() + }) + .collect(); + let units = vec![input[..12_000].to_vec(), input[12_000..].to_vec()]; + + let joined = process_complete_playback_chunk(&units, 1.25, SAMPLE_RATE) + .expect("process complete playback chunk"); + let expected = + process_complete_chunk(&input, 1.25, SAMPLE_RATE).expect("process already-joined input"); + + assert_eq!(joined, expected, "model units must share one stretcher run"); + let boundary = joined.len() / 2; + let largest_step = joined[boundary - 240..boundary + 240] + .windows(2) + .map(|pair| (pair[1] - pair[0]).abs()) + .fold(0.0_f32, f32::max); + assert!( + largest_step < 0.1, + "hidden model boundary introduced a {largest_step} sample discontinuity" + ); +} diff --git a/desktop/src-tauri/src/huddle/tts_settings.rs b/desktop/src-tauri/src/huddle/tts_settings.rs index 1b378af823..099e6ad34a 100644 --- a/desktop/src-tauri/src/huddle/tts_settings.rs +++ b/desktop/src-tauri/src/huddle/tts_settings.rs @@ -614,6 +614,7 @@ pub async fn preview_pocket_voice( .lock() .unwrap_or_else(|error| error.into_inner()) .clone(); + let playback_speed = state.tts_playback_speed.clone(); let voice_name = pocket_voice_reference(&app, std::slice::from_ref(&voice_key))?; tokio::task::spawn_blocking(move || { let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -624,6 +625,7 @@ pub async fn preview_pocket_voice( cancel, &voice_name, output_device, + playback_speed, )?; pipeline.speak("Hello! This is how I’ll read agent responses.".to_string())?; let started = std::time::Instant::now(); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 6814008f0d..ac3f7f1f85 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -417,6 +417,7 @@ pub fn run() { // that will be lost on restart, as that silently breaks channel // memberships, DMs, and relay identity. let state = app_handle.state::(); + huddle::playback_speed::load_playback_speed(&app_handle, &state.tts_playback_speed); if let Err(e) = resolve_persisted_identity(&app_handle, &state) { eprintln!("buzz-desktop: fatal: identity resolution failed: {e}"); std::process::exit(1); @@ -906,6 +907,8 @@ pub fn run() { list_audio_output_devices, set_audio_output_device, get_audio_output_device, + huddle::playback_speed::get_tts_playback_speed, + huddle::playback_speed::set_tts_playback_speed, start_pairing, confirm_pairing_sas, cancel_pairing, diff --git a/desktop/src/features/settings/ui/SpeechPlaybackSettings.tsx b/desktop/src/features/settings/ui/SpeechPlaybackSettings.tsx new file mode 100644 index 0000000000..254667265e --- /dev/null +++ b/desktop/src/features/settings/ui/SpeechPlaybackSettings.tsx @@ -0,0 +1,144 @@ +import { useCallback, useEffect, useState } from "react"; +import { + getTtsPlaybackSpeed, + setTtsPlaybackSpeed, +} from "@/shared/api/ttsPlayback"; +import { Button } from "@/shared/ui/button"; +import { createPlaybackSpeedPersistence } from "./playbackSpeedPersistence"; +import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; + +const DEFAULT_SPEED = 1; +const SLIDER_MIN_SPEED = 0.5; +const SLIDER_MAX_SPEED = 2; +const INPUT_MIN_SPEED = 0.25; +const INPUT_MAX_SPEED = 4; +const SPEED_STEP = 0.05; +const playbackSpeedPersistence = createPlaybackSpeedPersistence( + setTtsPlaybackSpeed, + DEFAULT_SPEED, +); + +export function SpeechPlaybackSettings() { + const [speed, setSpeed] = useState(DEFAULT_SPEED); + const [speedInput, setSpeedInput] = useState(String(DEFAULT_SPEED)); + const [error, setError] = useState(null); + const [loaded, setLoaded] = useState(false); + const setDisplayedSpeed = useCallback((nextSpeed: number) => { + setSpeed(nextSpeed); + setSpeedInput(String(nextSpeed)); + }, []); + + useEffect(() => { + let loadActive = true; + const unsubscribe = playbackSpeedPersistence.subscribe({ + setError, + setSpeed: setDisplayedSpeed, + }); + getTtsPlaybackSpeed() + .then((savedSpeed) => { + if (loadActive) { + playbackSpeedPersistence.applyLoaded(savedSpeed); + } + }) + .catch((cause) => { + if (loadActive) setError(String(cause)); + }) + .finally(() => { + if (loadActive) setLoaded(true); + }); + return () => { + loadActive = false; + unsubscribe(); + }; + }, [setDisplayedSpeed]); + + const commitSpeed = (nextSpeed: number) => { + playbackSpeedPersistence.commit(nextSpeed); + }; + + const commitTypedSpeed = () => { + const nextSpeed = Number(speedInput); + if (Number.isFinite(nextSpeed)) { + commitSpeed(nextSpeed); + } else { + setSpeedInput(String(speed)); + } + }; + + return ( + + +
+
+
+ +
+
+
+ setSpeedInput(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + commitTypedSpeed(); + event.currentTarget.blur(); + } + }} + step={SPEED_STEP} + type="number" + value={speedInput} + /> + +
+ +
+
+ commitSpeed(Number(event.currentTarget.value))} + onChange={(event) => + setDisplayedSpeed(Number(event.currentTarget.value)) + } + onKeyUp={(event) => commitSpeed(Number(event.currentTarget.value))} + onPointerUp={(event) => + commitSpeed(Number(event.currentTarget.value)) + } + step={SPEED_STEP} + type="range" + value={speed} + /> + {error ? ( +

+ Could not save playback speed: {error} +

+ ) : null} +
+ + + ); +} diff --git a/desktop/src/features/settings/ui/VoiceSettingsCard.tsx b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx index 30d259fece..b2654f26fb 100644 --- a/desktop/src/features/settings/ui/VoiceSettingsCard.tsx +++ b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx @@ -24,6 +24,7 @@ import { import { Switch } from "@/shared/ui/switch"; import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; import { SettingsSectionHeader } from "./SettingsSectionHeader"; +import { SpeechPlaybackSettings } from "./SpeechPlaybackSettings"; import { selectedVoiceForBackend, type VoiceRegistryEntry, @@ -324,6 +325,8 @@ export function VoiceSettingsCard() { + + {error && (

{ + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +} + +function harness() { + const calls = []; + const saves = []; + const persist = (speed) => { + calls.push(speed); + const save = deferred(); + saves.push(save); + return save.promise; + }; + return { + calls, + controller: createPlaybackSpeedPersistence(persist), + saves, + }; +} + +function observe(controller) { + const errors = []; + const speeds = []; + const unsubscribe = controller.subscribe({ + setError: (error) => errors.push(error), + setSpeed: (speed) => speeds.push(speed), + }); + return { errors, speeds, unsubscribe }; +} + +test("rapid intents persist serially and finish at the latest speed", async () => { + const h = harness(); + const listener = observe(h.controller); + h.controller.commit(1.25); + h.controller.commit(1.5); + assert.deepEqual(h.calls, [1.25]); + + h.saves[0].resolve(); + await Promise.resolve(); + assert.deepEqual(h.calls, [1.25, 1.5]); + h.saves[1].resolve(); + await Promise.resolve(); + + assert.equal(listener.speeds.at(-1), 1.5); +}); + +test("a stale failure does not roll back a newer successful intent", async () => { + const h = harness(); + const listener = observe(h.controller); + h.controller.commit(1.25); + h.controller.commit(1.5); + + h.saves[0].reject(new Error("first failed")); + await Promise.resolve(); + assert.deepEqual(h.calls, [1.25, 1.5]); + h.saves[1].resolve(); + await Promise.resolve(); + + assert.equal(listener.speeds.at(-1), 1.5); + assert.equal(listener.errors.at(-1), null); +}); + +test("the latest failure rolls back to the last confirmed speed", async () => { + const h = harness(); + const listener = observe(h.controller); + h.controller.commit(1.25); + h.saves[0].reject(new Error("save failed")); + await Promise.resolve(); + await Promise.resolve(); + + assert.equal(listener.speeds.at(-1), 1); + assert.equal(listener.errors.at(-1), "Error: save failed"); +}); + +test("a delayed initial load cannot overwrite a local intent", async () => { + const h = harness(); + const listener = observe(h.controller); + h.controller.commit(1.5); + h.controller.applyLoaded(1); + + assert.equal(listener.speeds.at(-1), 1.5); + h.saves[0].resolve(); + await Promise.resolve(); +}); + +test("the latest intent survives unmount and is visible after remount", async () => { + const h = harness(); + const firstMount = observe(h.controller); + h.controller.commit(1.25); + h.controller.commit(1.5); + firstMount.unsubscribe(); + + h.saves[0].resolve(); + await Promise.resolve(); + assert.deepEqual(h.calls, [1.25, 1.5]); + h.saves[1].resolve(); + await Promise.resolve(); + + const secondMount = observe(h.controller); + assert.equal(secondMount.speeds.at(-1), 1.5); + assert.equal(secondMount.errors.at(-1), null); +}); diff --git a/desktop/src/features/settings/ui/playbackSpeedPersistence.ts b/desktop/src/features/settings/ui/playbackSpeedPersistence.ts new file mode 100644 index 0000000000..59664096aa --- /dev/null +++ b/desktop/src/features/settings/ui/playbackSpeedPersistence.ts @@ -0,0 +1,72 @@ +type PlaybackSpeedPersistenceCallbacks = { + setSpeed: (speed: number) => void; + setError: (error: string | null) => void; +}; + +export function createPlaybackSpeedPersistence( + persist: (speed: number) => Promise, + initialSpeed = 1, +) { + let confirmedSpeed = initialSpeed; + let desiredSpeed = initialSpeed; + let error: string | null = null; + let flushPromise: Promise | null = null; + let hasLocalIntent = false; + const listeners = new Set(); + + const notify = () => { + for (const listener of listeners) { + listener.setSpeed(desiredSpeed); + listener.setError(error); + } + }; + + const flush = async () => { + while (desiredSpeed !== confirmedSpeed) { + const target = desiredSpeed; + try { + await persist(target); + confirmedSpeed = target; + } catch (cause) { + if (desiredSpeed === target) { + desiredSpeed = confirmedSpeed; + error = String(cause); + notify(); + return; + } + } + } + }; + + const ensureFlush = () => { + if (flushPromise) return; + const currentFlush = flush().finally(() => { + if (flushPromise !== currentFlush) return; + flushPromise = null; + if (desiredSpeed !== confirmedSpeed) ensureFlush(); + }); + flushPromise = currentFlush; + }; + + return { + applyLoaded(savedSpeed: number) { + if (hasLocalIntent) return; + confirmedSpeed = savedSpeed; + desiredSpeed = savedSpeed; + notify(); + }, + commit(nextSpeed: number) { + hasLocalIntent = true; + desiredSpeed = nextSpeed; + error = null; + notify(); + ensureFlush(); + }, + subscribe(callbacks: PlaybackSpeedPersistenceCallbacks) { + listeners.add(callbacks); + callbacks.setSpeed(desiredSpeed); + callbacks.setError(error); + return () => listeners.delete(callbacks); + }, + }; +} diff --git a/desktop/src/shared/api/ttsPlayback.ts b/desktop/src/shared/api/ttsPlayback.ts new file mode 100644 index 0000000000..7a0c3c2dcd --- /dev/null +++ b/desktop/src/shared/api/ttsPlayback.ts @@ -0,0 +1,7 @@ +import { invokeTauri } from "@/shared/api/tauri"; + +export const getTtsPlaybackSpeed = () => + invokeTauri("get_tts_playback_speed"); + +export const setTtsPlaybackSpeed = (speed: number) => + invokeTauri("set_tts_playback_speed", { speed }); diff --git a/desktop/src/testing/e2eBridge.ts b/desktop/src/testing/e2eBridge.ts index 818144415a..a58b76092a 100644 --- a/desktop/src/testing/e2eBridge.ts +++ b/desktop/src/testing/e2eBridge.ts @@ -2873,6 +2873,7 @@ let mockClosedChannelLiveSubscription = false; const realSockets = new Map(); let mockManagedAgents: MockManagedAgent[] = []; let mockManagedAgentRuntimes: MockManagedAgentRuntimeRow[] = []; +let mockTtsPlaybackSpeed = 1; // Mutable `save_subscriptions` table mirror — TEST-ONLY. // @@ -10206,6 +10207,11 @@ export function maybeInstallE2eTauriMocks() { registry: await handleMockCommand("list_voice_registry", null), }; } + case "get_tts_playback_speed": + return mockTtsPlaybackSpeed; + case "set_tts_playback_speed": + mockTtsPlaybackSpeed = (payload as { speed: number }).speed; + return null; case "get_builderlab_auth": return activeConfig?.mock?.builderlabAuth ?? null; case "start_builderlab_login": {