diff --git a/Cargo.lock b/Cargo.lock index 88a5309..b790613 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -193,6 +193,12 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "anymap3" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170433209e817da6aae2c51aa0dd443009a613425dd041ebfb2492d1c4c11a25" + [[package]] name = "arbitrary" version = "1.4.2" @@ -222,6 +228,12 @@ dependencies = [ "syn 2.0.113", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayref" version = "0.3.9" @@ -339,6 +351,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -769,6 +792,31 @@ dependencies = [ "half", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -1070,7 +1118,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.11.1", "syn 2.0.113", ] @@ -1098,12 +1146,125 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7381b67da416b639690ac77c73b86a7b5e64a29e31d1f75fb3b1102301ef355a" +dependencies = [ + "dasp_envelope", + "dasp_frame", + "dasp_interpolate", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", + "dasp_signal", + "dasp_slice", + "dasp_window", +] + +[[package]] +name = "dasp_envelope" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ec617ce7016f101a87fe85ed44180839744265fae73bb4aa43e7ece1b7668b6" +dependencies = [ + "dasp_frame", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", +] + +[[package]] +name = "dasp_frame" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a3937f5fe2135702897535c8d4a5553f8b116f76c1529088797f2eee7c5cd6" +dependencies = [ + "dasp_sample", +] + +[[package]] +name = "dasp_interpolate" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc975a6563bb7ca7ec0a6c784ead49983a21c24835b0bc96eea11ee407c7486" +dependencies = [ + "dasp_frame", + "dasp_ring_buffer", + "dasp_sample", +] + +[[package]] +name = "dasp_peak" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf88559d79c21f3d8523d91250c397f9a15b5fc72fbb3f87fdb0a37b79915bf" +dependencies = [ + "dasp_frame", + "dasp_sample", +] + +[[package]] +name = "dasp_ring_buffer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d79e19b89618a543c4adec9c5a347fe378a19041699b3278e616e387511ea1" + +[[package]] +name = "dasp_rms" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6c5dcb30b7e5014486e2822537ea2beae50b19722ffe2ed7549ab03774575aa" +dependencies = [ + "dasp_frame", + "dasp_ring_buffer", + "dasp_sample", +] + [[package]] name = "dasp_sample" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "dasp_signal" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa1ab7d01689c6ed4eae3d38fe1cea08cba761573fbd2d592528d55b421077e7" +dependencies = [ + "dasp_envelope", + "dasp_frame", + "dasp_interpolate", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", + "dasp_window", +] + +[[package]] +name = "dasp_slice" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e1c7335d58e7baedafa516cb361360ff38d6f4d3f9d9d5ee2a2fc8e27178fa1" +dependencies = [ + "dasp_frame", + "dasp_sample", +] + +[[package]] +name = "dasp_window" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ded7b88821d2ce4e8b842c9f1c86ac911891ab89443cc1de750cae764c5076" +dependencies = [ + "dasp_sample", +] + [[package]] name = "deranged" version = "0.5.5" @@ -1341,6 +1502,19 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" +[[package]] +name = "easyfft" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767e39eef2ad8a3b6f1d733be3ec70364d21d437d06d4f18ea76ce08df20b75f" +dependencies = [ + "array-init", + "generic_singleton", + "num-complex", + "realfft", + "rustfft", +] + [[package]] name = "either" version = "1.15.0" @@ -2080,6 +2254,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "generic_singleton" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2d5de0fc83987dac514f3b910c5d08392b220efe8cf72086c660029a197bf73" +dependencies = [ + "anymap3", + "lazy_static", + "parking_lot", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -2404,6 +2589,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.5.2" @@ -2863,6 +3057,7 @@ dependencies = [ "intent-classifier", "kira", "log", + "nnnoiseless", "once_cell", "parking_lot", "platform-dirs", @@ -2894,6 +3089,7 @@ dependencies = [ "serde", "serde_json", "simple-log", + "sysinfo", "systemstat", "tauri", "tauri-build", @@ -3448,6 +3644,22 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nnnoiseless" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805d5964d1e7a0006a7fdced7dae75084d66d18b35f1dfe81bd76929b1f8da0c" +dependencies = [ + "anyhow", + "clap", + "dasp", + "dasp_interpolate", + "dasp_ring_buffer", + "easyfft", + "hound", + "once_cell", +] + [[package]] name = "nodrop" version = "0.1.14" @@ -3479,6 +3691,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" +[[package]] +name = "ntapi" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" +dependencies = [ + "winapi", +] + [[package]] name = "num" version = "0.4.3" @@ -3511,6 +3732,7 @@ checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "bytemuck", "num-traits", + "serde", ] [[package]] @@ -3577,7 +3799,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi", + "hermit-abi 0.5.2", "libc", ] @@ -3885,6 +4107,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-io-surface" version = "0.3.2" @@ -4118,6 +4350,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "owned_ttf_parser" version = "0.25.1" @@ -4439,7 +4677,7 @@ checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi", + "hermit-abi 0.5.2", "pin-project-lite", "rustix 1.1.3", "windows-sys 0.61.2", @@ -5753,6 +5991,12 @@ dependencies = [ "quote", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -5987,6 +6231,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sysinfo" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows 0.61.3", +] + [[package]] name = "system-deps" version = "6.2.2" @@ -6407,6 +6665,21 @@ dependencies = [ "utf-8", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + [[package]] name = "thiserror" version = "1.0.69" diff --git a/Cargo.toml b/Cargo.toml index b050f67..1704188 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,4 +32,8 @@ rustpotter = { git = "https://github.com/Priler/rustpotter" } image = "0.25" parking_lot = "0.12.5" toml = "0.9.8" -sha2 = "0.10" \ No newline at end of file +sha2 = "0.10" +nnnoiseless = "0.5" +sysinfo = "0.37.2" +tokio-tungstenite = "0.28.0" +futures-util = "0.3" \ No newline at end of file diff --git a/crates/jarvis-app/src/app.rs b/crates/jarvis-app/src/app.rs index 4821fe0..7ec0318 100644 --- a/crates/jarvis-app/src/app.rs +++ b/crates/jarvis-app/src/app.rs @@ -1,6 +1,6 @@ use std::time::SystemTime; -use jarvis_core::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST, intent}; +use jarvis_core::{audio, audio_processing, commands, config, listener, recorder, stt, COMMANDS_LIST, intent}; use rand::prelude::*; pub fn start() -> Result<(), ()> { @@ -14,6 +14,7 @@ fn main_loop() -> Result<(), ()> { let sounds_directory = audio::get_sound_directory().unwrap(); let frame_length: usize = 512; // default for every wake-word engine let mut frame_buffer: Vec = vec![0; frame_length]; + let mut silence_frames: u32 = 0; // play some run phrase // @TODO. Different sounds? Or better make it via commands or upcoming events system. @@ -33,16 +34,26 @@ fn main_loop() -> Result<(), ()> { // read from microphone recorder::read_microphone(&mut frame_buffer); + // process audio (gain -> noise suppression -> VAD) + let processed = audio_processing::process(&frame_buffer); + + // skip if no voice detected (vad) + if !processed.is_voice { + continue 'wake_word; + } + // recognize wake-word match listener::data_callback(&frame_buffer) { Some(_keyword_index) => { - // reset speech recognizer + // reset some things stt::reset_wake_recognizer(); stt::reset_speech_recognizer(); + audio_processing::reset(); // wake-word activated, process further commands // capture current time start = SystemTime::now(); + silence_frames = 0; // play some greet phrase // @TODO. Make it via commands or upcoming events system. @@ -58,6 +69,20 @@ fn main_loop() -> Result<(), ()> { // read from microphone recorder::read_microphone(&mut frame_buffer); + // process first + let processed = audio_processing::process(&frame_buffer); + + // detect silence, return to wake-word if silence + if processed.is_voice { + silence_frames = 0; + } else { + silence_frames += 1; + if silence_frames > config::VAD_SILENCE_FRAMES * 2 { + info!("Long silence detected, returning to wake word mode."); + break 'voice_recognition; + } + } + // stt part (without partials) if let Some(mut recognized_voice) = stt::recognize(&frame_buffer, false) { // something was recognized @@ -81,6 +106,7 @@ fn main_loop() -> Result<(), ()> { // reset timer and continue listening start = SystemTime::now(); + silence_frames = 0; stt::reset_speech_recognizer(); continue 'voice_recognition; } @@ -149,8 +175,9 @@ fn main_loop() -> Result<(), ()> { _ => (), } - // reset wake recognizer + // reset things stt::reset_wake_recognizer(); + audio_processing::reset(); } } None => (), diff --git a/crates/jarvis-app/src/main.rs b/crates/jarvis-app/src/main.rs index 0456757..0a9f814 100644 --- a/crates/jarvis-app/src/main.rs +++ b/crates/jarvis-app/src/main.rs @@ -3,7 +3,7 @@ use std::sync::Arc; // include core use jarvis_core::{ - audio, commands, config, db, listener, recorder, stt, intent, + audio, audio_processing, commands, config, db, listener, recorder, stt, intent, APP_CONFIG_DIR, APP_LOG_DIR, COMMANDS_LIST, DB, }; @@ -90,6 +90,12 @@ fn main() -> Result<(), String> { } }); + // init audio processing + info!("Initializing audio processing..."); + if let Err(e) = audio_processing::init() { + warn!("Audio processing init failed: {}", e); + } + // start the app (in the background thread) std::thread::spawn(|| { let _ = app::start(); diff --git a/crates/jarvis-core/Cargo.toml b/crates/jarvis-core/Cargo.toml index 4c05d63..6551a9f 100644 --- a/crates/jarvis-core/Cargo.toml +++ b/crates/jarvis-core/Cargo.toml @@ -23,6 +23,7 @@ rustpotter.workspace = true parking_lot.workspace = true toml.workspace = true sha2.workspace = true +nnnoiseless = { workspace = true, optional = true } # pv_recorder = { workspace = true, optional = true } vosk = { version = "0.3.1", optional = true } @@ -33,5 +34,5 @@ tokio = { version = "1", features = ["sync"], optional = true } [features] default = ["jarvis_app"] -jarvis_app = ["vosk", "intent-classifier", "tokio"] +jarvis_app = ["vosk", "intent-classifier", "tokio", "nnnoiseless"] intent = ["intent-classifier", "tokio"] \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing.rs b/crates/jarvis-core/src/audio_processing.rs new file mode 100644 index 0000000..df3ac47 --- /dev/null +++ b/crates/jarvis-core/src/audio_processing.rs @@ -0,0 +1,118 @@ +pub mod noise_suppression; +pub mod vad; +pub mod gain_normalizer; + +use once_cell::sync::OnceCell; +use std::sync::Mutex; + +use crate::config::structs::{NoiseSuppressionBackend, VadBackend}; +use crate::DB; + +static PROCESSOR: OnceCell> = OnceCell::new(); + +#[derive(Debug, Clone)] +pub struct ProcessedAudio { + pub samples: Vec, + pub is_voice: bool, + pub vad_confidence: f32, +} + +struct AudioProcessor { + ns_backend: NoiseSuppressionBackend, + vad_backend: VadBackend, + gain_enabled: bool, +} + +impl AudioProcessor { + fn new(ns: NoiseSuppressionBackend, vad: VadBackend, gain: bool) -> Self { + // init backends + noise_suppression::init(ns); + vad::init(vad); + if gain { + gain_normalizer::init(); + } + + Self { + ns_backend: ns, + vad_backend: vad, + gain_enabled: gain, + } + } + + fn process(&mut self, input: &[i16]) -> ProcessedAudio { + let mut samples = input.to_vec(); + + // step 1: gain normalization (before other processing) + if self.gain_enabled { + samples = gain_normalizer::normalize(&samples); + } + + // step 2: noise suppression + samples = noise_suppression::process(&samples); + + // step 3: VAD + let (is_voice, confidence) = vad::detect(&samples); + + ProcessedAudio { + samples, + is_voice, + vad_confidence: confidence, + } + } + + fn reset(&mut self) { + noise_suppression::reset(); + vad::reset(); + gain_normalizer::reset(); + } +} + + + +pub fn init() -> Result<(), String> { + if PROCESSOR.get().is_some() { + return Ok(()); + } + + let (ns, vad, gain) = get_settings(); + info!("Initializing audio processing: NS={:?}, VAD={:?}, Gain={}", ns, vad, gain); + + let processor = AudioProcessor::new(ns, vad, gain); + PROCESSOR + .set(Mutex::new(processor)) + .map_err(|_| "Audio processor already initialized")?; + + info!("Audio processing initialized."); + Ok(()) +} + +pub fn process(input: &[i16]) -> ProcessedAudio { + match PROCESSOR.get() { + Some(p) => p.lock().unwrap().process(input), + None => ProcessedAudio { + samples: input.to_vec(), + is_voice: true, + vad_confidence: 1.0, + }, + } +} + +pub fn reset() { + if let Some(p) = PROCESSOR.get() { + p.lock().unwrap().reset(); + } +} + +fn get_settings() -> (NoiseSuppressionBackend, VadBackend, bool) { + match DB.get() { + Some(db) => { + let settings = db.read(); + (settings.noise_suppression, settings.vad, settings.gain_normalizer) + } + None => ( + crate::config::DEFAULT_NOISE_SUPPRESSION, + crate::config::DEFAULT_VAD, + crate::config::DEFAULT_GAIN_NORMALIZER, + ), + } +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/gain_normalizer.rs b/crates/jarvis-core/src/audio_processing/gain_normalizer.rs new file mode 100644 index 0000000..b637d27 --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/gain_normalizer.rs @@ -0,0 +1,28 @@ +mod simple; + +use once_cell::sync::OnceCell; +use std::sync::Mutex; + +static NORMALIZER: OnceCell> = OnceCell::new(); + +pub fn init() { + if NORMALIZER.get().is_some() { + return; + } + + NORMALIZER.set(Mutex::new(simple::GainNormalizer::new())).ok(); + info!("Gain normalizer: enabled"); +} + +pub fn normalize(input: &[i16]) -> Vec { + match NORMALIZER.get() { + Some(n) => n.lock().unwrap().normalize(input), + None => input.to_vec(), + } +} + +pub fn reset() { + if let Some(n) = NORMALIZER.get() { + n.lock().unwrap().reset(); + } +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/gain_normalizer/simple.rs b/crates/jarvis-core/src/audio_processing/gain_normalizer/simple.rs new file mode 100644 index 0000000..b58da1f --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/gain_normalizer/simple.rs @@ -0,0 +1,47 @@ +use crate::config; + +pub struct GainNormalizer { + current_gain: f32, +} + +impl GainNormalizer { + pub fn new() -> Self { + Self { current_gain: 1.0 } + } + + pub fn normalize(&mut self, input: &[i16]) -> Vec { + let rms = self.calculate_rms(input); + + if rms < 1.0 { + return input.to_vec(); + } + + let target_gain = config::GAIN_TARGET_RMS / rms; + let clamped_gain = target_gain.clamp(config::GAIN_MIN, config::GAIN_MAX); + + self.current_gain = self.current_gain * 0.9 + clamped_gain * 0.1; + + input.iter() + .map(|&s| { + let amplified = (s as f32) * self.current_gain; + amplified.clamp(i16::MIN as f32, i16::MAX as f32) as i16 + }) + .collect() + } + + pub fn reset(&mut self) { + self.current_gain = 1.0; + } + + fn calculate_rms(&self, samples: &[i16]) -> f32 { + if samples.is_empty() { + return 0.0; + } + + let sum: f64 = samples.iter() + .map(|&s| (s as f64).powi(2)) + .sum(); + + (sum / samples.len() as f64).sqrt() as f32 + } +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/noise_suppression.rs b/crates/jarvis-core/src/audio_processing/noise_suppression.rs new file mode 100644 index 0000000..e683bf1 --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/noise_suppression.rs @@ -0,0 +1,66 @@ +mod none; + +#[cfg(feature = "nnnoiseless")] +mod nnnoiseless; + +use once_cell::sync::OnceCell; +use std::sync::Mutex; + +use crate::config::structs::NoiseSuppressionBackend; + +static BACKEND: OnceCell = OnceCell::new(); + +#[cfg(feature = "nnnoiseless")] +static NNNOISELESS_STATE: OnceCell> = OnceCell::new(); + +pub fn init(backend: NoiseSuppressionBackend) { + if BACKEND.get().is_some() { + return; + } + + BACKEND.set(backend).ok(); + + match backend { + NoiseSuppressionBackend::None => { + info!("Noise suppression: disabled"); + } + #[cfg(feature = "nnnoiseless")] + NoiseSuppressionBackend::Nnnoiseless => { + NNNOISELESS_STATE.set(Mutex::new(nnnoiseless::NnnoiselessNS::new())).ok(); + info!("Noise suppression: Nnnoiseless"); + } + #[cfg(not(feature = "nnnoiseless"))] + NoiseSuppressionBackend::Nnnoiseless => { + warn!("Nnnoiseless not compiled in, falling back to None"); + BACKEND.set(NoiseSuppressionBackend::None).ok(); + } + } +} + +pub fn process(input: &[i16]) -> Vec { + match BACKEND.get() { + Some(NoiseSuppressionBackend::None) | None => none::process(input), + #[cfg(feature = "nnnoiseless")] + Some(NoiseSuppressionBackend::Nnnoiseless) => { + if let Some(state) = NNNOISELESS_STATE.get() { + state.lock().unwrap().process(input) + } else { + none::process(input) + } + } + #[cfg(not(feature = "nnnoiseless"))] + Some(NoiseSuppressionBackend::Nnnoiseless) => none::process(input), + } +} + +pub fn reset() { + match BACKEND.get() { + #[cfg(feature = "nnnoiseless")] + Some(NoiseSuppressionBackend::Nnnoiseless) => { + if let Some(state) = NNNOISELESS_STATE.get() { + state.lock().unwrap().reset(); + } + } + _ => {} + } +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/noise_suppression/nnnoiseless.rs b/crates/jarvis-core/src/audio_processing/noise_suppression/nnnoiseless.rs new file mode 100644 index 0000000..6ec4c27 --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/noise_suppression/nnnoiseless.rs @@ -0,0 +1,51 @@ +use nnnoiseless::DenoiseState; +use crate::config; + +pub struct NnnoiselessNS { + state: Box>, + buffer: Vec, +} + +impl NnnoiselessNS { + pub fn new() -> Self { + Self { + state: DenoiseState::new(), + buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2), + } + } + + pub fn process(&mut self, input: &[i16]) -> Vec { + for &sample in input { + self.buffer.push(sample as f32); + } + + let mut output: Vec = Vec::with_capacity(input.len()); + + while self.buffer.len() >= config::NNNOISELESS_FRAME_SIZE { + let mut input_frame = [0.0f32; 480]; + let mut output_frame = [0.0f32; 480]; + + input_frame.copy_from_slice(&self.buffer[..config::NNNOISELESS_FRAME_SIZE]); + self.buffer.drain(..config::NNNOISELESS_FRAME_SIZE); + + // process: input -> output (denoised) + let _ = self.state.process_frame(&mut output_frame, &input_frame); + + for &sample in &output_frame { + let clamped = sample.clamp(i16::MIN as f32, i16::MAX as f32); + output.push(clamped as i16); + } + } + + if output.is_empty() { + return input.to_vec(); + } + + output + } + + pub fn reset(&mut self) { + self.state = DenoiseState::new(); + self.buffer.clear(); + } +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/noise_suppression/none.rs b/crates/jarvis-core/src/audio_processing/noise_suppression/none.rs new file mode 100644 index 0000000..87174f9 --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/noise_suppression/none.rs @@ -0,0 +1,4 @@ +// return unprocessed input +pub fn process(input: &[i16]) -> Vec { + input.to_vec() +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/vad.rs b/crates/jarvis-core/src/audio_processing/vad.rs new file mode 100644 index 0000000..f6b624a --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/vad.rs @@ -0,0 +1,72 @@ +mod none; +mod energy; + +#[cfg(feature = "nnnoiseless")] +mod nnnoiseless; + +use once_cell::sync::OnceCell; +use std::sync::Mutex; + +use crate::config::structs::VadBackend; + +static BACKEND: OnceCell = OnceCell::new(); + +#[cfg(feature = "nnnoiseless")] +static NNNOISELESS_STATE: OnceCell> = OnceCell::new(); + +pub fn init(backend: VadBackend) { + if BACKEND.get().is_some() { + return; + } + + BACKEND.set(backend).ok(); + + match backend { + VadBackend::None => { + info!("VAD: disabled"); + } + VadBackend::Energy => { + info!("VAD: Energy-based"); + } + #[cfg(feature = "nnnoiseless")] + VadBackend::Nnnoiseless => { + NNNOISELESS_STATE.set(Mutex::new(nnnoiseless::NnnoiselessVAD::new())).ok(); + info!("VAD: Nnnoiseless"); + } + #[cfg(not(feature = "nnnoiseless"))] + VadBackend::Nnnoiseless => { + warn!("Nnnoiseless not compiled in, falling back to Energy"); + BACKEND.set(VadBackend::Energy).ok(); + } + } +} + +// Returns (is_voice, confidence) +pub fn detect(input: &[i16]) -> (bool, f32) { + match BACKEND.get() { + Some(VadBackend::None) | None => none::detect(input), + Some(VadBackend::Energy) => energy::detect(input), + #[cfg(feature = "nnnoiseless")] + Some(VadBackend::Nnnoiseless) => { + if let Some(state) = NNNOISELESS_STATE.get() { + state.lock().unwrap().detect(input) + } else { + energy::detect(input) + } + } + #[cfg(not(feature = "nnnoiseless"))] + Some(VadBackend::Nnnoiseless) => energy::detect(input), + } +} + +pub fn reset() { + match BACKEND.get() { + #[cfg(feature = "nnnoiseless")] + Some(VadBackend::Nnnoiseless) => { + if let Some(state) = NNNOISELESS_STATE.get() { + state.lock().unwrap().reset(); + } + } + _ => {} + } +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/vad/energy.rs b/crates/jarvis-core/src/audio_processing/vad/energy.rs new file mode 100644 index 0000000..9fd1011 --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/vad/energy.rs @@ -0,0 +1,24 @@ +use crate::config; + +// Simple energy-based VAD +pub fn detect(input: &[i16]) -> (bool, f32) { + let rms = calculate_rms(input); + let is_voice = rms > config::VAD_ENERGY_THRESHOLD; + + // normalize confidence to 0-1 range (rough approximation) + let confidence = (rms / (config::VAD_ENERGY_THRESHOLD * 2.0)).min(1.0); + + (is_voice, confidence) +} + +fn calculate_rms(samples: &[i16]) -> f32 { + if samples.is_empty() { + return 0.0; + } + + let sum: f64 = samples.iter() + .map(|&s| (s as f64).powi(2)) + .sum(); + + (sum / samples.len() as f64).sqrt() as f32 +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/vad/nnnoiseless.rs b/crates/jarvis-core/src/audio_processing/vad/nnnoiseless.rs new file mode 100644 index 0000000..913ff2a --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/vad/nnnoiseless.rs @@ -0,0 +1,51 @@ +use nnnoiseless::DenoiseState; +use crate::config; + +pub struct NnnoiselessVAD { + state: Box>, + buffer: Vec, +} + +impl NnnoiselessVAD { + pub fn new() -> Self { + Self { + state: DenoiseState::new(), + buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2), + } + } + + pub fn detect(&mut self, input: &[i16]) -> (bool, f32) { + for &sample in input { + self.buffer.push(sample as f32); + } + + let mut total_vad = 0.0f32; + let mut frame_count = 0u32; + + while self.buffer.len() >= config::NNNOISELESS_FRAME_SIZE { + let mut input_frame = [0.0f32; 480]; + let mut output_frame = [0.0f32; 480]; + + input_frame.copy_from_slice(&self.buffer[..config::NNNOISELESS_FRAME_SIZE]); + self.buffer.drain(..config::NNNOISELESS_FRAME_SIZE); + + let vad_prob = self.state.process_frame(&mut output_frame, &input_frame); + total_vad += vad_prob; + frame_count += 1; + } + + if frame_count == 0 { + return (true, 0.5); + } + + let avg_vad = total_vad / frame_count as f32; + let is_voice = avg_vad >= config::VAD_NNNOISELESS_THRESHOLD; + + (is_voice, avg_vad) + } + + pub fn reset(&mut self) { + self.state = DenoiseState::new(); + self.buffer.clear(); + } +} \ No newline at end of file diff --git a/crates/jarvis-core/src/audio_processing/vad/none.rs b/crates/jarvis-core/src/audio_processing/vad/none.rs new file mode 100644 index 0000000..f4567c5 --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/vad/none.rs @@ -0,0 +1,4 @@ +// Always returns voice detected (no vad) +pub fn detect(_input: &[i16]) -> (bool, f32) { + (true, 1.0) +} \ No newline at end of file diff --git a/crates/jarvis-core/src/config.rs b/crates/jarvis-core/src/config.rs index 9cef61e..566b2d8 100644 --- a/crates/jarvis-core/src/config.rs +++ b/crates/jarvis-core/src/config.rs @@ -18,6 +18,8 @@ use rustpotter::{ }; use crate::IntentRecognitionEngine; +use crate::config::structs::NoiseSuppressionBackend; +use crate::config::structs::VadBackend; use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR}; #[allow(dead_code)] @@ -106,7 +108,11 @@ pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy = Lazy::new(|| { }, filters: FiltersConfig { gain_normalizer: GainNormalizationConfig { - enabled: true, + // enabled: true, + // gain_ref: None, + // min_gain: 0.7, + // max_gain: 1.0, + enabled: false, // disable, now we have separate gain normalizer implementation gain_ref: None, min_gain: 0.7, max_gain: 1.0, @@ -146,6 +152,26 @@ pub const VOSK_SPEECH_PARTIAL_WORDS: bool = false; // IRE (intents recognition) pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.75; + +// AUDIO PROCESSING DEFAULTS +pub const DEFAULT_NOISE_SUPPRESSION: NoiseSuppressionBackend = NoiseSuppressionBackend::None; +pub const DEFAULT_VAD: VadBackend = VadBackend::Energy; +pub const DEFAULT_GAIN_NORMALIZER: bool = false; + +// VAD settings +pub const VAD_ENERGY_THRESHOLD: f32 = 500.0; // RMS threshold for energy-based VAD +pub const VAD_NNNOISELESS_THRESHOLD: f32 = 0.5; // probability threshold for nnnoiseless +pub const VAD_SILENCE_FRAMES: u32 = 15; // frames of silence before speech end (~480ms) + +// gain normalizer settings +pub const GAIN_TARGET_RMS: f32 = 3000.0; // target RMS level +pub const GAIN_MIN: f32 = 0.5; // minimum gain multiplier +pub const GAIN_MAX: f32 = 3.0; // maximum gain multiplier + +// nnnoiseless frame size (fixed by library) +pub const NNNOISELESS_FRAME_SIZE: usize = 480; + + // ETC pub const CMD_RATIO_THRESHOLD: f64 = 65f64; pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15); diff --git a/crates/jarvis-core/src/config/structs.rs b/crates/jarvis-core/src/config/structs.rs index e83a032..118c7c0 100644 --- a/crates/jarvis-core/src/config/structs.rs +++ b/crates/jarvis-core/src/config/structs.rs @@ -1,23 +1,30 @@ use std::fmt; use serde::{Deserialize, Serialize}; -#[derive(Clone, Copy, Serialize, Deserialize, Debug)] +#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)] pub enum WakeWordEngine { Rustpotter, Vosk, Porcupine, } -#[derive(Clone, Copy, Serialize, Deserialize, Debug)] +#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)] pub enum IntentRecognitionEngine { IntentClassifier, Rasa, } -impl fmt::Display for WakeWordEngine { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self) - } +#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)] +pub enum NoiseSuppressionBackend { + None, + Nnnoiseless, +} + +#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)] +pub enum VadBackend { + None, + Energy, + Nnnoiseless, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -25,12 +32,6 @@ pub enum SpeechToTextEngine { Vosk, } -impl fmt::Display for SpeechToTextEngine { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self) - } -} - #[derive(PartialEq, Debug)] pub enum RecorderType { Cpal, @@ -44,6 +45,38 @@ pub enum AudioType { Kira, } + + +impl fmt::Display for WakeWordEngine { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl fmt::Display for SpeechToTextEngine { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl fmt::Display for IntentRecognitionEngine { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl fmt::Display for NoiseSuppressionBackend { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl fmt::Display for VadBackend { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self) + } +} + // pub enum TextToSpeechEngine {} // pub enum IntentRecognitionEngine {} diff --git a/crates/jarvis-core/src/db/structs.rs b/crates/jarvis-core/src/db/structs.rs index 74b980a..58901e4 100644 --- a/crates/jarvis-core/src/db/structs.rs +++ b/crates/jarvis-core/src/db/structs.rs @@ -4,6 +4,8 @@ use serde::{Deserialize, Serialize}; use crate::config::structs::SpeechToTextEngine; use crate::config::structs::WakeWordEngine; use crate::config::structs::IntentRecognitionEngine; +use crate::config::structs::NoiseSuppressionBackend; +use crate::config::structs::VadBackend; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Settings { @@ -13,9 +15,13 @@ pub struct Settings { pub wake_word_engine: WakeWordEngine, pub intent_recognition_engine: IntentRecognitionEngine, pub speech_to_text_engine: SpeechToTextEngine, - pub vosk_model: String, + // audio processing + pub noise_suppression: NoiseSuppressionBackend, + pub vad: VadBackend, + pub gain_normalizer: bool, + pub api_keys: ApiKeys, } @@ -28,9 +34,13 @@ impl Default for Settings { wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE, intent_recognition_engine: config::DEFAULT_INTENT_RECOGNITION_ENGINE, speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE, - vosk_model: String::from(""), // auto detect first available + // audio processing defaults + noise_suppression: config::DEFAULT_NOISE_SUPPRESSION, + vad: config::DEFAULT_VAD, + gain_normalizer: config::DEFAULT_GAIN_NORMALIZER, + api_keys: ApiKeys { picovoice: String::from(""), openai: String::from(""), diff --git a/crates/jarvis-core/src/lib.rs b/crates/jarvis-core/src/lib.rs index bc76285..b61eb08 100644 --- a/crates/jarvis-core/src/lib.rs +++ b/crates/jarvis-core/src/lib.rs @@ -25,6 +25,9 @@ pub mod intent; pub mod vosk_models; +#[cfg(feature = "jarvis_app")] +pub mod audio_processing; + // shared statics // pub static APP_DIR: Lazy = Lazy::new(|| std::env::current_dir().unwrap()); pub static APP_DIR: Lazy = Lazy::new(|| { diff --git a/crates/jarvis-core/src/recorder/pvrecorder.rs b/crates/jarvis-core/src/recorder/pvrecorder.rs index 669ce39..38f0ee3 100644 --- a/crates/jarvis-core/src/recorder/pvrecorder.rs +++ b/crates/jarvis-core/src/recorder/pvrecorder.rs @@ -114,6 +114,10 @@ pub fn list_audio_devices() -> Vec { } pub fn get_audio_device_name(idx: i32) -> String { + if idx == -1 { + return String::from("System Default"); + } + let audio_devices = list_audio_devices(); let mut first_device: String = String::new(); diff --git a/crates/jarvis-gui/Cargo.toml b/crates/jarvis-gui/Cargo.toml index 867b035..9e841da 100644 --- a/crates/jarvis-gui/Cargo.toml +++ b/crates/jarvis-gui/Cargo.toml @@ -16,6 +16,7 @@ tauri-plugin-fs = "2" peak_alloc = "0.3.0" systemstat = "0.2" lazy_static = "1.4" +sysinfo.workspace = true once_cell.workspace = true log.workspace = true diff --git a/crates/jarvis-gui/src/main.rs b/crates/jarvis-gui/src/main.rs index a4d4df4..9110c4c 100644 --- a/crates/jarvis-gui/src/main.rs +++ b/crates/jarvis-gui/src/main.rs @@ -61,6 +61,9 @@ fn main() { tauri_commands::get_peak_ram_usage, tauri_commands::get_cpu_temp, tauri_commands::get_cpu_usage, + tauri_commands::get_jarvis_app_stats, + tauri_commands::is_jarvis_app_running, + tauri_commands::run_jarvis_app, // vosk tauri_commands::list_vosk_models, diff --git a/crates/jarvis-gui/src/tauri_commands/db.rs b/crates/jarvis-gui/src/tauri_commands/db.rs index 80aa9f0..73fddb8 100644 --- a/crates/jarvis-gui/src/tauri_commands/db.rs +++ b/crates/jarvis-gui/src/tauri_commands/db.rs @@ -12,6 +12,9 @@ pub fn db_read(state: tauri::State<'_, AppState>, key: &str) -> String { "selected_intent_recognition_engine" => format!("{:?}", settings.intent_recognition_engine), "selected_vosk_model" => settings.vosk_model.clone(), "speech_to_text_engine" => format!("{:?}", settings.speech_to_text_engine), + "noise_suppression" => format!("{:?}", settings.noise_suppression), + "vad" => format!("{:?}", settings.vad), + "gain_normalizer" => settings.gain_normalizer.to_string(), "api_key__picovoice" => settings.api_keys.picovoice.clone(), "api_key__openai" => settings.api_keys.openai.clone(), _ => String::new(), @@ -53,6 +56,28 @@ pub fn db_write(state: tauri::State<'_, AppState>, key: &str, val: &str) -> bool "selected_vosk_model" => { settings.vosk_model = val.to_string(); } + "noise_suppression" => { + match val.to_lowercase().as_str() { + "none" => settings.noise_suppression = jarvis_core::config::structs::NoiseSuppressionBackend::None, + "nnnoiseless" => settings.noise_suppression = jarvis_core::config::structs::NoiseSuppressionBackend::Nnnoiseless, + _ => return false, + } + } + "vad" => { + match val.to_lowercase().as_str() { + "none" => settings.vad = jarvis_core::config::structs::VadBackend::None, + "energy" => settings.vad = jarvis_core::config::structs::VadBackend::Energy, + "nnnoiseless" => settings.vad = jarvis_core::config::structs::VadBackend::Nnnoiseless, + _ => return false, + } + } + "gain_normalizer" => { + match val.to_lowercase().as_str() { + "true" => settings.gain_normalizer = true, + "false" => settings.gain_normalizer = false, + _ => return false, + } + } "api_key__picovoice" => { settings.api_keys.picovoice = val.to_string(); } diff --git a/crates/jarvis-gui/src/tauri_commands/sys.rs b/crates/jarvis-gui/src/tauri_commands/sys.rs index 4ef19a2..9bc0b45 100644 --- a/crates/jarvis-gui/src/tauri_commands/sys.rs +++ b/crates/jarvis-gui/src/tauri_commands/sys.rs @@ -1,49 +1,152 @@ +use sysinfo::{System, Pid, ProcessRefreshKind, RefreshKind, CpuRefreshKind, Components}; use peak_alloc::PeakAlloc; +use std::sync::Mutex; +use once_cell::sync::Lazy; +use std::process::Command; +use std::env; #[global_allocator] static PEAK_ALLOC: PeakAlloc = PeakAlloc; -extern crate systemstat; -use std::thread; -use std::time::Duration; -use systemstat::{Platform, System}; -use lazy_static::lazy_static; +static SYS: Lazy> = Lazy::new(|| { + Mutex::new(System::new_with_specifics( + RefreshKind::nothing() + .with_processes(ProcessRefreshKind::nothing().with_memory().with_cpu()) + .with_cpu(CpuRefreshKind::everything()) + )) +}); -lazy_static! { - static ref SYS: System = System::new(); +static COMPONENTS: Lazy> = Lazy::new(|| { + Mutex::new(Components::new_with_refreshed_list()) +}); + +const JARVIS_APP_NAME: &str = "jarvis-app"; + +/// Find jarvis-app process and return its PID +fn find_jarvis_app_pid(sys: &System) -> Option { + for (pid, process) in sys.processes() { + let name = process.name().to_string_lossy().to_lowercase(); + if name.contains(JARVIS_APP_NAME) { + return Some(*pid); + } + } + None +} + +#[derive(serde::Serialize)] +pub struct JarvisAppStats { + pub running: bool, + pub ram_mb: u64, + pub cpu_usage: f32, } #[tauri::command] -pub fn get_current_ram_usage() -> String { - let result = String::from(format!("{}", PEAK_ALLOC.current_usage_as_mb())); - - result +pub fn get_jarvis_app_stats() -> JarvisAppStats { + let mut sys = SYS.lock().unwrap(); + + // refresh all processes to find jarvis-app + sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true); + + if let Some(pid) = find_jarvis_app_pid(&sys) { + if let Some(proc) = sys.process(pid) { + return JarvisAppStats { + running: true, + ram_mb: proc.memory() / 1024 / 1024, + cpu_usage: proc.cpu_usage(), + }; + } + } + + JarvisAppStats { + running: false, + ram_mb: 0, + cpu_usage: 0.0, + } } #[tauri::command] -pub fn get_peak_ram_usage() -> String { - let result = String::from(format!("{}", PEAK_ALLOC.peak_usage_as_gb())); +pub fn get_current_ram_usage() -> u64 { + let mut sys = SYS.lock().unwrap(); + sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true); + + if let Some(pid) = find_jarvis_app_pid(&sys) { + if let Some(proc) = sys.process(pid) { + return proc.memory() / 1024 / 1024; + } + } + + 0 +} - result +#[tauri::command] +pub fn is_jarvis_app_running() -> bool { + let mut sys = SYS.lock().unwrap(); + sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true); + find_jarvis_app_pid(&sys).is_some() } #[tauri::command] pub fn get_cpu_temp() -> String { - if let Ok(cpu_temp) = SYS.cpu_temp() { - String::from(format!("{}", cpu_temp)) - } else { - String::from("error") + let mut components = COMPONENTS.lock().unwrap(); + components.refresh(true); + + for component in components.iter() { + let label = component.label().to_lowercase(); + if label.contains("cpu") || label.contains("core") || label.contains("package") { + if let Some(temp) = component.temperature() { + return format!("{:.1}", temp); + } + } } + + if let Some(component) = components.iter().next() { + if let Some(temp) = component.temperature() { + return format!("{:.1}", temp); + } + } + + String::from("N/A") } -// https://github.com/valpackett/systemstat/blob/trunk/examples/info.rs -#[tauri::command(async)] -pub async fn get_cpu_usage() -> String { - if let Ok(cpu) = SYS.cpu_load_aggregate() { - thread::sleep(Duration::from_secs(1)); - let cpu = cpu.done().unwrap(); - String::from(format!("{}", cpu.user * 100.0)) - } else { - String::from("error") - } +#[tauri::command] +pub fn get_cpu_usage() -> f32 { + let mut sys = SYS.lock().unwrap(); + + sys.refresh_cpu_all(); + std::thread::sleep(std::time::Duration::from_millis(200)); + sys.refresh_cpu_all(); + + sys.global_cpu_usage() } + +#[tauri::command] +pub fn get_peak_ram_usage() -> String { + format!("{}", PEAK_ALLOC.peak_usage_as_gb()) +} + +#[tauri::command] +pub fn run_jarvis_app() -> Result<(), String> { + let exe_dir = std::env::current_exe() + .map_err(|e| format!("Failed to get exe path: {}", e))? + .parent() + .ok_or("Failed to get exe directory")? + .to_path_buf(); + + #[cfg(target_os = "windows")] + let jarvis_app_name = "jarvis-app.exe"; + + #[cfg(not(target_os = "windows"))] + let jarvis_app_name = "jarvis-app"; + + let jarvis_app_path = exe_dir.join(jarvis_app_name); + + if !jarvis_app_path.exists() { + return Err(format!("jarvis-app not found at: {}", jarvis_app_path.display())); + } + + std::process::Command::new(&jarvis_app_path) + .spawn() + .map_err(|e| format!("Failed to start jarvis-app: {}", e))?; + + Ok(()) +} \ No newline at end of file diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index e4f7866..f99b1cd 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -1,8 +1,26 @@ diff --git a/frontend/src/components/elements/Stats.svelte b/frontend/src/components/elements/Stats.svelte index daf6535..4546b1c 100644 --- a/frontend/src/components/elements/Stats.svelte +++ b/frontend/src/components/elements/Stats.svelte @@ -3,17 +3,24 @@ import { invoke } from "@tauri-apps/api/core" import { capitalizeFirstLetter } from "@/functions" + import { + Text, + } from "@svelteuidev/core" + + let jarvisStats = { running: false, ram_mb: 0, cpu_usage: 0 } + let microphoneLabel = "" let wakeWordEngine = "" let sttEngine = "Vosk" - let ramUsage = "-" + // let ramUsage = "-" - let interval: number | null = null + let statsUpdateInterval: number | null = null - async function updateRamUsage() { + async function updateStats() { try { - const usage = await invoke("get_current_ram_usage") - ramUsage = usage.toFixed(2) + jarvisStats = await invoke<{running: boolean, ram_mb: number, cpu_usage: number}>("get_jarvis_app_stats") + //const usage = await invoke("get_current_ram_usage") + //ramUsage = usage.toFixed(2) } catch (err) { console.error("failed to get ram usage:", err) } @@ -21,7 +28,8 @@ onMount(async () => { // start polling ram usage - interval = setInterval(updateRamUsage, 1000) as unknown as number + updateStats() + statsUpdateInterval = setInterval(updateStats, 5000) as unknown as number try { // load microphone info @@ -37,8 +45,8 @@ }) onDestroy(() => { - if (interval) { - clearInterval(interval) + if (statsUpdateInterval) { + clearInterval(statsUpdateInterval) } }) @@ -64,7 +72,12 @@
Ресурсы - RAM {ramUsage}mb + {#if jarvisStats.running} + RAM: {jarvisStats.ram_mb} MB + + {:else} + - + {/if}
diff --git a/frontend/src/functions.ts b/frontend/src/functions.ts index b970ec9..707c204 100644 --- a/frontend/src/functions.ts +++ b/frontend/src/functions.ts @@ -11,17 +11,3 @@ export function showInExplorer(path: string): void { invoke("show_in_folder", { path }) .catch(err => console.error("failed to open explorer:", err)) } - -// ### LISTENER FUNCTIONS -// removed since gui now doesn't handle listening - -export function startListening(): void { - // disabled in GUI - listening is handled by the tray app - console.log("[gui] listening not available in settings app") -} - -export function stopListening(callback?: () => void): void { - // disabled in GUI - just call the callback if provided - console.log("[gui] listening not available in settings app") - if (callback) callback() -} diff --git a/frontend/src/routes/index.svelte b/frontend/src/routes/index.svelte index 2f8d9d2..9801eb5 100644 --- a/frontend/src/routes/index.svelte +++ b/frontend/src/routes/index.svelte @@ -1,6 +1,7 @@ -{#if !listening} +{#if !running} - В данный момент ассистент не прослушивает команды.
- Пожалуйста, перейдите в настройки и введите ключ Picovoice. + В данный момент ассистент не запущен.
+ Но вы всё еще можете изменять его настройки.
+
+ + {:else} @@ -43,4 +75,4 @@ -