GitHub/jarvis
1
0
Fork 0
mirror of https://github.com/Priler/jarvis.git synced 2026-05-26 07:08:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

noise suppression added via nnnoiseless + vad + gain-normalizer + few frontend changes

Browse Source
This commit is contained in:
Priler
2026-01-06 23:32:58 +05:00
parent a640e6caea
commit eb0d40bae6
73 changed files with 1235 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

279
Cargo.lock generated
View File

@@ -193,6 +193,12 @@ version = "1.0.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
[[package]]
name = "anymap3"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "170433209e817da6aae2c51aa0dd443009a613425dd041ebfb2492d1c4c11a25"
[[package]]
name = "arbitrary"
version = "1.4.2"
@@ -222,6 +228,12 @@ dependencies = [
"syn 2.0.113",
]
[[package]]
name = "array-init"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
[[package]]
name = "arrayref"
version = "0.3.9"
@@ -339,6 +351,17 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi 0.1.19",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -769,6 +792,31 @@ dependencies = [
"half",
]
[[package]]
name = "clap"
version = "3.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
dependencies = [
"atty",
"bitflags 1.3.2",
"clap_lex",
"indexmap 1.9.3",
"once_cell",
"strsim 0.10.0",
"termcolor",
"textwrap",
]
[[package]]
name = "clap_lex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
dependencies = [
"os_str_bytes",
]
[[package]]
name = "color_quant"
version = "1.1.0"
@@ -1070,7 +1118,7 @@ dependencies = [
"ident_case",
"proc-macro2",
"quote",
"strsim",
"strsim 0.11.1",
"syn 2.0.113",
]
@@ -1098,12 +1146,125 @@ dependencies = [
"parking_lot_core",
]
[[package]]
name = "dasp"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7381b67da416b639690ac77c73b86a7b5e64a29e31d1f75fb3b1102301ef355a"
dependencies = [
"dasp_envelope",
"dasp_frame",
"dasp_interpolate",
"dasp_peak",
"dasp_ring_buffer",
"dasp_rms",
"dasp_sample",
"dasp_signal",
"dasp_slice",
"dasp_window",
]
[[package]]
name = "dasp_envelope"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ec617ce7016f101a87fe85ed44180839744265fae73bb4aa43e7ece1b7668b6"
dependencies = [
"dasp_frame",
"dasp_peak",
"dasp_ring_buffer",
"dasp_rms",
"dasp_sample",
]
[[package]]
name = "dasp_frame"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2a3937f5fe2135702897535c8d4a5553f8b116f76c1529088797f2eee7c5cd6"
dependencies = [
"dasp_sample",
]
[[package]]
name = "dasp_interpolate"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fc975a6563bb7ca7ec0a6c784ead49983a21c24835b0bc96eea11ee407c7486"
dependencies = [
"dasp_frame",
"dasp_ring_buffer",
"dasp_sample",
]
[[package]]
name = "dasp_peak"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cf88559d79c21f3d8523d91250c397f9a15b5fc72fbb3f87fdb0a37b79915bf"
dependencies = [
"dasp_frame",
"dasp_sample",
]
[[package]]
name = "dasp_ring_buffer"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07d79e19b89618a543c4adec9c5a347fe378a19041699b3278e616e387511ea1"
[[package]]
name = "dasp_rms"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6c5dcb30b7e5014486e2822537ea2beae50b19722ffe2ed7549ab03774575aa"
dependencies = [
"dasp_frame",
"dasp_ring_buffer",
"dasp_sample",
]
[[package]]
name = "dasp_sample"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
[[package]]
name = "dasp_signal"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa1ab7d01689c6ed4eae3d38fe1cea08cba761573fbd2d592528d55b421077e7"
dependencies = [
"dasp_envelope",
"dasp_frame",
"dasp_interpolate",
"dasp_peak",
"dasp_ring_buffer",
"dasp_rms",
"dasp_sample",
"dasp_window",
]
[[package]]
name = "dasp_slice"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e1c7335d58e7baedafa516cb361360ff38d6f4d3f9d9d5ee2a2fc8e27178fa1"
dependencies = [
"dasp_frame",
"dasp_sample",
]
[[package]]
name = "dasp_window"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99ded7b88821d2ce4e8b842c9f1c86ac911891ab89443cc1de750cae764c5076"
dependencies = [
"dasp_sample",
]
[[package]]
name = "deranged"
version = "0.5.5"
@@ -1341,6 +1502,19 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9"
[[package]]
name = "easyfft"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "767e39eef2ad8a3b6f1d733be3ec70364d21d437d06d4f18ea76ce08df20b75f"
dependencies = [
"array-init",
"generic_singleton",
"num-complex",
"realfft",
"rustfft",
]
[[package]]
name = "either"
version = "1.15.0"
@@ -2080,6 +2254,17 @@ dependencies = [
"version_check",
]
[[package]]
name = "generic_singleton"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2d5de0fc83987dac514f3b910c5d08392b220efe8cf72086c660029a197bf73"
dependencies = [
"anymap3",
"lazy_static",
"parking_lot",
]
[[package]]
name = "gethostname"
version = "1.1.0"
@@ -2404,6 +2589,15 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.5.2"
@@ -2863,6 +3057,7 @@ dependencies = [
"intent-classifier",
"kira",
"log",
"nnnoiseless",
"once_cell",
"parking_lot",
"platform-dirs",
@@ -2894,6 +3089,7 @@ dependencies = [
"serde",
"serde_json",
"simple-log",
"sysinfo",
"systemstat",
"tauri",
"tauri-build",
@@ -3448,6 +3644,22 @@ dependencies = [
"memoffset",
]
[[package]]
name = "nnnoiseless"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "805d5964d1e7a0006a7fdced7dae75084d66d18b35f1dfe81bd76929b1f8da0c"
dependencies = [
"anyhow",
"clap",
"dasp",
"dasp_interpolate",
"dasp_ring_buffer",
"easyfft",
"hound",
"once_cell",
]
[[package]]
name = "nodrop"
version = "0.1.14"
@@ -3479,6 +3691,15 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
[[package]]
name = "ntapi"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081"
dependencies = [
"winapi",
]
[[package]]
name = "num"
version = "0.4.3"
@@ -3511,6 +3732,7 @@ checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"bytemuck",
"num-traits",
"serde",
]
[[package]]
@@ -3577,7 +3799,7 @@ version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
dependencies = [
"hermit-abi",
"hermit-abi 0.5.2",
"libc",
]
@@ -3885,6 +4107,16 @@ dependencies = [
"objc2-core-foundation",
]
[[package]]
name = "objc2-io-kit"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15"
dependencies = [
"libc",
"objc2-core-foundation",
]
[[package]]
name = "objc2-io-surface"
version = "0.3.2"
@@ -4118,6 +4350,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "os_str_bytes"
version = "6.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
[[package]]
name = "owned_ttf_parser"
version = "0.25.1"
@@ -4439,7 +4677,7 @@ checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218"
dependencies = [
"cfg-if",
"concurrent-queue",
"hermit-abi",
"hermit-abi 0.5.2",
"pin-project-lite",
"rustix 1.1.3",
"windows-sys 0.61.2",
@@ -5753,6 +5991,12 @@ dependencies = [
"quote",
]
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "strsim"
version = "0.11.1"
@@ -5987,6 +6231,20 @@ dependencies = [
"walkdir",
]
[[package]]
name = "sysinfo"
version = "0.37.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f"
dependencies = [
"libc",
"memchr",
"ntapi",
"objc2-core-foundation",
"objc2-io-kit",
"windows 0.61.3",
]
[[package]]
name = "system-deps"
version = "6.2.2"
@@ -6407,6 +6665,21 @@ dependencies = [
"utf-8",
]
[[package]]
name = "termcolor"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
dependencies = [
"winapi-util",
]
[[package]]
name = "textwrap"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057"
[[package]]
name = "thiserror"
version = "1.0.69"

6
Cargo.toml
View File

@@ -32,4 +32,8 @@ rustpotter = { git = "https://github.com/Priler/rustpotter" }
image = "0.25"
parking_lot = "0.12.5"
toml = "0.9.8"
sha2 = "0.10"
sha2 = "0.10"
nnnoiseless = "0.5"
sysinfo = "0.37.2"
tokio-tungstenite = "0.28.0"
futures-util = "0.3"

33
crates/jarvis-app/src/app.rs
View File

@@ -1,6 +1,6 @@
use std::time::SystemTime;
use jarvis_core::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST, intent};
use jarvis_core::{audio, audio_processing, commands, config, listener, recorder, stt, COMMANDS_LIST, intent};
use rand::prelude::*;
pub fn start() -> Result<(), ()> {
@@ -14,6 +14,7 @@ fn main_loop() -> Result<(), ()> {
let sounds_directory = audio::get_sound_directory().unwrap();
let frame_length: usize = 512; // default for every wake-word engine
let mut frame_buffer: Vec<i16> = vec![0; frame_length];
let mut silence_frames: u32 = 0;
// play some run phrase
// @TODO. Different sounds? Or better make it via commands or upcoming events system.
@@ -33,16 +34,26 @@ fn main_loop() -> Result<(), ()> {
// read from microphone
recorder::read_microphone(&mut frame_buffer);
// process audio (gain -> noise suppression -> VAD)
let processed = audio_processing::process(&frame_buffer);
// skip if no voice detected (vad)
if !processed.is_voice {
continue 'wake_word;
}
// recognize wake-word
match listener::data_callback(&frame_buffer) {
Some(_keyword_index) => {
// reset speech recognizer
// reset some things
stt::reset_wake_recognizer();
stt::reset_speech_recognizer();
audio_processing::reset();
// wake-word activated, process further commands
// capture current time
start = SystemTime::now();
silence_frames = 0;
// play some greet phrase
// @TODO. Make it via commands or upcoming events system.
@@ -58,6 +69,20 @@ fn main_loop() -> Result<(), ()> {
// read from microphone
recorder::read_microphone(&mut frame_buffer);
// process first
let processed = audio_processing::process(&frame_buffer);
// detect silence, return to wake-word if silence
if processed.is_voice {
silence_frames = 0;
} else {
silence_frames += 1;
if silence_frames > config::VAD_SILENCE_FRAMES * 2 {
info!("Long silence detected, returning to wake word mode.");
break 'voice_recognition;
}
}
// stt part (without partials)
if let Some(mut recognized_voice) = stt::recognize(&frame_buffer, false) {
// something was recognized
@@ -81,6 +106,7 @@ fn main_loop() -> Result<(), ()> {
// reset timer and continue listening
start = SystemTime::now();
silence_frames = 0;
stt::reset_speech_recognizer();
continue 'voice_recognition;
}
@@ -149,8 +175,9 @@ fn main_loop() -> Result<(), ()> {
_ => (),
}
// reset wake recognizer
// reset things
stt::reset_wake_recognizer();
audio_processing::reset();
}
}
None => (),

8
crates/jarvis-app/src/main.rs
View File

@@ -3,7 +3,7 @@ use std::sync::Arc;
// include core
use jarvis_core::{
audio, commands, config, db, listener, recorder, stt, intent,
audio, audio_processing, commands, config, db, listener, recorder, stt, intent,
APP_CONFIG_DIR, APP_LOG_DIR, COMMANDS_LIST, DB,
};
@@ -90,6 +90,12 @@ fn main() -> Result<(), String> {
}
});
// init audio processing
info!("Initializing audio processing...");
if let Err(e) = audio_processing::init() {
warn!("Audio processing init failed: {}", e);
}
// start the app (in the background thread)
std::thread::spawn(|| {
let _ = app::start();

3
crates/jarvis-core/Cargo.toml
View File

@@ -23,6 +23,7 @@ rustpotter.workspace = true
parking_lot.workspace = true
toml.workspace = true
sha2.workspace = true
nnnoiseless = { workspace = true, optional = true }
# pv_recorder = { workspace = true, optional = true }
vosk = { version = "0.3.1", optional = true }
@@ -33,5 +34,5 @@ tokio = { version = "1", features = ["sync"], optional = true }
[features]
default = ["jarvis_app"]
jarvis_app = ["vosk", "intent-classifier", "tokio"]
jarvis_app = ["vosk", "intent-classifier", "tokio", "nnnoiseless"]
intent = ["intent-classifier", "tokio"]

118
crates/jarvis-core/src/audio_processing.rs Normal file
View File

@@ -0,0 +1,118 @@
pub mod noise_suppression;
pub mod vad;
pub mod gain_normalizer;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
use crate::config::structs::{NoiseSuppressionBackend, VadBackend};
use crate::DB;
static PROCESSOR: OnceCell<Mutex<AudioProcessor>> = OnceCell::new();
#[derive(Debug, Clone)]
pub struct ProcessedAudio {
pub samples: Vec<i16>,
pub is_voice: bool,
pub vad_confidence: f32,
}
struct AudioProcessor {
ns_backend: NoiseSuppressionBackend,
vad_backend: VadBackend,
gain_enabled: bool,
}
impl AudioProcessor {
fn new(ns: NoiseSuppressionBackend, vad: VadBackend, gain: bool) -> Self {
// init backends
noise_suppression::init(ns);
vad::init(vad);
if gain {
gain_normalizer::init();
}
Self {
ns_backend: ns,
vad_backend: vad,
gain_enabled: gain,
}
}
fn process(&mut self, input: &[i16]) -> ProcessedAudio {
let mut samples = input.to_vec();
// step 1: gain normalization (before other processing)
if self.gain_enabled {
samples = gain_normalizer::normalize(&samples);
}
// step 2: noise suppression
samples = noise_suppression::process(&samples);
// step 3: VAD
let (is_voice, confidence) = vad::detect(&samples);
ProcessedAudio {
samples,
is_voice,
vad_confidence: confidence,
}
}
fn reset(&mut self) {
noise_suppression::reset();
vad::reset();
gain_normalizer::reset();
}
}
pub fn init() -> Result<(), String> {
if PROCESSOR.get().is_some() {
return Ok(());
}
let (ns, vad, gain) = get_settings();
info!("Initializing audio processing: NS={:?}, VAD={:?}, Gain={}", ns, vad, gain);
let processor = AudioProcessor::new(ns, vad, gain);
PROCESSOR
.set(Mutex::new(processor))
.map_err(|_| "Audio processor already initialized")?;
info!("Audio processing initialized.");
Ok(())
}
pub fn process(input: &[i16]) -> ProcessedAudio {
match PROCESSOR.get() {
Some(p) => p.lock().unwrap().process(input),
None => ProcessedAudio {
samples: input.to_vec(),
is_voice: true,
vad_confidence: 1.0,
},
}
}
pub fn reset() {
if let Some(p) = PROCESSOR.get() {
p.lock().unwrap().reset();
}
}
fn get_settings() -> (NoiseSuppressionBackend, VadBackend, bool) {
match DB.get() {
Some(db) => {
let settings = db.read();
(settings.noise_suppression, settings.vad, settings.gain_normalizer)
}
None => (
crate::config::DEFAULT_NOISE_SUPPRESSION,
crate::config::DEFAULT_VAD,
crate::config::DEFAULT_GAIN_NORMALIZER,
),
}
}

28
crates/jarvis-core/src/audio_processing/gain_normalizer.rs Normal file
View File

@@ -0,0 +1,28 @@
mod simple;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
static NORMALIZER: OnceCell<Mutex<simple::GainNormalizer>> = OnceCell::new();
pub fn init() {
if NORMALIZER.get().is_some() {
return;
}
NORMALIZER.set(Mutex::new(simple::GainNormalizer::new())).ok();
info!("Gain normalizer: enabled");
}
pub fn normalize(input: &[i16]) -> Vec<i16> {
match NORMALIZER.get() {
Some(n) => n.lock().unwrap().normalize(input),
None => input.to_vec(),
}
}
pub fn reset() {
if let Some(n) = NORMALIZER.get() {
n.lock().unwrap().reset();
}
}

47
crates/jarvis-core/src/audio_processing/gain_normalizer/simple.rs Normal file
View File

@@ -0,0 +1,47 @@
use crate::config;
pub struct GainNormalizer {
current_gain: f32,
}
impl GainNormalizer {
pub fn new() -> Self {
Self { current_gain: 1.0 }
}
pub fn normalize(&mut self, input: &[i16]) -> Vec<i16> {
let rms = self.calculate_rms(input);
if rms < 1.0 {
return input.to_vec();
}
let target_gain = config::GAIN_TARGET_RMS / rms;
let clamped_gain = target_gain.clamp(config::GAIN_MIN, config::GAIN_MAX);
self.current_gain = self.current_gain * 0.9 + clamped_gain * 0.1;
input.iter()
.map(|&s| {
let amplified = (s as f32) * self.current_gain;
amplified.clamp(i16::MIN as f32, i16::MAX as f32) as i16
})
.collect()
}
pub fn reset(&mut self) {
self.current_gain = 1.0;
}
fn calculate_rms(&self, samples: &[i16]) -> f32 {
if samples.is_empty() {
return 0.0;
}
let sum: f64 = samples.iter()
.map(|&s| (s as f64).powi(2))
.sum();
(sum / samples.len() as f64).sqrt() as f32
}
}

66
crates/jarvis-core/src/audio_processing/noise_suppression.rs Normal file
View File

@@ -0,0 +1,66 @@
mod none;
#[cfg(feature = "nnnoiseless")]
mod nnnoiseless;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
use crate::config::structs::NoiseSuppressionBackend;
static BACKEND: OnceCell<NoiseSuppressionBackend> = OnceCell::new();
#[cfg(feature = "nnnoiseless")]
static NNNOISELESS_STATE: OnceCell<Mutex<nnnoiseless::NnnoiselessNS>> = OnceCell::new();
pub fn init(backend: NoiseSuppressionBackend) {
if BACKEND.get().is_some() {
return;
}
BACKEND.set(backend).ok();
match backend {
NoiseSuppressionBackend::None => {
info!("Noise suppression: disabled");
}
#[cfg(feature = "nnnoiseless")]
NoiseSuppressionBackend::Nnnoiseless => {
NNNOISELESS_STATE.set(Mutex::new(nnnoiseless::NnnoiselessNS::new())).ok();
info!("Noise suppression: Nnnoiseless");
}
#[cfg(not(feature = "nnnoiseless"))]
NoiseSuppressionBackend::Nnnoiseless => {
warn!("Nnnoiseless not compiled in, falling back to None");
BACKEND.set(NoiseSuppressionBackend::None).ok();
}
}
}
pub fn process(input: &[i16]) -> Vec<i16> {
match BACKEND.get() {
Some(NoiseSuppressionBackend::None) | None => none::process(input),
#[cfg(feature = "nnnoiseless")]
Some(NoiseSuppressionBackend::Nnnoiseless) => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().unwrap().process(input)
} else {
none::process(input)
}
}
#[cfg(not(feature = "nnnoiseless"))]
Some(NoiseSuppressionBackend::Nnnoiseless) => none::process(input),
}
}
pub fn reset() {
match BACKEND.get() {
#[cfg(feature = "nnnoiseless")]
Some(NoiseSuppressionBackend::Nnnoiseless) => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().unwrap().reset();
}
}
_ => {}
}
}

51
crates/jarvis-core/src/audio_processing/noise_suppression/nnnoiseless.rs Normal file
View File

@@ -0,0 +1,51 @@
use nnnoiseless::DenoiseState;
use crate::config;
pub struct NnnoiselessNS {
state: Box<DenoiseState<'static>>,
buffer: Vec<f32>,
}
impl NnnoiselessNS {
pub fn new() -> Self {
Self {
state: DenoiseState::new(),
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
}
}
pub fn process(&mut self, input: &[i16]) -> Vec<i16> {
for &sample in input {
self.buffer.push(sample as f32);
}
let mut output: Vec<i16> = Vec::with_capacity(input.len());
while self.buffer.len() >= config::NNNOISELESS_FRAME_SIZE {
let mut input_frame = [0.0f32; 480];
let mut output_frame = [0.0f32; 480];
input_frame.copy_from_slice(&self.buffer[..config::NNNOISELESS_FRAME_SIZE]);
self.buffer.drain(..config::NNNOISELESS_FRAME_SIZE);
// process: input -> output (denoised)
let _ = self.state.process_frame(&mut output_frame, &input_frame);
for &sample in &output_frame {
let clamped = sample.clamp(i16::MIN as f32, i16::MAX as f32);
output.push(clamped as i16);
}
}
if output.is_empty() {
return input.to_vec();
}
output
}
pub fn reset(&mut self) {
self.state = DenoiseState::new();
self.buffer.clear();
}
}

4
crates/jarvis-core/src/audio_processing/noise_suppression/none.rs Normal file
View File

@@ -0,0 +1,4 @@
// return unprocessed input
pub fn process(input: &[i16]) -> Vec<i16> {
input.to_vec()
}

72
crates/jarvis-core/src/audio_processing/vad.rs Normal file
View File

@@ -0,0 +1,72 @@
mod none;
mod energy;
#[cfg(feature = "nnnoiseless")]
mod nnnoiseless;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
use crate::config::structs::VadBackend;
static BACKEND: OnceCell<VadBackend> = OnceCell::new();
#[cfg(feature = "nnnoiseless")]
static NNNOISELESS_STATE: OnceCell<Mutex<nnnoiseless::NnnoiselessVAD>> = OnceCell::new();
pub fn init(backend: VadBackend) {
if BACKEND.get().is_some() {
return;
}
BACKEND.set(backend).ok();
match backend {
VadBackend::None => {
info!("VAD: disabled");
}
VadBackend::Energy => {
info!("VAD: Energy-based");
}
#[cfg(feature = "nnnoiseless")]
VadBackend::Nnnoiseless => {
NNNOISELESS_STATE.set(Mutex::new(nnnoiseless::NnnoiselessVAD::new())).ok();
info!("VAD: Nnnoiseless");
}
#[cfg(not(feature = "nnnoiseless"))]
VadBackend::Nnnoiseless => {
warn!("Nnnoiseless not compiled in, falling back to Energy");
BACKEND.set(VadBackend::Energy).ok();
}
}
}
// Returns (is_voice, confidence)
pub fn detect(input: &[i16]) -> (bool, f32) {
match BACKEND.get() {
Some(VadBackend::None) | None => none::detect(input),
Some(VadBackend::Energy) => energy::detect(input),
#[cfg(feature = "nnnoiseless")]
Some(VadBackend::Nnnoiseless) => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().unwrap().detect(input)
} else {
energy::detect(input)
}
}
#[cfg(not(feature = "nnnoiseless"))]
Some(VadBackend::Nnnoiseless) => energy::detect(input),
}
}
pub fn reset() {
match BACKEND.get() {
#[cfg(feature = "nnnoiseless")]
Some(VadBackend::Nnnoiseless) => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().unwrap().reset();
}
}
_ => {}
}
}

24
crates/jarvis-core/src/audio_processing/vad/energy.rs Normal file
View File

@@ -0,0 +1,24 @@
use crate::config;
// Simple energy-based VAD
pub fn detect(input: &[i16]) -> (bool, f32) {
let rms = calculate_rms(input);
let is_voice = rms > config::VAD_ENERGY_THRESHOLD;
// normalize confidence to 0-1 range (rough approximation)
let confidence = (rms / (config::VAD_ENERGY_THRESHOLD * 2.0)).min(1.0);
(is_voice, confidence)
}
fn calculate_rms(samples: &[i16]) -> f32 {
if samples.is_empty() {
return 0.0;
}
let sum: f64 = samples.iter()
.map(|&s| (s as f64).powi(2))
.sum();
(sum / samples.len() as f64).sqrt() as f32
}

51
crates/jarvis-core/src/audio_processing/vad/nnnoiseless.rs Normal file
View File

@@ -0,0 +1,51 @@
use nnnoiseless::DenoiseState;
use crate::config;
pub struct NnnoiselessVAD {
state: Box<DenoiseState<'static>>,
buffer: Vec<f32>,
}
impl NnnoiselessVAD {
pub fn new() -> Self {
Self {
state: DenoiseState::new(),
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
}
}
pub fn detect(&mut self, input: &[i16]) -> (bool, f32) {
for &sample in input {
self.buffer.push(sample as f32);
}
let mut total_vad = 0.0f32;
let mut frame_count = 0u32;
while self.buffer.len() >= config::NNNOISELESS_FRAME_SIZE {
let mut input_frame = [0.0f32; 480];
let mut output_frame = [0.0f32; 480];
input_frame.copy_from_slice(&self.buffer[..config::NNNOISELESS_FRAME_SIZE]);
self.buffer.drain(..config::NNNOISELESS_FRAME_SIZE);
let vad_prob = self.state.process_frame(&mut output_frame, &input_frame);
total_vad += vad_prob;
frame_count += 1;
}
if frame_count == 0 {
return (true, 0.5);
}
let avg_vad = total_vad / frame_count as f32;
let is_voice = avg_vad >= config::VAD_NNNOISELESS_THRESHOLD;
(is_voice, avg_vad)
}
pub fn reset(&mut self) {
self.state = DenoiseState::new();
self.buffer.clear();
}
}

4
crates/jarvis-core/src/audio_processing/vad/none.rs Normal file
View File

@@ -0,0 +1,4 @@
// Always returns voice detected (no vad)
pub fn detect(_input: &[i16]) -> (bool, f32) {
(true, 1.0)
}

28
crates/jarvis-core/src/config.rs
View File

@@ -18,6 +18,8 @@ use rustpotter::{
};
use crate::IntentRecognitionEngine;
use crate::config::structs::NoiseSuppressionBackend;
use crate::config::structs::VadBackend;
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
#[allow(dead_code)]
@@ -106,7 +108,11 @@ pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
},
filters: FiltersConfig {
gain_normalizer: GainNormalizationConfig {
enabled: true,
// enabled: true,
// gain_ref: None,
// min_gain: 0.7,
// max_gain: 1.0,
enabled: false, // disable, now we have separate gain normalizer implementation
gain_ref: None,
min_gain: 0.7,
max_gain: 1.0,
@@ -146,6 +152,26 @@ pub const VOSK_SPEECH_PARTIAL_WORDS: bool = false;
// IRE (intents recognition)
pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.75;
// AUDIO PROCESSING DEFAULTS
pub const DEFAULT_NOISE_SUPPRESSION: NoiseSuppressionBackend = NoiseSuppressionBackend::None;
pub const DEFAULT_VAD: VadBackend = VadBackend::Energy;
pub const DEFAULT_GAIN_NORMALIZER: bool = false;
// VAD settings
pub const VAD_ENERGY_THRESHOLD: f32 = 500.0; // RMS threshold for energy-based VAD
pub const VAD_NNNOISELESS_THRESHOLD: f32 = 0.5; // probability threshold for nnnoiseless
pub const VAD_SILENCE_FRAMES: u32 = 15; // frames of silence before speech end (~480ms)
// gain normalizer settings
pub const GAIN_TARGET_RMS: f32 = 3000.0; // target RMS level
pub const GAIN_MIN: f32 = 0.5; // minimum gain multiplier
pub const GAIN_MAX: f32 = 3.0; // maximum gain multiplier
// nnnoiseless frame size (fixed by library)
pub const NNNOISELESS_FRAME_SIZE: usize = 480;
// ETC
pub const CMD_RATIO_THRESHOLD: f64 = 65f64;
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);

57
crates/jarvis-core/src/config/structs.rs
View File

@@ -1,23 +1,30 @@
use std::fmt;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum WakeWordEngine {
Rustpotter,
Vosk,
Porcupine,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum IntentRecognitionEngine {
IntentClassifier,
Rasa,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum NoiseSuppressionBackend {
None,
Nnnoiseless,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum VadBackend {
None,
Energy,
Nnnoiseless,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
@@ -25,12 +32,6 @@ pub enum SpeechToTextEngine {
Vosk,
}
impl fmt::Display for SpeechToTextEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[derive(PartialEq, Debug)]
pub enum RecorderType {
Cpal,
@@ -44,6 +45,38 @@ pub enum AudioType {
Kira,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for SpeechToTextEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for IntentRecognitionEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for NoiseSuppressionBackend {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for VadBackend {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// pub enum TextToSpeechEngine {}
// pub enum IntentRecognitionEngine {}

14
crates/jarvis-core/src/db/structs.rs
View File

@@ -4,6 +4,8 @@ use serde::{Deserialize, Serialize};
use crate::config::structs::SpeechToTextEngine;
use crate::config::structs::WakeWordEngine;
use crate::config::structs::IntentRecognitionEngine;
use crate::config::structs::NoiseSuppressionBackend;
use crate::config::structs::VadBackend;
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Settings {
@@ -13,9 +15,13 @@ pub struct Settings {
pub wake_word_engine: WakeWordEngine,
pub intent_recognition_engine: IntentRecognitionEngine,
pub speech_to_text_engine: SpeechToTextEngine,
pub vosk_model: String,
// audio processing
pub noise_suppression: NoiseSuppressionBackend,
pub vad: VadBackend,
pub gain_normalizer: bool,
pub api_keys: ApiKeys,
}
@@ -28,9 +34,13 @@ impl Default for Settings {
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
intent_recognition_engine: config::DEFAULT_INTENT_RECOGNITION_ENGINE,
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
vosk_model: String::from(""), // auto detect first available
// audio processing defaults
noise_suppression: config::DEFAULT_NOISE_SUPPRESSION,
vad: config::DEFAULT_VAD,
gain_normalizer: config::DEFAULT_GAIN_NORMALIZER,
api_keys: ApiKeys {
picovoice: String::from(""),
openai: String::from(""),

3
crates/jarvis-core/src/lib.rs
View File

@@ -25,6 +25,9 @@ pub mod intent;
pub mod vosk_models;
#[cfg(feature = "jarvis_app")]
pub mod audio_processing;
// shared statics
// pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| std::env::current_dir().unwrap());
pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| {

4
crates/jarvis-core/src/recorder/pvrecorder.rs
View File

@@ -114,6 +114,10 @@ pub fn list_audio_devices() -> Vec<String> {
}
pub fn get_audio_device_name(idx: i32) -> String {
if idx == -1 {
return String::from("System Default");
}
let audio_devices = list_audio_devices();
let mut first_device: String = String::new();

1
crates/jarvis-gui/Cargo.toml
View File

@@ -16,6 +16,7 @@ tauri-plugin-fs = "2"
peak_alloc = "0.3.0"
systemstat = "0.2"
lazy_static = "1.4"
sysinfo.workspace = true
once_cell.workspace = true
log.workspace = true

3
crates/jarvis-gui/src/main.rs
View File

@@ -61,6 +61,9 @@ fn main() {
tauri_commands::get_peak_ram_usage,
tauri_commands::get_cpu_temp,
tauri_commands::get_cpu_usage,
tauri_commands::get_jarvis_app_stats,
tauri_commands::is_jarvis_app_running,
tauri_commands::run_jarvis_app,
// vosk
tauri_commands::list_vosk_models,

25
crates/jarvis-gui/src/tauri_commands/db.rs
View File

@@ -12,6 +12,9 @@ pub fn db_read(state: tauri::State<'_, AppState>, key: &str) -> String {
"selected_intent_recognition_engine" => format!("{:?}", settings.intent_recognition_engine),
"selected_vosk_model" => settings.vosk_model.clone(),
"speech_to_text_engine" => format!("{:?}", settings.speech_to_text_engine),
"noise_suppression" => format!("{:?}", settings.noise_suppression),
"vad" => format!("{:?}", settings.vad),
"gain_normalizer" => settings.gain_normalizer.to_string(),
"api_key__picovoice" => settings.api_keys.picovoice.clone(),
"api_key__openai" => settings.api_keys.openai.clone(),
_ => String::new(),
@@ -53,6 +56,28 @@ pub fn db_write(state: tauri::State<'_, AppState>, key: &str, val: &str) -> bool
"selected_vosk_model" => {
settings.vosk_model = val.to_string();
}
"noise_suppression" => {
match val.to_lowercase().as_str() {
"none" => settings.noise_suppression = jarvis_core::config::structs::NoiseSuppressionBackend::None,
"nnnoiseless" => settings.noise_suppression = jarvis_core::config::structs::NoiseSuppressionBackend::Nnnoiseless,
_ => return false,
}
}
"vad" => {
match val.to_lowercase().as_str() {
"none" => settings.vad = jarvis_core::config::structs::VadBackend::None,
"energy" => settings.vad = jarvis_core::config::structs::VadBackend::Energy,
"nnnoiseless" => settings.vad = jarvis_core::config::structs::VadBackend::Nnnoiseless,
_ => return false,
}
}
"gain_normalizer" => {
match val.to_lowercase().as_str() {
"true" => settings.gain_normalizer = true,
"false" => settings.gain_normalizer = false,
_ => return false,
}
}
"api_key__picovoice" => {
settings.api_keys.picovoice = val.to_string();
}

159
crates/jarvis-gui/src/tauri_commands/sys.rs
View File

@@ -1,49 +1,152 @@
use sysinfo::{System, Pid, ProcessRefreshKind, RefreshKind, CpuRefreshKind, Components};
use peak_alloc::PeakAlloc;
use std::sync::Mutex;
use once_cell::sync::Lazy;
use std::process::Command;
use std::env;
#[global_allocator]
static PEAK_ALLOC: PeakAlloc = PeakAlloc;
extern crate systemstat;
use std::thread;
use std::time::Duration;
use systemstat::{Platform, System};
use lazy_static::lazy_static;
static SYS: Lazy<Mutex<System>> = Lazy::new(|| {
Mutex::new(System::new_with_specifics(
RefreshKind::nothing()
.with_processes(ProcessRefreshKind::nothing().with_memory().with_cpu())
.with_cpu(CpuRefreshKind::everything())
))
});
lazy_static! {
static ref SYS: System = System::new();
static COMPONENTS: Lazy<Mutex<Components>> = Lazy::new(|| {
Mutex::new(Components::new_with_refreshed_list())
});
const JARVIS_APP_NAME: &str = "jarvis-app";
/// Find jarvis-app process and return its PID
fn find_jarvis_app_pid(sys: &System) -> Option<Pid> {
for (pid, process) in sys.processes() {
let name = process.name().to_string_lossy().to_lowercase();
if name.contains(JARVIS_APP_NAME) {
return Some(*pid);
}
}
None
}
#[derive(serde::Serialize)]
pub struct JarvisAppStats {
pub running: bool,
pub ram_mb: u64,
pub cpu_usage: f32,
}
#[tauri::command]
pub fn get_current_ram_usage() -> String {
let result = String::from(format!("{}", PEAK_ALLOC.current_usage_as_mb()));
result
pub fn get_jarvis_app_stats() -> JarvisAppStats {
let mut sys = SYS.lock().unwrap();
// refresh all processes to find jarvis-app
sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true);
if let Some(pid) = find_jarvis_app_pid(&sys) {
if let Some(proc) = sys.process(pid) {
return JarvisAppStats {
running: true,
ram_mb: proc.memory() / 1024 / 1024,
cpu_usage: proc.cpu_usage(),
};
}
}
JarvisAppStats {
running: false,
ram_mb: 0,
cpu_usage: 0.0,
}
}
#[tauri::command]
pub fn get_peak_ram_usage() -> String {
let result = String::from(format!("{}", PEAK_ALLOC.peak_usage_as_gb()));
pub fn get_current_ram_usage() -> u64 {
let mut sys = SYS.lock().unwrap();
sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true);
if let Some(pid) = find_jarvis_app_pid(&sys) {
if let Some(proc) = sys.process(pid) {
return proc.memory() / 1024 / 1024;
}
}
0
}
result
#[tauri::command]
pub fn is_jarvis_app_running() -> bool {
let mut sys = SYS.lock().unwrap();
sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true);
find_jarvis_app_pid(&sys).is_some()
}
#[tauri::command]
pub fn get_cpu_temp() -> String {
if let Ok(cpu_temp) = SYS.cpu_temp() {
String::from(format!("{}", cpu_temp))
} else {
String::from("error")
let mut components = COMPONENTS.lock().unwrap();
components.refresh(true);
for component in components.iter() {
let label = component.label().to_lowercase();
if label.contains("cpu") || label.contains("core") || label.contains("package") {
if let Some(temp) = component.temperature() {
return format!("{:.1}", temp);
}
}
}
if let Some(component) = components.iter().next() {
if let Some(temp) = component.temperature() {
return format!("{:.1}", temp);
}
}
String::from("N/A")
}
// https://github.com/valpackett/systemstat/blob/trunk/examples/info.rs
#[tauri::command(async)]
pub async fn get_cpu_usage() -> String {
if let Ok(cpu) = SYS.cpu_load_aggregate() {
thread::sleep(Duration::from_secs(1));
let cpu = cpu.done().unwrap();
String::from(format!("{}", cpu.user * 100.0))
} else {
String::from("error")
}
#[tauri::command]
pub fn get_cpu_usage() -> f32 {
let mut sys = SYS.lock().unwrap();
sys.refresh_cpu_all();
std::thread::sleep(std::time::Duration::from_millis(200));
sys.refresh_cpu_all();
sys.global_cpu_usage()
}
#[tauri::command]
pub fn get_peak_ram_usage() -> String {
format!("{}", PEAK_ALLOC.peak_usage_as_gb())
}
#[tauri::command]
pub fn run_jarvis_app() -> Result<(), String> {
let exe_dir = std::env::current_exe()
.map_err(|e| format!("Failed to get exe path: {}", e))?
.parent()
.ok_or("Failed to get exe directory")?
.to_path_buf();
#[cfg(target_os = "windows")]
let jarvis_app_name = "jarvis-app.exe";
#[cfg(not(target_os = "windows"))]
let jarvis_app_name = "jarvis-app";
let jarvis_app_path = exe_dir.join(jarvis_app_name);
if !jarvis_app_path.exists() {
return Err(format!("jarvis-app not found at: {}", jarvis_app_path.display()));
}
std::process::Command::new(&jarvis_app_path)
.spawn()
.map_err(|e| format!("Failed to start jarvis-app: {}", e))?;
Ok(())
}

18
frontend/src/App.svelte
View File

@@ -1,8 +1,26 @@
<script lang="ts">
import { onMount, onDestroy } from "svelte"
import { Router } from "@roxi/routify"
import routes from "../.routify/routes.default.js"
import { SvelteUIProvider } from "@svelteuidev/core"
import Events from "./Events.svelte"
import {
loadVoiceSetting,
loadAppInfo,
startStatsPolling,
stopStatsPolling
} from "@/stores"
onMount(() => {
loadVoiceSetting()
loadAppInfo()
startStatsPolling(5000)
})
onDestroy(() => {
stopStatsPolling()
})
</script>
<SvelteUIProvider themeObserver="dark" withNormalizeCSS withGlobalStyles>

31
frontend/src/components/elements/Stats.svelte
View File

@@ -3,17 +3,24 @@
import { invoke } from "@tauri-apps/api/core"
import { capitalizeFirstLetter } from "@/functions"
import {
Text,
} from "@svelteuidev/core"
let jarvisStats = { running: false, ram_mb: 0, cpu_usage: 0 }
let microphoneLabel = ""
let wakeWordEngine = ""
let sttEngine = "Vosk"
let ramUsage = "-"
// let ramUsage = "-"
let interval: number | null = null
let statsUpdateInterval: number | null = null
async function updateRamUsage() {
async function updateStats() {
try {
const usage = await invoke<number>("get_current_ram_usage")
ramUsage = usage.toFixed(2)
jarvisStats = await invoke<{running: boolean, ram_mb: number, cpu_usage: number}>("get_jarvis_app_stats")
//const usage = await invoke<number>("get_current_ram_usage")
//ramUsage = usage.toFixed(2)
} catch (err) {
console.error("failed to get ram usage:", err)
}
@@ -21,7 +28,8 @@
onMount(async () => {
// start polling ram usage
interval = setInterval(updateRamUsage, 1000) as unknown as number
updateStats()
statsUpdateInterval = setInterval(updateStats, 5000) as unknown as number
try {
// load microphone info
@@ -37,8 +45,8 @@
})
onDestroy(() => {
if (interval) {
clearInterval(interval)
if (statsUpdateInterval) {
clearInterval(statsUpdateInterval)
}
})
</script>
@@ -64,7 +72,12 @@
<div class="pulse"><div class="wave"></div></div>
<div class="info">
<span class="num">Ресурсы</span>
<small>RAM {ramUsage}mb</small>
{#if jarvisStats.running}
<small>RAM: {jarvisStats.ram_mb} MB</small>
<!--<Text>CPU: {jarvisStats.cpu_usage.toFixed(1)}%</Text>-->
{:else}
<Text color="gray">-</Text>
{/if}
</div>
</div>
</div>

14
frontend/src/functions.ts
View File

@@ -11,17 +11,3 @@ export function showInExplorer(path: string): void {
invoke("show_in_folder", { path })
.catch(err => console.error("failed to open explorer:", err))
}
// ### LISTENER FUNCTIONS
// removed since gui now doesn't handle listening
export function startListening(): void {
// disabled in GUI - listening is handled by the tray app
console.log("[gui] listening not available in settings app")
}
export function stopListening(callback?: () => void): void {
// disabled in GUI - just call the callback if provided
console.log("[gui] listening not available in settings app")
if (callback) callback()
}

50
frontend/src/routes/index.svelte
View File

@@ -1,6 +1,7 @@
<script lang="ts">
import { onMount, onDestroy } from "svelte"
import { Notification, Space } from "@svelteuidev/core"
import { invoke } from "@tauri-apps/api/core"
import { Notification, Space, Button } from "@svelteuidev/core"
import { InfoCircled } from "radix-icons-svelte"
import SearchBar from "@/components/elements/SearchBar.svelte"
@@ -9,11 +10,13 @@
import Stats from "@/components/elements/Stats.svelte"
import Footer from "@/components/Footer.svelte"
import { isListening } from "@/stores"
import { isJarvisRunning, updateJarvisStats } from "@/stores"
let listening = false
isListening.subscribe(value => {
listening = value
let running = false
let launching = false
isJarvisRunning.subscribe(value => {
running = value
})
onMount(() => {
@@ -23,19 +26,48 @@
onDestroy(() => {
document.body.classList.remove("assist-page")
})
async function runAssistant() {
launching = true
try {
await invoke("run_jarvis_app")
// wait a bit then check if it's running
setTimeout(() => {
updateJarvisStats()
launching = false
}, 2000)
} catch (err) {
console.error("Failed to run jarvis-app:", err)
launching = false
}
}
</script>
<HDivider />
{#if !listening}
{#if !running}
<Notification
title="Внимание!"
icon={InfoCircled}
color="cyan"
withCloseButton={false}
>
В данный момент ассистент не прослушивает команды.<br />
Пожалуйста, <a href="/settings">перейдите в настройки</a> и введите ключ Picovoice.
В данный момент ассистент не запущен.<br />
Но вы всё еще можете изменять его настройки.<br />
<br />
<Button
color="lime"
radius="md"
size="sm"
uppercase
ripple
fullSize
on:click={runAssistant}
disabled={launching}
>
{launching ? "Запуск..." : "Запустить"}
</Button>
</Notification>
{:else}
<ArcReactor />
@@ -43,4 +75,4 @@
<HDivider noMargin />
<Stats />
<Footer />
<Footer />

84
frontend/src/routes/settings/index.svelte
View File

@@ -4,7 +4,7 @@
import { goto } from "@roxi/routify"
import { setTimeout } from "worker-timers"
import { showInExplorer, stopListening, startListening } from "@/functions"
import { showInExplorer } from "@/functions"
import { appInfo, assistantVoice } from "@/stores"
import HDivider from "@/components/elements/HDivider.svelte"
@@ -19,7 +19,8 @@
Alert,
Input,
InputWrapper,
NativeSelect
NativeSelect,
Switch
} from "@svelteuidev/core"
import {
@@ -43,12 +44,15 @@
let settingsSaved = false
let saveButtonDisabled = false
// form values
// form values (state vars)
let voiceVal = ""
let selectedMicrophone = ""
let selectedWakeWordEngine = ""
let selectedIntentRecognitionEngine = ""
let selectedVoskModel = ""
let selectedNoiseSuppression = ""
let selectedVad = ""
let gainNormalizerEnabled = false
let apiKeyPicovoice = ""
let apiKeyOpenai = ""
@@ -76,6 +80,11 @@
invoke("db_write", { key: "selected_wake_word_engine", val: selectedWakeWordEngine }),
invoke("db_write", { key: "selected_intent_recognition_engine", val: selectedIntentRecognitionEngine }),
invoke("db_write", { key: "selected_vosk_model", val: selectedVoskModel }),
invoke("db_write", { key: "noise_suppression", val: selectedNoiseSuppression }),
invoke("db_write", { key: "vad", val: selectedVad }),
invoke("db_write", { key: "gain_normalizer", val: gainNormalizerEnabled.toString() }),
invoke("db_write", { key: "api_key__picovoice", val: apiKeyPicovoice }),
invoke("db_write", { key: "api_key__openai", val: apiKeyOpenai })
])
@@ -90,7 +99,7 @@
}, 5000)
// restart listening with new settings
stopListening(() => startListening())
// stopListening(() => startListening())
} catch (err) {
console.error("failed to save settings:", err)
}
@@ -105,10 +114,13 @@
try {
// load microphones
const mics = await invoke<string[]>("pv_get_audio_devices")
availableMicrophones = mics.map((name, idx) => ({
label: name,
value: String(idx)
}))
availableMicrophones = [
{ label: "По умолчанию (Система)", value: "-1" }, // system default
...mics.map((name, idx) => ({
label: name,
value: String(idx)
}))
]
// load vosk models
const voskModels = await invoke<{ name: string; language: string; size: string }[]>("list_vosk_models")
@@ -118,11 +130,18 @@
}))
// load settings from db
const [mic, wakeWord, intentReco, voskModel, pico, openai] = await Promise.all([
const [mic, wakeWord, intentReco, voskModel,
noiseSuppression, vad, gainNormalizer,
pico, openai] = await Promise.all([
invoke<string>("db_read", { key: "selected_microphone" }),
invoke<string>("db_read", { key: "selected_wake_word_engine" }),
invoke<string>("db_read", { key: "selected_intent_recognition_engine" }),
invoke<string>("db_read", { key: "selected_vosk_model" }),
invoke<string>("db_read", { key: "noise_suppression" }),
invoke<string>("db_read", { key: "vad" }),
invoke<string>("db_read", { key: "gain_normalizer" }),
invoke<string>("db_read", { key: "api_key__picovoice" }),
invoke<string>("db_read", { key: "api_key__openai" })
])
@@ -131,6 +150,9 @@
selectedWakeWordEngine = wakeWord
selectedIntentRecognitionEngine = intentReco
selectedVoskModel = voskModel
selectedNoiseSuppression = noiseSuppression
selectedVad = vad
gainNormalizerEnabled = gainNormalizer === "true"
apiKeyPicovoice = pico
apiKeyOpenai = openai
} catch (err) {
@@ -179,7 +201,9 @@
<Space h="sm" />
<NativeSelect
data={[
{ label: "Jarvis ремейк (от Хауди)", value: "jarvis-remake" },
{ label: "Jarvis New (ремастер)", value: "jarvis-remaster" },
{ label: "Рик из «Рик и Морти»", value: "rick-morty" },
{ label: "Jarvis (от Хауди)", value: "jarvis-howdy" },
{ label: "Jarvis OG (из фильмов)", value: "jarvis-og" }
]}
label="Голос ассистента"
@@ -281,6 +305,46 @@
<Space h="xl" />
<NativeSelect
data={[
{ label: "Отключено", value: "None" },
{ label: "Nnnoiseless", value: "Nnnoiseless" }
]}
label="Шумоподавление"
description="Уменьшает фоновый шум. Может ухудшить распознавание в некоторых случаях."
variant="filled"
bind:value={selectedNoiseSuppression}
/>
<Space h="md" />
<NativeSelect
data={[
{ label: "Отключено", value: "None" },
{ label: "Energy (простой)", value: "Energy" },
{ label: "Nnnoiseless (нейросеть)", value: "Nnnoiseless" }
]}
label="Определение голосой активности (VAD)"
description="Пропускает тишину, экономит ресурсы CPU."
variant="filled"
bind:value={selectedVad}
/>
<Space h="md" />
<InputWrapper label="Нормализация громкости">
<Text size="sm" color="gray">
Автоматически регулирует уровень громкости.
</Text>
<Space h="xs" />
<Switch
label={gainNormalizerEnabled ? "Включено" : "Выключено"}
bind:checked={gainNormalizerEnabled}
/>
</InputWrapper>
<Space h="xl" />
<InputWrapper label="Ключ OpenAI">
<Text size="sm" color="gray">
В данный момент ChatGPT <u>не поддерживается</u>.

62
frontend/src/stores.ts
View File

@@ -1,26 +1,15 @@
import { writable, get } from "svelte/store"
import { writable } from "svelte/store"
import { invoke } from "@tauri-apps/api/core"
// ### LISTENING STATE
// note: defaults to false since GUI doesn't have listening capability
export const isListening = writable(false)
// ### RUNNING STATE
export const isJarvisRunning = writable(false)
export const jarvisRamUsage = writable(0)
export const jarvisCpuUsage = writable(0)
// ### ASSISTANT VOICE
export const assistantVoice = writable("")
// load voice setting from db
async function loadVoiceSetting() {
try {
const voice = await invoke<string>("db_read", { key: "assistant_voice" })
assistantVoice.set(voice)
} catch (err) {
console.error("failed to load voice setting:", err)
}
}
loadVoiceSetting()
// ### APP INFO
// these are loaded once on startup
export const appInfo = writable({
tgOfficialLink: "",
feedbackLink: "",
@@ -28,7 +17,17 @@ export const appInfo = writable({
logFilePath: ""
})
async function loadAppInfo() {
// ### INIT FUNCTIONS (call these from a component)
export async function loadVoiceSetting() {
try {
const voice = await invoke<string>("db_read", { key: "assistant_voice" })
assistantVoice.set(voice)
} catch (err) {
console.error("failed to load voice setting:", err)
}
}
export async function loadAppInfo() {
try {
const [tg, feedback, repo, logPath] = await Promise.all([
invoke<string>("get_tg_official_link"),
@@ -47,4 +46,31 @@ async function loadAppInfo() {
console.error("failed to load app info:", err)
}
}
loadAppInfo()
export async function updateJarvisStats() {
try {
const stats = await invoke<{running: boolean, ram_mb: number, cpu_usage: number}>("get_jarvis_app_stats")
isJarvisRunning.set(stats.running)
jarvisRamUsage.set(stats.ram_mb)
jarvisCpuUsage.set(stats.cpu_usage)
} catch (err) {
console.error("failed to get jarvis stats:", err)
}
}
// polling manager
let statsInterval: ReturnType<typeof setInterval> | null = null
export function startStatsPolling(intervalMs = 5000) {
if (statsInterval) return // already running
updateJarvisStats()
statsInterval = setInterval(updateJarvisStats, intervalMs)
}
export function stopStatsPolling() {
if (statsInterval) {
clearInterval(statsInterval)
statsInterval = null
}
}

0
resources/sound/jarvis-remake/game_mode.wav → resources/sound/jarvis-howdy/game_mode.wav
View File

0
resources/sound/jarvis-remake/greet1.wav → resources/sound/jarvis-howdy/greet1.wav
View File

0
resources/sound/jarvis-remake/greet2.wav → resources/sound/jarvis-howdy/greet2.wav
View File

0
resources/sound/jarvis-remake/greet3.wav → resources/sound/jarvis-howdy/greet3.wav
View File

0
resources/sound/jarvis-remake/joke1.wav → resources/sound/jarvis-howdy/joke1.wav
View File

0
resources/sound/jarvis-remake/joke2.wav → resources/sound/jarvis-howdy/joke2.wav
View File

0
resources/sound/jarvis-remake/joke3.wav → resources/sound/jarvis-howdy/joke3.wav
View File

0
resources/sound/jarvis-remake/joke4.wav → resources/sound/jarvis-howdy/joke4.wav
View File

0
resources/sound/jarvis-remake/joke5.wav → resources/sound/jarvis-howdy/joke5.wav
View File

0
resources/sound/jarvis-remake/not_found.wav → resources/sound/jarvis-howdy/not_found.wav
View File

0
resources/sound/jarvis-remake/ok1.wav → resources/sound/jarvis-howdy/ok1.wav
View File

0
resources/sound/jarvis-remake/ok2.wav → resources/sound/jarvis-howdy/ok2.wav
View File

0
resources/sound/jarvis-remake/ok3.wav → resources/sound/jarvis-howdy/ok3.wav
View File

0
resources/sound/jarvis-remake/ok4.wav → resources/sound/jarvis-howdy/ok4.wav
View File

0
resources/sound/jarvis-remake/ready.wav → resources/sound/jarvis-howdy/ready.wav
View File

0
resources/sound/jarvis-remake/run.wav → resources/sound/jarvis-howdy/run.wav
View File

0
resources/sound/jarvis-remake/stupid.wav → resources/sound/jarvis-howdy/stupid.wav
View File

0
resources/sound/jarvis-remake/thanks.wav → resources/sound/jarvis-howdy/thanks.wav
View File

BIN
resources/sound/jarvis-remaster/greet1.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/greet_day.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/greet_evening.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/greet_morning.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/greet_night.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/joke1.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/joke2.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/joke3.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/joke4.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/joke5.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/joke6.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/joke7.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/joke8.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/ok1.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/ok2.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/ok3.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/ok4.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/reply1.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/reply2.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/reply3.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/reply4.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/reply5.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/reply6.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/stupid.mp3 Normal file
View File

Binary file not shown.

BIN
resources/sound/jarvis-remaster/thanks.mp3 Normal file
View File

Binary file not shown.