diff --git a/package.json b/package.json index f993033..cc3adcf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "jarvis-app", "private": true, - "version": "0.0.1", + "version": "0.0.2", "type": "module", "scripts": { "dev": "routify -c dev:vite", diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 25a3cf1..feedbde 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -48,7 +48,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8512c9117059663fb5606788fbca3619e2a91dac0e3fe516242eab1fa6be5e44" dependencies = [ "alsa-sys", - "bitflags", + "bitflags 1.3.2", "libc", "nix", ] @@ -69,6 +69,12 @@ version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "arrayvec" version = "0.7.2" @@ -82,7 +88,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c3d816ce6f0e2909a96830d6911c2aff044370b1ef92d7f267b43bae5addedd" dependencies = [ "atk-sys", - "bitflags", + "bitflags 1.3.2", "glib", "libc", ] @@ -99,6 +105,17 @@ dependencies = [ "system-deps 6.0.5", ] +[[package]] +name = "atomic_enum" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6227a8d6fdb862bcb100c4314d0d9579e5cd73fa6df31a2e6f6e1acd3c5f1207" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -123,7 +140,7 @@ version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cexpr", "clang-sys", "lazy_static", @@ -137,6 +154,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bitflags" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" + [[package]] name = "bitflags" version = "1.3.2" @@ -225,7 +248,7 @@ version = "0.15.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c76ee391b03d35510d9fa917357c7f1855bd9a6659c95a1b392e33f49b3369bc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cairo-sys-rs", "glib", "libc", @@ -362,7 +385,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f425db7937052c684daec3bd6375c8abe2d146dca4b8b143d6db777c39138f3a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "block", "cocoa-foundation", "core-foundation", @@ -378,7 +401,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "931d3837c286f56e3c58423ce4eba12d08db2374461a785c86f672b08b5650d6" dependencies = [ - "bitflags", + "bitflags 1.3.2", "block", "core-foundation", "core-graphics-types", @@ -437,7 +460,7 @@ version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-graphics-types", "foreign-types", @@ -450,7 +473,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a68b68b3446082644c91ac778bf50cd4104bfb002b5a6a7c44cca5a2c70788b" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "foreign-types", "libc", @@ -462,7 +485,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb17e2d1795b1996419648915df94bc7103c28f7b48062d7acf4652fc371b2ff" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation-sys 0.6.2", "coreaudio-sys", ] @@ -901,7 +924,7 @@ version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6e05c1f572ab0e1f15be94217f0dc29088c248b14f792a5ff0af0d84bcda9e8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cairo-rs", "gdk-pixbuf", "gdk-sys", @@ -917,7 +940,7 @@ version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad38dd9cc8b099cceecdf41375bb6d481b1b5a7cd5cd603e10a69a9383f8619a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "gdk-pixbuf-sys", "gio", "glib", @@ -1018,7 +1041,7 @@ version = "0.15.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68fdbc90312d462781a395f7a16d96a2b379bb6ef8cd6310a2df272771c4283b" dependencies = [ - "bitflags", + "bitflags 1.3.2", "futures-channel", "futures-core", "futures-io", @@ -1048,7 +1071,7 @@ version = "0.15.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb0306fbad0ab5428b0ca674a23893db909a98582969c9b537be4ced78c505d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "futures-channel", "futures-core", "futures-executor", @@ -1124,7 +1147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e3004a2d5d6d8b5057d2b57b3712c9529b62e82c77f25c1fecde1fd5c23bd0" dependencies = [ "atk", - "bitflags", + "bitflags 1.3.2", "cairo-rs", "field-offset", "futures-channel", @@ -1361,14 +1384,17 @@ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "jarvis-app" -version = "0.0.1" +version = "0.0.2" dependencies = [ + "arc-swap", + "atomic_enum", "hound", "lazy_static", "log", "once_cell", "peak_alloc", "pickledb", + "portaudio", "pv_porcupine", "pv_recorder", "rand 0.8.5", @@ -1391,7 +1417,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf053e7843f2812ff03ef5afe34bb9c06ffee120385caad4f6b9967fcd37d41c" dependencies = [ - "bitflags", + "bitflags 1.3.2", "glib", "javascriptcore-rs-sys", ] @@ -1670,7 +1696,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2032c77e030ddee34a6787a64166008da93f6a352b629261d0fee232b8742dd4" dependencies = [ - "bitflags", + "bitflags 1.3.2", "jni-sys", "ndk-sys 0.3.0", "num_enum", @@ -1683,7 +1709,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "jni-sys", "ndk-sys 0.4.1+23.1.7779620", "num_enum", @@ -1727,7 +1753,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", ] @@ -1758,6 +1784,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" +dependencies = [ + "num-integer", + "num-iter", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.3" @@ -1788,6 +1825,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.1" @@ -1938,7 +1986,7 @@ version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22e4045548659aee5313bde6c582b0d83a627b7904dd20dc2d9ef0895d414e4f" dependencies = [ - "bitflags", + "bitflags 1.3.2", "glib", "libc", "once_cell", @@ -2156,13 +2204,25 @@ version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aaeebc51f9e7d2c150d3f3bfeb667f2aa985db5ef1e3d212847bdedb488beeaa" dependencies = [ - "bitflags", + "bitflags 1.3.2", "crc32fast", "fdeflate", "flate2", "miniz_oxide 0.7.1", ] +[[package]] +name = "portaudio" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d289315f6155a7608b6d8757786c79ed2243afeab8a5eda8989effda3fdc5c3" +dependencies = [ + "bitflags 0.7.0", + "libc", + "num", + "pkg-config", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2379,7 +2439,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2388,7 +2448,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2519,7 +2579,7 @@ version = "0.37.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -2585,7 +2645,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cssparser", "derive_more", "fxhash", @@ -2802,7 +2862,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2b4d76501d8ba387cf0fefbe055c3e0a59891d09f0f995ae4e4b16f6b60f3c0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "gio", "glib", "libc", @@ -2816,7 +2876,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "009ef427103fcb17f802871647a7fa6c60cbb654b4c4e4c0ac60a31c5f6dc9cf" dependencies = [ - "bitflags", + "bitflags 1.3.2", "gio-sys", "glib-sys", "gobject-sys", @@ -2895,7 +2955,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55a0846e7a2c9a8081ff799fc83a975170417ad2a143f644a77ec2e3e82a2b73" dependencies = [ - "bitflags", + "bitflags 1.3.2", "lazy_static", "log", "symphonia-core", @@ -2909,7 +2969,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b9567e2d8a5f866b2f94f5d366d811e0c6826babcff6d37de9e1a6690d38869" dependencies = [ "arrayvec", - "bitflags", + "bitflags 1.3.2", "bytemuck", "lazy_static", "log", @@ -2995,7 +3055,7 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac8e6399427c8494f9849b58694754d7cc741293348a6836b6c8d2c5aa82d8e6" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cairo-rs", "cc", "cocoa", @@ -3691,7 +3751,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8f859735e4a452aeb28c6c56a852967a8a76c8eb1cc32dbf931ad28a13d6370" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cairo-rs", "gdk", "gdk-sys", @@ -3716,7 +3776,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d76ca6ecc47aeba01ec61e480139dda143796abcae6f83bcddf50d6b5b1dcf3" dependencies = [ "atk-sys", - "bitflags", + "bitflags 1.3.2", "cairo-sys-rs", "gdk-pixbuf-sys", "gdk-sys", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 3bf2065..d3dfc63 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jarvis-app" -version = "0.0.1" +version = "0.0.2" description = "Jarvis Voice Assistant" authors = ["Abraham Tugalov"] license = "GPL-3.0-only" @@ -36,6 +36,9 @@ rustpotter = "2.0.0" simple-logging = "2.0.2" log = "0.4.17" once_cell = "1.17.1" +arc-swap = "1.6.0" +atomic_enum = "0.2.0" +portaudio = "0.7.0" [features] # this feature is used for production builds or when `devPath` points to the filesystem diff --git a/src-tauri/rustpotter/___default.rpw b/src-tauri/rustpotter/___default.rpw new file mode 100644 index 0000000..7998694 Binary files /dev/null and b/src-tauri/rustpotter/___default.rpw differ diff --git a/src-tauri/rustpotter/jarvis-community-1.rpw b/src-tauri/rustpotter/jarvis-community-1.rpw new file mode 100644 index 0000000..07874a5 Binary files /dev/null and b/src-tauri/rustpotter/jarvis-community-1.rpw differ diff --git a/src-tauri/rustpotter/jarvis-community-2.rpw b/src-tauri/rustpotter/jarvis-community-2.rpw new file mode 100644 index 0000000..be4df0b Binary files /dev/null and b/src-tauri/rustpotter/jarvis-community-2.rpw differ diff --git a/src-tauri/rustpotter/jarvis-community-3.rpw b/src-tauri/rustpotter/jarvis-community-3.rpw new file mode 100644 index 0000000..43b4dea Binary files /dev/null and b/src-tauri/rustpotter/jarvis-community-3.rpw differ diff --git a/src-tauri/rustpotter/jarvis-community-4.rpw b/src-tauri/rustpotter/jarvis-community-4.rpw new file mode 100644 index 0000000..0a0341a Binary files /dev/null and b/src-tauri/rustpotter/jarvis-community-4.rpw differ diff --git a/src-tauri/rustpotter/jarvis-community-5.rpw b/src-tauri/rustpotter/jarvis-community-5.rpw new file mode 100644 index 0000000..e5d2a1f Binary files /dev/null and b/src-tauri/rustpotter/jarvis-community-5.rpw differ diff --git a/src-tauri/rustpotter/jarvis-default.rpw b/src-tauri/rustpotter/jarvis-default.rpw new file mode 100644 index 0000000..47dea84 Binary files /dev/null and b/src-tauri/rustpotter/jarvis-default.rpw differ diff --git a/src-tauri/src/.cargo/config.toml b/src-tauri/src/.cargo/config.toml new file mode 100644 index 0000000..02759fd --- /dev/null +++ b/src-tauri/src/.cargo/config.toml @@ -0,0 +1,2 @@ +[env] +PORTAUDIO_ONLY_STATIC = true \ No newline at end of file diff --git a/src-tauri/src/assistant_commands.rs b/src-tauri/src/assistant_commands.rs index 0a44135..e9550ce 100644 --- a/src-tauri/src/assistant_commands.rs +++ b/src-tauri/src/assistant_commands.rs @@ -3,6 +3,7 @@ use seqdiff::ratio; use serde_yaml; use std::path::Path; use std::{fs, fs::File}; +use log::{info, warn, error}; use core::time::Duration; use std::path::PathBuf; @@ -36,7 +37,7 @@ pub fn parse_commands() -> Result, String> { if let Ok(parse_result) = serde_yaml::from_reader::(cc_reader) { cc_yaml = parse_result; } else { - println!("Can't parse {}, skipping ...", &cc_file.display()); + warn!("Can't parse {}, skipping ...", &cc_file.display()); continue; // return Err(format!("Can't parse {}", &cc_file.display())); } @@ -52,9 +53,11 @@ pub fn parse_commands() -> Result, String> { if commands.len() > 0 { Ok(commands) } else { + error!("No commands were found"); Err("No commands were found".into()) } } else { + error!("Error reading commands directory"); return Err("Error reading commands directory".into()); } } @@ -95,6 +98,7 @@ pub fn fetch_command<'a>( if let Some((cmd_path, scmd)) = result_scmd { println!("Ratio is: {}", current_max_ratio); + info!("CMD is: {cmd_path:?}, SCMD is: {scmd:?}, Ratio is: {}", current_max_ratio); Some((&cmd_path, &scmd)) } else { None @@ -144,6 +148,7 @@ pub fn execute_command( Ok(()) } else { + error!("AHK process spawn error (does exe path is valid?)"); Err("AHK process spawn error (does exe path is valid?)".into()) } } @@ -169,6 +174,7 @@ pub fn execute_command( Ok(()) } else { + error!("Shell process spawn error (does cli command is valid?)"); Err("Shell process spawn error (does cli command is valid?)".into()) } } @@ -184,6 +190,9 @@ pub fn execute_command( std::thread::sleep(Duration::from_secs(2)); std::process::exit(0); } - _ => Err("Command type unknown".into()), + _ => { + error!("Command type unknown"); + Err("Command type unknown".into()) + }, } } diff --git a/src-tauri/src/config.rs b/src-tauri/src/config.rs index 3ecb824..f50bef8 100644 --- a/src-tauri/src/config.rs +++ b/src-tauri/src/config.rs @@ -7,7 +7,15 @@ // "./public" // }; -pub const WAKE_WORD_ENGINES: [&str; 2] = ["rustpotter", "picovoice"]; +// APP +// pub const WAKE_WORD_ENGINES: [&str; 3] = ["rustpotter", "vosk", "picovoice"]; +pub enum WakeWordEngine { + Rustpotter, + Vosk, + Porcupine +} + +pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter; pub const DB_FILE_NAME: &str = "app.db"; pub const LOG_FILE_NAME: &str = "log.txt"; @@ -15,12 +23,20 @@ pub const APP_VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION"); pub const AUTHOR_NAME: Option<&str> = option_env!("CARGO_PKG_AUTHORS"); pub const REPOSITORY_LINK: Option<&str> = option_env!("CARGO_PKG_REPOSITORY"); +// RUSPOTTER +pub const RUSPOTTER_MIN_SCORE: f32 = 0.64; + +// PICOVOICE pub const COMMANDS_PATH: &str = "commands/"; pub const KEYWORDS_PATH: &str = "picovoice/keywords/"; +// VOSK // pub const VOSK_MODEL_PATH: &str = const_concat!(PUBLIC_PATH, "/vosk/model_small"); +pub const VOSK_FETCH_PHRASE: &str = "джарвис"; pub const VOSK_MODEL_PATH: &str = "vosk/model_small"; +pub const VOSK_MIN_RATIO: f64 = 70.0; +// ETC pub const CMD_RATIO_THRESHOLD: f64 = 60f64; pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(10); diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index 4f66a52..80d5fe9 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -33,6 +33,11 @@ lazy_static! { static ref APP_CONFIG_DIR: Mutex = Mutex::new(String::new()); } +// data dir +lazy_static! { + static ref APP_LOG_DIR: Mutex = Mutex::new(String::new()); +} + // init PickleDb connection lazy_static! { static ref DB: Mutex = Mutex::new( @@ -58,9 +63,6 @@ lazy_static! { } fn main() { - // log to file - simple_logging::log_to_file(config::LOG_FILE_NAME, LevelFilter::max()).expect("Failed to start logger ... is directory writable?"); - // init vosk vosk::init_vosk(); @@ -70,6 +72,14 @@ fn main() { std::fs::create_dir_all(app.path_resolver().app_config_dir().unwrap())?; APP_CONFIG_DIR.lock().unwrap().push_str(app.path_resolver().app_config_dir().unwrap().to_str().unwrap()); + std::fs::create_dir_all(app.path_resolver().app_log_dir().unwrap())?; + APP_LOG_DIR.lock().unwrap().push_str(app.path_resolver().app_log_dir().unwrap().to_str().unwrap()); + + // log to file + let log_file_path = format!("{}/{}", APP_LOG_DIR.lock().unwrap(), config::LOG_FILE_NAME); + println!("!!!===============!!!\nLOGGING TO {}\n!!!===============!!!\n", &log_file_path); + simple_logging::log_to_file(log_file_path, LevelFilter::max()).expect("Failed to start logger ... is directory writable?"); + Ok(()) }) .invoke_handler(tauri::generate_handler![ diff --git a/src-tauri/src/recorder.rs b/src-tauri/src/recorder.rs index f8fad40..502bb87 100644 --- a/src-tauri/src/recorder.rs +++ b/src-tauri/src/recorder.rs @@ -1,52 +1,127 @@ -use once_cell::sync::OnceCell; -use std::sync::atomic::{AtomicU32, AtomicBool, Ordering}; -use pv_recorder::{Recorder, RecorderBuilder}; -use log::{info}; +// use once_cell::sync::OnceCell; +use std::sync::atomic::{AtomicU32, Ordering}; +use log::{info, warn, error}; +use atomic_enum::atomic_enum; + +mod pvrecorder; +// mod cpal; +// mod portaudio; use crate::DB; +#[atomic_enum] +#[derive(PartialEq)] +pub enum RecorderType { + Cpal, + PvRecorder, + PortAudio +} + +pub static RECORDER_TYPE: AtomicRecorderType = AtomicRecorderType::new(RecorderType::PvRecorder); // use pvrecorder as default pub static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0); -static RECORDER: OnceCell = OnceCell::new(); -pub static IS_RECORDING: AtomicBool = AtomicBool::new(false); -fn init_microphone() { - if RECORDER.get().is_none() { - RECORDER.get_or_init(|| RecorderBuilder::new() - .device_index(get_selected_microphone_index()) - .frame_length(FRAME_LENGTH.load(Ordering::SeqCst) as i32) - .init() - .expect("Failed to initialize pvrecorder")); - info!("Microphone recorder initialized!") +pub fn init() { + match RECORDER_TYPE.load(Ordering::SeqCst) { + RecorderType::PvRecorder => { + // Init Pv Recorder + info!("Initializing Pv Recorder audio backend."); + match pvrecorder::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) { + false => { + // Switch to CPAL recorder + warn!("Pv Recorder audio backend failed."); + // RECORDER_TYPE.store(RecorderType::PortAudio, Ordering::SeqCst); + + // init again + init(); + }, + _ => () + } + }, + RecorderType::PortAudio => { + // Init PortAudio + info!("Initializing PortAudio audio backend"); + todo!(); + // match portaudio::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) { + // false => { + // // Switch to PortAudio recorder + // error!("PortAudio audio backend failed."); + // }, + // _ => () + // } + }, + RecorderType::Cpal => { + // Init CPAL + info!("Initializing CPAL audio backend"); + todo!(); + // match cpal::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) { + // false => { + // // Switch to CPAL recorder + // error!("CPAL audio backend failed."); + // }, + // _ => () + // } + } } } pub fn read_microphone(frame_buffer: &mut [i16]) { - // ensure microphone is initialized - init_microphone(); - - // read to frame buffer - RECORDER.get().unwrap().read(frame_buffer).expect("Failed to read audio frame"); + match RECORDER_TYPE.load(Ordering::SeqCst) { + RecorderType::PvRecorder => { + pvrecorder::read_microphone(frame_buffer); + }, + RecorderType::PortAudio => { + todo!(); + // portaudio::read_microphone(frame_buffer); + }, + RecorderType::Cpal => { + // cpal::read_microphone(frame_buffer); + panic!("Cpal should be used via callback assignment"); + } + } } pub fn start_recording() { - // ensure microphone is initialized - init_microphone(); - - RECORDER.get().unwrap().start().expect("Failed to start audio recording!"); - IS_RECORDING.store(true, Ordering::SeqCst); - info!("START recording from microphone ..."); + match RECORDER_TYPE.load(Ordering::SeqCst) { + RecorderType::PvRecorder => { + pvrecorder::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)); + }, + RecorderType::PortAudio => { + todo!(); + // portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)); + }, + RecorderType::Cpal => { + // cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)); + } + } } pub fn stop_recording() { - // ensure microphone is initialized - init_microphone(); - - RECORDER.get().unwrap().start().expect("Failed to start audio recording!"); - IS_RECORDING.store(false, Ordering::SeqCst); - info!("STOP recording from microphone ..."); + match RECORDER_TYPE.load(Ordering::SeqCst) { + RecorderType::PvRecorder => { + pvrecorder::stop_recording(); + }, + RecorderType::PortAudio => { + todo!(); + // portaudio::stop_recording(); + }, + RecorderType::Cpal => { + // cpal::stop_recording(); + } + } } +// pub fn update_selected_microphone_index() -> i32 { +// let selected_microphone: i32 = get_selected_microphone_index(); + +// // store current microphone idx +// SELECTED_MICROPHONE_IDX.store(selected_microphone, Ordering::SeqCst); + +// // return microphone index +// info!("Selected microphone index = {selected_microphone}"); +// selected_microphone +// } + pub fn get_selected_microphone_index() -> i32 { let selected_microphone: i32; @@ -57,7 +132,5 @@ pub fn get_selected_microphone_index() -> i32 { selected_microphone = -1; } - // return microphone index - info!("Selected microphone index = {selected_microphone}"); selected_microphone } \ No newline at end of file diff --git a/src-tauri/src/recorder/cpal.rs b/src-tauri/src/recorder/cpal.rs new file mode 100644 index 0000000..27840fe --- /dev/null +++ b/src-tauri/src/recorder/cpal.rs @@ -0,0 +1,186 @@ +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{BufferSize, StreamConfig, SampleRate, Host, Device, Stream, SampleFormat}; +use log::{info, warn, error}; + +use once_cell::sync::OnceCell; +use std::sync::Arc; +use arc_swap::ArcSwap; +use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; + +use crate::tauri_commands::cpal_data_callback; + +static HOST: OnceCell = OnceCell::new(); +thread_local!(static RECORDER: OnceCell> = OnceCell::new()); +static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0); +static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0); +static IS_RECORDING: AtomicBool = AtomicBool::new(false); + +pub fn init_microphone(device_index: i32, frame_length: u32) -> bool { + // init host & frame buffer for the callback + if HOST.get().is_none() { + HOST.set(cpal::default_host()); + + // FRAME_BUFFER.set(Mutex::new(vec![0; FRAME_LENGTH.load(Ordering::SeqCst) as usize])); + } + + // init microphone + RECORDER.with(|recorder| { + match recorder.get().is_none() { + true => { + if let Some(device) = get_device(device_index as usize) { + // store + recorder.set(ArcSwap::from_pointee(create_stream(device, frame_length))); + + // remember current configuration + SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst); + FRAME_LENGTH.store(frame_length, Ordering::SeqCst); + + // success + true + } else { + false + } + }, + false => { + // check if re-initialization required (i.e. selecetd microphoneor frame-length was changed ) + if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index + || + FRAME_LENGTH.load(Ordering::SeqCst) != frame_length { + warn!("Selected microphone or frame length was changed, re-initializing ..."); + // initialize again with new device index + if IS_RECORDING.load(Ordering::SeqCst) { + stop_recording(); + } + + // remember new configuration + SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst); + FRAME_LENGTH.store(frame_length, Ordering::SeqCst); + + if let Some(device) = get_device(device_index as usize) { + // store + recorder.get().unwrap().store(Arc::new(create_stream(device, frame_length))); + + // success + return true + } else { + return false + } + } + + // success + true + } + } + }) +} + +fn create_stream(device: Device, frame_length: u32) -> Stream { + // get default input stream config + // let default_config = device.default_input_config().unwrap(); + + // create config for the stream + // let config: StreamConfig = StreamConfig { + // channels: default_config.channels(), + // sample_rate: SampleRate(16000), + // buffer_size: BufferSize::Fixed(frame_length) + // }; + + let config = device + .default_input_config() + .expect("Failed to load default input config"); + + let channels = config.channels(); + + let err_fn = move |err| { + eprintln!("an error occurred on stream: {}", err); + }; + + match config.sample_format() { + SampleFormat::F32 => device.build_input_stream( + &config.into(), + move |data: &[f32], info| { + cpal_data_callback(data, channels); + }, + err_fn, + None + ), + SampleFormat::U16 => device.build_input_stream( + &config.into(), + move |data: &[u16], info| { + cpal_data_callback(data, channels); + }, + err_fn, + None + ), + SampleFormat::I16 => device.build_input_stream( + &config.into(), + move |data: &[i16], info| { + cpal_data_callback(data, channels); + }, + err_fn, + None + ), + _ => todo!() + }.unwrap() +} + +pub fn stereo_to_mono(input_data: &[i16]) -> Vec { + let mut result = Vec::with_capacity(input_data.len() / 2); + result.extend( + input_data + .chunks_exact(2) + .map(|chunk| chunk[0] / 2 + chunk[1] / 2), + ); + + result +} + +fn get_device(device_index: usize) -> Option { + if let Some(device) = HOST.get().unwrap().input_devices().expect("Get devices error ...").nth(device_index) { + Some(device) + } else { + if let Some(default) = HOST.get().unwrap().default_input_device() { + Some(default) + } else { + error!("No default input device ..."); + + None + } + } +} + +pub fn start_recording(device_index: i32, frame_length: u32) { + // ensure microphone is initialized + init_microphone(device_index, frame_length); + + // start recording + RECORDER.with(|recorder| { + match recorder.get().unwrap().load().play() { + Err(msg) => { + error!("[CPAL] Audio stream PLAY error ... {:?}", msg); + }, + _ => () + }; + + IS_RECORDING.store(true, Ordering::SeqCst); + info!("START recording from microphone ..."); + }); +} + +pub fn stop_recording() { + // ensure microphone is initialized + RECORDER.with(|recorder| { + if !recorder.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) { + // pause instead of stop + match recorder.get().unwrap().load().pause() { + Err(msg) => { + error!("[CPAL] Audio stream PAUSE error ... {:?}", msg); + }, + _ => () + }; + + IS_RECORDING.store(false, Ordering::SeqCst); + info!("STOP recording from microphone ..."); + } + }); +} \ No newline at end of file diff --git a/src-tauri/src/recorder/portaudio.rs b/src-tauri/src/recorder/portaudio.rs new file mode 100644 index 0000000..e69a374 --- /dev/null +++ b/src-tauri/src/recorder/portaudio.rs @@ -0,0 +1,201 @@ +use portaudio as pa; +use pa::{DeviceIndex, Stream}; +use log::{info, warn, error}; + +use once_cell::sync::OnceCell; +use std::sync::{Arc, Mutex}; +use arc_swap::ArcSwap; +use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; + +thread_local!(static RECORDER: OnceCell, pa::Input>>>> = OnceCell::new()); +static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0); +static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0); +static IS_RECORDING: AtomicBool = AtomicBool::new(false); + +const CHANNELS: i32 = 1; +const SAMPLE_RATE: f64 = 16_000.0; + +pub fn init_microphone(device_index: i32, frame_length: u32) -> bool { + RECORDER.with(|r| { + match r.get().is_none() { + true => { + match create_stream(device_index, frame_length) { + Ok(stream) => { + // store + r.set(ArcSwap::from_pointee(Mutex::new(stream))); + + // remember current configuration + SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst); + FRAME_LENGTH.store(frame_length, Ordering::SeqCst); + + // success + true + }, + Err(msg) => { + error!("Failed to initialize portaudio.\nError details: {:?}", msg); + + // fail + false + } + } + }, + _ => { + // check if re-initialization required (i.e. selecetd microphoneor frame-length was changed ) + if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index + || + FRAME_LENGTH.load(Ordering::SeqCst) != frame_length { + warn!("Selected microphone or frame length was changed, re-initializing ..."); + // initialize again with new device index + if IS_RECORDING.load(Ordering::SeqCst) { + // RECORDER.get().unwrap().load().stop().expect("Failed to start audio recording!"); + stop_recording(); + } + + // store + match create_stream(device_index, frame_length) { + Ok(stream) => { + // store new stream + r.get().unwrap().store(Arc::new(Mutex::new(stream))); + + // remember new configuration + SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst); + FRAME_LENGTH.store(frame_length, Ordering::SeqCst); + + // success + return true + }, + Err(msg) => { + error!("Failed to initialize portaudio.\nError details: {:?}", msg); + + // fail + return false + } + } + } + + // success + true + } + } + }) +} + +fn create_stream(device_index: i32, frame_length: u32) -> Result, pa::Input>, pa::Error> { + let pa_recorder: Result = pa::PortAudio::new(); + + match pa_recorder { + Ok(pa) => { + let input_settings = match get_input_settings(DeviceIndex(device_index as u32), &pa, SAMPLE_RATE, frame_length, CHANNELS) { + Ok(settings) => settings, + Err(error) => panic!("{}", String::from(error)) + }; + + // Construct a stream with input and output sample types of i16 + match pa.open_blocking_stream(input_settings) { + Ok(strm) => Ok(strm), + Err(error) => panic!("{}", error.to_string()), + } + }, + Err(msg) => Err(msg) + } +} + +fn get_input_latency(audio_port: &pa::PortAudio, input_index: pa::DeviceIndex) -> Result +{ + let input_device_information = audio_port.device_info(input_index).or_else(|error| Err(String::from(format!("{}", error)))); + Ok(input_device_information.unwrap().default_low_input_latency) +} + +fn get_input_stream_parameters(input_index: pa::DeviceIndex, latency: f64, channels: i32) -> Result, String> +{ + const INTERLEAVED: bool = true; + Ok(pa::StreamParameters::::new(input_index, channels, INTERLEAVED, latency)) +} + +fn get_input_settings(input_index: pa::DeviceIndex, audio_port: &pa::PortAudio, sample_rate: f64, frames: u32, channels: i32) -> Result, String> +{ + Ok( + pa::InputStreamSettings::new( + (get_input_stream_parameters( + input_index, + (get_input_latency( + &audio_port, + input_index, + ))?, + channels + ))?, + sample_rate, + frames, + ) + ) +} + +// We'll use this function to wait for read/write availability. +fn wait_for_stream(f: F, name: &str) -> u32 +where + F: Fn() -> Result, +{ + loop { + match f() { + Ok(available) => match available { + pa::StreamAvailable::Frames(frames) => return frames as u32, + pa::StreamAvailable::InputOverflowed => println!("Input stream has overflowed"), + pa::StreamAvailable::OutputUnderflowed => { + println!("Output stream has underflowed") + } + }, + Err(err) => panic!( + "An error occurred while waiting for the {} stream: {}", + name, err + ), + } + } +} + +pub fn read_microphone(frame_buffer: &mut [i16]) { + // ensure microphone is initialized + RECORDER.with(|r| { + if !r.get().is_none() { + let cell = r.get().unwrap().load(); + let mut lock = cell.lock(); + let stream = lock.as_mut().unwrap(); + + // read to frame buffer + let in_frames = wait_for_stream(|| stream.read_available(), "Read"); + + if in_frames > 0 { + // let input_samples = stream.read(in_frames).expect("Cannot read frames ..."); + // println!("Read {:?} frames from the input stream.", in_frames); + + let input_samples = stream.read(in_frames).expect("Cannot read frames ..."); + println!("Read: {} (required {})", input_samples.len(), frame_buffer.len()); + frame_buffer.copy_from_slice(input_samples.chunks(frame_buffer.len()).last().unwrap()); + } + // r.get().unwrap().load().read(frame_buffer).expect("Failed to read audio frame"); + } + }); +} + +pub fn start_recording(device_index: i32, frame_length: u32) { + // ensure microphone is initialized + init_microphone(device_index, frame_length); + + // start recording + RECORDER.with(|r| { + r.get().unwrap().load().lock().unwrap().start().expect("Failed to start audio recording!"); + IS_RECORDING.store(true, Ordering::SeqCst); + info!("START recording from microphone ..."); + }); +} + +pub fn stop_recording() { + RECORDER.with(|r| { + if !r.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) { + // stop recording + let pa = r.get().unwrap().load(); + r.get().unwrap().load().lock().unwrap().stop().expect("Failed to stop audio recording!"); + IS_RECORDING.store(false, Ordering::SeqCst); + info!("STOP recording from microphone ..."); + } + }); +} \ No newline at end of file diff --git a/src-tauri/src/recorder/pvrecorder.rs b/src-tauri/src/recorder/pvrecorder.rs new file mode 100644 index 0000000..183a7d2 --- /dev/null +++ b/src-tauri/src/recorder/pvrecorder.rs @@ -0,0 +1,105 @@ +use pv_recorder::{Recorder, RecorderBuilder}; +use log::{info, warn, error}; + +use once_cell::sync::OnceCell; +use std::sync::Arc; +use arc_swap::ArcSwap; +use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; + +static RECORDER: OnceCell> = OnceCell::new(); +static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0); +static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0); +static IS_RECORDING: AtomicBool = AtomicBool::new(false); + +pub fn init_microphone(device_index: i32, frame_length: u32) -> bool { + match RECORDER.get().is_none() { + true => { + let pv_recorder = RecorderBuilder::new() + .device_index(device_index) + .frame_length(frame_length as i32) + .init(); + + match pv_recorder { + Ok(pv) => { + // store + RECORDER.set(ArcSwap::from_pointee(pv)); + + // remember current configuration + SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst); + FRAME_LENGTH.store(frame_length, Ordering::SeqCst); + + // success + true + }, + Err(msg) => { + error!("Failed to initialize pvrecorder.\nError details: {:?}", msg); + + // fail + false + } + } + }, + _ => { + // check if re-initialization required (i.e. selecetd microphoneor frame-length was changed ) + if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index + || + RECORDER.get().unwrap().load().frame_length() != frame_length as usize { + warn!("Selected microphone or frame length was changed, re-initializing ..."); + // initialize again with new device index + if IS_RECORDING.load(Ordering::SeqCst) { + // RECORDER.get().unwrap().load().stop().expect("Failed to start audio recording!"); + stop_recording(); + } + + // remember new configuration + SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst); + FRAME_LENGTH.store(frame_length, Ordering::SeqCst); + + // store + RECORDER.get().unwrap().store(Arc::new(RecorderBuilder::new() + .device_index(device_index) + .frame_length(frame_length as i32) + .init() + .expect("Failed to initialize pvrecorder"))); + } + + // success + true + } + } +} + +pub fn read_microphone(frame_buffer: &mut [i16]) { + // ensure microphone is initialized + if !RECORDER.get().is_none() { + // read to frame buffer + match RECORDER.get().unwrap().load().read(frame_buffer) { + Err(msg) => { + // @TODO: Fix somehow. PvRecorder always wait for PCM buffer size of 512. + // error!("Failed to read audio frame. {:?}", msg); + // eprintln!("Failed to read audio frame. {:?}", msg); + }, + _ => () + } + } +} + +pub fn start_recording(device_index: i32, frame_length: u32) { + // ensure microphone is initialized + init_microphone(device_index, frame_length); + + // start recording + RECORDER.get().unwrap().load().start().expect("Failed to start audio recording!"); + IS_RECORDING.store(true, Ordering::SeqCst); + info!("START recording from microphone ..."); +} + +pub fn stop_recording() { + // ensure microphone is initialized + if !RECORDER.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) { + // stop recording + RECORDER.get().unwrap().load().stop().expect("Failed to stop audio recording!"); + IS_RECORDING.store(false, Ordering::SeqCst); + info!("STOP recording from microphone ..."); + } +} \ No newline at end of file diff --git a/src-tauri/src/tauri_commands.rs b/src-tauri/src/tauri_commands.rs index 398b0bb..250928d 100644 --- a/src-tauri/src/tauri_commands.rs +++ b/src-tauri/src/tauri_commands.rs @@ -3,8 +3,8 @@ mod db; pub use db::*; // import RECORDER commands -mod recorder; -pub use recorder::*; +mod audio; +pub use audio::*; // import PORCUPINE commands mod listener; diff --git a/src-tauri/src/tauri_commands/recorder.rs b/src-tauri/src/tauri_commands/audio.rs similarity index 100% rename from src-tauri/src/tauri_commands/recorder.rs rename to src-tauri/src/tauri_commands/audio.rs diff --git a/src-tauri/src/tauri_commands/listener.rs b/src-tauri/src/tauri_commands/listener.rs index c535d56..5367780 100644 --- a/src-tauri/src/tauri_commands/listener.rs +++ b/src-tauri/src/tauri_commands/listener.rs @@ -2,19 +2,23 @@ use porcupine::{Porcupine, PorcupineBuilder}; use std::sync::atomic::{AtomicBool, Ordering}; use std::path::Path; use log::{info, warn, error}; +use rustpotter::{Rustpotter, RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig}; +// use dasp::{sample::ToSample, Sample}; // use crate::events::Payload; use tauri::Manager; use rand::seq::SliceRandom; use std::time::SystemTime; +use once_cell::sync::OnceCell; +use std::sync::Mutex; use crate::assistant_commands; use crate::events; use crate::config; use crate::vosk; -use crate::recorder; +use crate::recorder::{self, FRAME_LENGTH}; use crate::COMMANDS; use crate::DB; @@ -25,6 +29,15 @@ static LISTENING: AtomicBool = AtomicBool::new(false); // stop listening with Atomic flag (to make it work between different threads) static STOP_LISTENING: AtomicBool = AtomicBool::new(false); +// store tauri app_handle +static TAURI_APP_HANDLE: OnceCell = OnceCell::new(); + +// store porcupine instance +static PORCUPINE: OnceCell = OnceCell::new(); + +// store rustpotter instance +static RUSTPOTTER: OnceCell> = OnceCell::new(); + #[tauri::command] pub fn is_listening() -> bool { LISTENING.load(Ordering::SeqCst) @@ -34,12 +47,31 @@ pub fn is_listening() -> bool { pub fn stop_listening() { if is_listening() { STOP_LISTENING.store(true, Ordering::SeqCst); + stop_recording(); } // wait until listening stops while is_listening() {} } +fn get_wake_word_engine() -> config::WakeWordEngine { + let selected_wake_word_engine; + if let Some(wwengine) = DB.lock().unwrap().get::("selected_wake_word_engine") { + // from db + match wwengine.trim().to_lowercase().as_str() { + "rustpotter" => selected_wake_word_engine = config::WakeWordEngine::Rustpotter, + "vosk" => selected_wake_word_engine = config::WakeWordEngine::Vosk, + "picovoice" => selected_wake_word_engine = config::WakeWordEngine::Porcupine, + &_ => todo!() + } + } else { + // default + selected_wake_word_engine = config::DEFAULT_WAKE_WORD_ENGINE; // set default wake_word engine + } + + selected_wake_word_engine +} + #[tauri::command(async)] pub fn start_listening(app_handle: tauri::AppHandle) -> Result { // only one listener thread is allowed @@ -47,44 +79,29 @@ pub fn start_listening(app_handle: tauri::AppHandle) -> Result { return Err("Already listening.".into()); } - // Retrieve selected wake-word engine from DB - let selected_wake_word_engine; - if let Some(wwengine) = DB.lock().unwrap().get::("selected_wake_word_engine") { - // from db - selected_wake_word_engine = wwengine; - } else { - // default - selected_wake_word_engine = config::WAKE_WORD_ENGINES.first().expect("No wake-word engines found ...").to_string(); // set default wake_word engine + // keep app handle + if TAURI_APP_HANDLE.get().is_none() { + TAURI_APP_HANDLE.set(app_handle); } // call selected wake-word engine listener command - match selected_wake_word_engine.as_str() { - "rustpotter" => { - info!("Starting rustpotter wake-word engine ..."); - return picovoice_listen(&app_handle, |_app| { - // Greet user - events::play("run", &app_handle); - }, |app, kidx| keyword_callback(app, kidx)); + match get_wake_word_engine() { + config::WakeWordEngine::Rustpotter => { + info!("Starting RUSTPOTTER wake-word engine ..."); + return rustpotter_init(); }, - "vosk" => { - info!("Starting vosk wake-word engine ..."); - return vosk_listen(&app_handle, |_app| { - // Greet user - events::play("run", &app_handle); - }, |app, kidx| keyword_callback(app, kidx)); + config::WakeWordEngine::Vosk => { + info!("Starting VOSK wake-word engine ..."); + return vosk_init(); }, - "picovoice" => { - info!("Starting picovoice wake-word engine ..."); - return picovoice_listen(&app_handle, |_app| { - // Greet user - events::play("run", &app_handle); - }, |app, kidx| keyword_callback(app, kidx)); - }, - _ => Err("No wake-word engine selected ...".into()) + config::WakeWordEngine::Porcupine => { + info!("Starting PICOVOICE PORCUPINE wake-word engine ..."); + return picovoice_init(); + } } } -pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) { +fn keyword_callback(_keyword_index: i32) { // vars let mut start: SystemTime = SystemTime::now(); let mut frame_buffer = vec![0; recorder::FRAME_LENGTH.load(Ordering::SeqCst) as usize]; @@ -94,11 +111,11 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) { config::ASSISTANT_GREET_PHRASES .choose(&mut rand::thread_rng()) .unwrap(), - &app_handle, + TAURI_APP_HANDLE.get().unwrap(), ); // emit assistant greet event - app_handle + TAURI_APP_HANDLE.get().unwrap() .emit_all(events::EventTypes::AssistantGreet.get(), ()) .unwrap(); @@ -128,7 +145,7 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) { let cmd_result = assistant_commands::execute_command( &cmd_path, &cmd_config, - &app_handle, + TAURI_APP_HANDLE.get().unwrap(), ); match cmd_result { @@ -142,7 +159,7 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) { } } - app_handle + TAURI_APP_HANDLE.get().unwrap() .emit_all(events::EventTypes::AssistantWaiting.get(), ()) .unwrap(); break; // return to picovoice after command execution (no matter successfull or not) @@ -153,7 +170,7 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) { match start.elapsed() { Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => { // return to picovoice after N seconds - app_handle + TAURI_APP_HANDLE.get().unwrap() .emit_all(events::EventTypes::AssistantWaiting.get(), ()) .unwrap(); break; @@ -163,60 +180,191 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) { } } -pub fn vosk_listen<'s, S, K>(app_handle: &tauri::AppHandle, start_callback: S, mut keyword_callback: K) -> Result - where S: Fn(&tauri::AppHandle), - K: FnMut(&tauri::AppHandle, i32) { +pub fn data_callback(frame_buffer: &[i16]) { + // println!("DATA CALLBACK {}", frame_buffer.len()); + match get_wake_word_engine() { + config::WakeWordEngine::Rustpotter => { + let mut lock = RUSTPOTTER.get().unwrap().lock(); + let rustpotter = lock.as_mut().unwrap(); + let detection = rustpotter.process_i16(&frame_buffer); - // vars - let fetch_phrase = "джарвис".chars().collect::>(); - let frame_length: usize = 128; - let min_ratio: f64 = 0.8; + if let Some(detection) = detection { + if detection.score > config::RUSPOTTER_MIN_SCORE { + info!("Rustpotter detection info:\n{:?}", detection); + keyword_callback(0); + } else { + info!("Rustpotter detection info:\n{:?}", detection); + } + } + }, + config::WakeWordEngine::Vosk => { + // recognize & convert to sequence + let recognized_phrase = vosk::recognize(&frame_buffer, true).unwrap_or("".into()); - // Start recording - let mut frame_buffer = vec![0; frame_length]; - recorder::FRAME_LENGTH.store(frame_length as u32, Ordering::SeqCst); - recorder::start_recording(); - LISTENING.store(true, Ordering::SeqCst); + if !recognized_phrase.trim().is_empty() { + info!("Rec: {}", recognized_phrase); + let recognized_phrases = recognized_phrase.split_whitespace(); + for phrase in recognized_phrases { + let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::>(); + + // compare + let compare_ratio = seqdiff::ratio(&config::VOSK_FETCH_PHRASE.chars().collect::>(), &recognized_phrase_chars); + info!("OG phrase: {:?}", &config::VOSK_FETCH_PHRASE); + info!("Recognized phrase: {:?}", &recognized_phrase_chars); + info!("Compare ratio: {}", compare_ratio); - // run start callback - start_callback(app_handle); - - // Listen until stop flag will be true - while !STOP_LISTENING.load(Ordering::SeqCst) { - recorder::read_microphone(&mut frame_buffer); - - // recognize & convert to sequence - let recognized_phrase = vosk::recognize(&frame_buffer, true).unwrap_or("".into()); - - if !recognized_phrase.trim().is_empty() { - info!("Rec: {}", recognized_phrase); - let recognized_phrases = recognized_phrase.split_whitespace(); - for phrase in recognized_phrases { - let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::>(); - - // compare - if seqdiff::ratio(&fetch_phrase, &recognized_phrase_chars) >= min_ratio { - info!("Phrase: {:?}", &fetch_phrase); - info!("Compare: {:?}", &recognized_phrase_chars); - keyword_callback(&app_handle, 0); - break; + if compare_ratio >= config::VOSK_MIN_RATIO { + info!("Phrase activated."); + keyword_callback(0); + break; + } + } + } + }, + config::WakeWordEngine::Porcupine => { + if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) { + if keyword_index >= 0 { + // println!("Yes, sir! {}", keyword_index); + keyword_callback(keyword_index); } } } } - - // Stop listening - recorder::stop_recording(); - LISTENING.store(false, Ordering::SeqCst); - STOP_LISTENING.store(false, Ordering::SeqCst); - - Ok(true) } -pub fn picovoice_listen<'s, S, K>(app_handle: &tauri::AppHandle, start_callback: S, mut keyword_callback: K) -> Result - where S: Fn(&tauri::AppHandle), - K: FnMut(&tauri::AppHandle, i32) { +fn start_recording() -> Result { + // vars + let frame_length: usize; + // idenfity frame length + match get_wake_word_engine() { + config::WakeWordEngine::Rustpotter => { + // start recording for Rustpotter + // You need a buffer of size `rustpotter.get_samples_per_frame()` when using samples. + // You need a buffer of size `rustpotter.get_bytes_per_frame()` when using bytes. + frame_length = RUSTPOTTER.get().unwrap().lock().unwrap().get_samples_per_frame(); + recorder::FRAME_LENGTH.store(frame_length as u32, Ordering::SeqCst); + }, + config::WakeWordEngine::Vosk => { + // start recording for Vosk + frame_length = 128; + recorder::FRAME_LENGTH.store(frame_length as u32, Ordering::SeqCst); + }, + config::WakeWordEngine::Porcupine => { + // start recording for Porcupine + frame_length = PORCUPINE.get().unwrap().frame_length() as usize; + recorder::FRAME_LENGTH.store(PORCUPINE.get().unwrap().frame_length(), Ordering::SeqCst); + } + } + + // define frame buffer + let mut frame_buffer: Vec = vec![0; frame_length]; + + // init stuff + recorder::init(); // init + recorder::start_recording(); // start + LISTENING.store(true, Ordering::SeqCst); + info!("START listening ..."); + + // greet user + events::play("run", TAURI_APP_HANDLE.get().unwrap()); + + // record + match recorder::RECORDER_TYPE.load(Ordering::SeqCst) { + recorder::RecorderType::PvRecorder => { + while !STOP_LISTENING.load(Ordering::SeqCst) { + recorder::read_microphone(&mut frame_buffer); + data_callback(&frame_buffer); + } + + // stop + stop_recording(); + + Ok(true) + }, + recorder::RecorderType::PortAudio => { + while !STOP_LISTENING.load(Ordering::SeqCst) { + recorder::read_microphone(&mut frame_buffer); + data_callback(&frame_buffer); + } + + // stop + stop_recording(); + + Ok(true) + } + recorder::RecorderType::Cpal => { + todo!() + } + } +} + +fn stop_recording() { + // Stop listening + recorder::stop_recording(); + + LISTENING.store(false, Ordering::SeqCst); + STOP_LISTENING.store(false, Ordering::SeqCst); + info!("STOP listening ..."); +} + +fn rustpotter_init() -> Result { + + // init rustpotter + let rustpotter_config = RustpotterConfig { + fmt: WavFmt::default(), + detector: DetectorConfig { + avg_threshold: 0., + threshold: 0.5, + min_scores: 15, + score_mode: ScoreMode::Max, + comparator_band_size: 5, + comparator_ref: 0.22 + }, + filters: FiltersConfig { + gain_normalizer: GainNormalizationConfig { + enabled: true, + gain_ref: None, + min_gain: 0.5, + max_gain: 1.0, + }, + band_pass: BandPassConfig { + enabled: true, + low_cutoff: 80., + high_cutoff: 400., + } + } + }; + let mut rustpotter = Rustpotter::new(&rustpotter_config).unwrap(); + + // load a wakeword + let rustpotter_wake_word_files: [&str; 5] = [ + "rustpotter/jarvis-default.rpw", + "rustpotter/jarvis-community-1.rpw", + "rustpotter/jarvis-community-2.rpw", + "rustpotter/jarvis-community-3.rpw", + "rustpotter/jarvis-community-4.rpw", + // "rustpotter/jarvis-community-5.rpw", + ]; + + for rpw in rustpotter_wake_word_files { + rustpotter.add_wakeword_from_file(rpw).unwrap(); + } + + // store rustpotter + if RUSTPOTTER.get().is_none() { + RUSTPOTTER.set(Mutex::new(rustpotter)); + } + + // start recording + start_recording() +} + +fn vosk_init() -> Result { + start_recording() +} + +fn picovoice_init() -> Result { // VARS let porcupine: Porcupine; let picovoice_api_key: String; @@ -248,31 +396,11 @@ pub fn picovoice_listen<'s, S, K>(app_handle: &tauri::AppHandle, start_callback: } } - // Start recording - let mut frame_buffer = vec![0; porcupine.frame_length() as usize]; - recorder::FRAME_LENGTH.store(porcupine.frame_length(), Ordering::SeqCst); - recorder::start_recording(); - LISTENING.store(true, Ordering::SeqCst); - - // run start callback - start_callback(app_handle); - - // Listen until stop flag will be true - while !STOP_LISTENING.load(Ordering::SeqCst) { - recorder::read_microphone(&mut frame_buffer); - - if let Ok(keyword_index) = porcupine.process(&frame_buffer) { - if keyword_index >= 0 { - // println!("Yes, sir! {}", keyword_index); - keyword_callback(&app_handle, keyword_index); - } - } + // store + if PORCUPINE.get().is_none() { + PORCUPINE.set(porcupine); } - // Stop listening - recorder::stop_recording(); - LISTENING.store(false, Ordering::SeqCst); - STOP_LISTENING.store(false, Ordering::SeqCst); - - Ok(true) -} + // start recording + start_recording() +} \ No newline at end of file diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 6b6559f..3ec137c 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -8,7 +8,7 @@ }, "package": { "productName": "jarvis-app", - "version": "0.0.1" + "version": "0.0.2" }, "tauri": { "allowlist": { @@ -41,6 +41,7 @@ "sound", "vosk/model_small", "picovoice", + "rustpotter", "libvosk.dll", "libstdc++-6.dll", "libwinpthread-1.dll", diff --git a/src/pages/settings.svelte b/src/pages/settings.svelte index 2552871..0322166 100644 --- a/src/pages/settings.svelte +++ b/src/pages/settings.svelte @@ -16,6 +16,7 @@ // VARIABLES let available_microphones = []; let settings_saved = false; + let save_button_disabled = false; let assistant_voice_val = ""; // shared let selected_microphone = ""; @@ -32,6 +33,7 @@ // FUNCTIONS async function save_settings() { + save_button_disabled = true; // disable save button for a while settings_saved = false; // hide alert await invoke("db_write", {key: "assistant_voice", val: assistant_voice_val}); @@ -49,6 +51,10 @@ settings_saved = false; // hide alert again after N seconds }, 5000); + setTimeout(() => { + save_button_disabled = false; // enable save button again + }, 1000); + // restart listening everytime new settings is saved stopListening(() => { startListening(); @@ -157,7 +163,7 @@ -