GitHub/jarvis
1
0
Fork 0
mirror of https://github.com/Priler/jarvis.git synced 2026-05-26 07:08:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Recorder rewritten. + Attempt to integrate cpal and portaudio.

Browse Source
This commit is contained in:
Abraham
2023-04-30 22:32:48 +05:00
parent a988f3edc3
commit 2255479ec3
23 changed files with 983 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -1,7 +1,7 @@
{
"name": "jarvis-app",
"private": true,
"version": "0.0.1",
"version": "0.0.2",
"type": "module",
"scripts": {
"dev": "routify -c dev:vite",

124
src-tauri/Cargo.lock generated
View File

@@ -48,7 +48,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8512c9117059663fb5606788fbca3619e2a91dac0e3fe516242eab1fa6be5e44"
dependencies = [
"alsa-sys",
"bitflags",
"bitflags 1.3.2",
"libc",
"nix",
]
@@ -69,6 +69,12 @@ version = "1.0.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4"
[[package]]
name = "arc-swap"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
[[package]]
name = "arrayvec"
version = "0.7.2"
@@ -82,7 +88,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c3d816ce6f0e2909a96830d6911c2aff044370b1ef92d7f267b43bae5addedd"
dependencies = [
"atk-sys",
"bitflags",
"bitflags 1.3.2",
"glib",
"libc",
]
@@ -99,6 +105,17 @@ dependencies = [
"system-deps 6.0.5",
]
[[package]]
name = "atomic_enum"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6227a8d6fdb862bcb100c4314d0d9579e5cd73fa6df31a2e6f6e1acd3c5f1207"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "autocfg"
version = "1.1.0"
@@ -123,7 +140,7 @@ version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cexpr",
"clang-sys",
"lazy_static",
@@ -137,6 +154,12 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "bitflags"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -225,7 +248,7 @@ version = "0.15.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c76ee391b03d35510d9fa917357c7f1855bd9a6659c95a1b392e33f49b3369bc"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cairo-sys-rs",
"glib",
"libc",
@@ -362,7 +385,7 @@ version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f425db7937052c684daec3bd6375c8abe2d146dca4b8b143d6db777c39138f3a"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"block",
"cocoa-foundation",
"core-foundation",
@@ -378,7 +401,7 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "931d3837c286f56e3c58423ce4eba12d08db2374461a785c86f672b08b5650d6"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"block",
"core-foundation",
"core-graphics-types",
@@ -437,7 +460,7 @@ version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"core-foundation",
"core-graphics-types",
"foreign-types",
@@ -450,7 +473,7 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a68b68b3446082644c91ac778bf50cd4104bfb002b5a6a7c44cca5a2c70788b"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"core-foundation",
"foreign-types",
"libc",
@@ -462,7 +485,7 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb17e2d1795b1996419648915df94bc7103c28f7b48062d7acf4652fc371b2ff"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"core-foundation-sys 0.6.2",
"coreaudio-sys",
]
@@ -901,7 +924,7 @@ version = "0.15.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6e05c1f572ab0e1f15be94217f0dc29088c248b14f792a5ff0af0d84bcda9e8"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cairo-rs",
"gdk-pixbuf",
"gdk-sys",
@@ -917,7 +940,7 @@ version = "0.15.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad38dd9cc8b099cceecdf41375bb6d481b1b5a7cd5cd603e10a69a9383f8619a"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"gdk-pixbuf-sys",
"gio",
"glib",
@@ -1018,7 +1041,7 @@ version = "0.15.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68fdbc90312d462781a395f7a16d96a2b379bb6ef8cd6310a2df272771c4283b"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"futures-channel",
"futures-core",
"futures-io",
@@ -1048,7 +1071,7 @@ version = "0.15.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edb0306fbad0ab5428b0ca674a23893db909a98582969c9b537be4ced78c505d"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"futures-channel",
"futures-core",
"futures-executor",
@@ -1124,7 +1147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92e3004a2d5d6d8b5057d2b57b3712c9529b62e82c77f25c1fecde1fd5c23bd0"
dependencies = [
"atk",
"bitflags",
"bitflags 1.3.2",
"cairo-rs",
"field-offset",
"futures-channel",
@@ -1361,14 +1384,17 @@ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
[[package]]
name = "jarvis-app"
version = "0.0.1"
version = "0.0.2"
dependencies = [
"arc-swap",
"atomic_enum",
"hound",
"lazy_static",
"log",
"once_cell",
"peak_alloc",
"pickledb",
"portaudio",
"pv_porcupine",
"pv_recorder",
"rand 0.8.5",
@@ -1391,7 +1417,7 @@ version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf053e7843f2812ff03ef5afe34bb9c06ffee120385caad4f6b9967fcd37d41c"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"glib",
"javascriptcore-rs-sys",
]
@@ -1670,7 +1696,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2032c77e030ddee34a6787a64166008da93f6a352b629261d0fee232b8742dd4"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"jni-sys",
"ndk-sys 0.3.0",
"num_enum",
@@ -1683,7 +1709,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"jni-sys",
"ndk-sys 0.4.1+23.1.7779620",
"num_enum",
@@ -1727,7 +1753,7 @@ version = "0.24.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cfg-if",
"libc",
]
@@ -1758,6 +1784,17 @@ dependencies = [
"winapi",
]
[[package]]
name = "num"
version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e"
dependencies = [
"num-integer",
"num-iter",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.4.3"
@@ -1788,6 +1825,17 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.1"
@@ -1938,7 +1986,7 @@ version = "0.15.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e4045548659aee5313bde6c582b0d83a627b7904dd20dc2d9ef0895d414e4f"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"glib",
"libc",
"once_cell",
@@ -2156,13 +2204,25 @@ version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaeebc51f9e7d2c150d3f3bfeb667f2aa985db5ef1e3d212847bdedb488beeaa"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide 0.7.1",
]
[[package]]
name = "portaudio"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d289315f6155a7608b6d8757786c79ed2243afeab8a5eda8989effda3fdc5c3"
dependencies = [
"bitflags 0.7.0",
"libc",
"num",
"pkg-config",
]
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@@ -2379,7 +2439,7 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
"bitflags 1.3.2",
]
[[package]]
@@ -2388,7 +2448,7 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags",
"bitflags 1.3.2",
]
[[package]]
@@ -2519,7 +2579,7 @@ version = "0.37.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"errno",
"io-lifetimes",
"libc",
@@ -2585,7 +2645,7 @@ version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cssparser",
"derive_more",
"fxhash",
@@ -2802,7 +2862,7 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2b4d76501d8ba387cf0fefbe055c3e0a59891d09f0f995ae4e4b16f6b60f3c0"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"gio",
"glib",
"libc",
@@ -2816,7 +2876,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "009ef427103fcb17f802871647a7fa6c60cbb654b4c4e4c0ac60a31c5f6dc9cf"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"gio-sys",
"glib-sys",
"gobject-sys",
@@ -2895,7 +2955,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55a0846e7a2c9a8081ff799fc83a975170417ad2a143f644a77ec2e3e82a2b73"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"lazy_static",
"log",
"symphonia-core",
@@ -2909,7 +2969,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b9567e2d8a5f866b2f94f5d366d811e0c6826babcff6d37de9e1a6690d38869"
dependencies = [
"arrayvec",
"bitflags",
"bitflags 1.3.2",
"bytemuck",
"lazy_static",
"log",
@@ -2995,7 +3055,7 @@ version = "0.15.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac8e6399427c8494f9849b58694754d7cc741293348a6836b6c8d2c5aa82d8e6"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cairo-rs",
"cc",
"cocoa",
@@ -3691,7 +3751,7 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8f859735e4a452aeb28c6c56a852967a8a76c8eb1cc32dbf931ad28a13d6370"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cairo-rs",
"gdk",
"gdk-sys",
@@ -3716,7 +3776,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d76ca6ecc47aeba01ec61e480139dda143796abcae6f83bcddf50d6b5b1dcf3"
dependencies = [
"atk-sys",
"bitflags",
"bitflags 1.3.2",
"cairo-sys-rs",
"gdk-pixbuf-sys",
"gdk-sys",

5
src-tauri/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-app"
version = "0.0.1"
version = "0.0.2"
description = "Jarvis Voice Assistant"
authors = ["Abraham Tugalov"]
license = "GPL-3.0-only"
@@ -36,6 +36,9 @@ rustpotter = "2.0.0"
simple-logging = "2.0.2"
log = "0.4.17"
once_cell = "1.17.1"
arc-swap = "1.6.0"
atomic_enum = "0.2.0"
portaudio = "0.7.0"
[features]
# this feature is used for production builds or when `devPath` points to the filesystem

BIN
src-tauri/rustpotter/___default.rpw Normal file
View File

Binary file not shown.

BIN
src-tauri/rustpotter/jarvis-community-1.rpw Normal file
View File

Binary file not shown.

BIN
src-tauri/rustpotter/jarvis-community-2.rpw Normal file
View File

Binary file not shown.

BIN
src-tauri/rustpotter/jarvis-community-3.rpw Normal file
View File

Binary file not shown.

BIN
src-tauri/rustpotter/jarvis-community-4.rpw Normal file
View File

Binary file not shown.

BIN
src-tauri/rustpotter/jarvis-community-5.rpw Normal file
View File

Binary file not shown.

BIN
src-tauri/rustpotter/jarvis-default.rpw Normal file
View File

Binary file not shown.

2
src-tauri/src/.cargo/config.toml Normal file
View File

@@ -0,0 +1,2 @@
[env]
PORTAUDIO_ONLY_STATIC = true

13
src-tauri/src/assistant_commands.rs
View File

@@ -3,6 +3,7 @@ use seqdiff::ratio;
use serde_yaml;
use std::path::Path;
use std::{fs, fs::File};
use log::{info, warn, error};
use core::time::Duration;
use std::path::PathBuf;
@@ -36,7 +37,7 @@ pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
if let Ok(parse_result) = serde_yaml::from_reader::<File, CommandsList>(cc_reader) {
cc_yaml = parse_result;
} else {
println!("Can't parse {}, skipping ...", &cc_file.display());
warn!("Can't parse {}, skipping ...", &cc_file.display());
continue;
// return Err(format!("Can't parse {}", &cc_file.display()));
}
@@ -52,9 +53,11 @@ pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
if commands.len() > 0 {
Ok(commands)
} else {
error!("No commands were found");
Err("No commands were found".into())
}
} else {
error!("Error reading commands directory");
return Err("Error reading commands directory".into());
}
}
@@ -95,6 +98,7 @@ pub fn fetch_command<'a>(
if let Some((cmd_path, scmd)) = result_scmd {
println!("Ratio is: {}", current_max_ratio);
info!("CMD is: {cmd_path:?}, SCMD is: {scmd:?}, Ratio is: {}", current_max_ratio);
Some((&cmd_path, &scmd))
} else {
None
@@ -144,6 +148,7 @@ pub fn execute_command(
Ok(())
} else {
error!("AHK process spawn error (does exe path is valid?)");
Err("AHK process spawn error (does exe path is valid?)".into())
}
}
@@ -169,6 +174,7 @@ pub fn execute_command(
Ok(())
} else {
error!("Shell process spawn error (does cli command is valid?)");
Err("Shell process spawn error (does cli command is valid?)".into())
}
}
@@ -184,6 +190,9 @@ pub fn execute_command(
std::thread::sleep(Duration::from_secs(2));
std::process::exit(0);
}
_ => Err("Command type unknown".into()),
_ => {
error!("Command type unknown");
Err("Command type unknown".into())
},
}
}

18
src-tauri/src/config.rs
View File

@@ -7,7 +7,15 @@
// "./public"
// };
pub const WAKE_WORD_ENGINES: [&str; 2] = ["rustpotter", "picovoice"];
// APP
// pub const WAKE_WORD_ENGINES: [&str; 3] = ["rustpotter", "vosk", "picovoice"];
pub enum WakeWordEngine {
Rustpotter,
Vosk,
Porcupine
}
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter;
pub const DB_FILE_NAME: &str = "app.db";
pub const LOG_FILE_NAME: &str = "log.txt";
@@ -15,12 +23,20 @@ pub const APP_VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION");
pub const AUTHOR_NAME: Option<&str> = option_env!("CARGO_PKG_AUTHORS");
pub const REPOSITORY_LINK: Option<&str> = option_env!("CARGO_PKG_REPOSITORY");
// RUSPOTTER
pub const RUSPOTTER_MIN_SCORE: f32 = 0.64;
// PICOVOICE
pub const COMMANDS_PATH: &str = "commands/";
pub const KEYWORDS_PATH: &str = "picovoice/keywords/";
// VOSK
// pub const VOSK_MODEL_PATH: &str = const_concat!(PUBLIC_PATH, "/vosk/model_small");
pub const VOSK_FETCH_PHRASE: &str = "джарвис";
pub const VOSK_MODEL_PATH: &str = "vosk/model_small";
pub const VOSK_MIN_RATIO: f64 = 70.0;
// ETC
pub const CMD_RATIO_THRESHOLD: f64 = 60f64;
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(10);

16
src-tauri/src/main.rs
View File

@@ -33,6 +33,11 @@ lazy_static! {
static ref APP_CONFIG_DIR: Mutex<String> = Mutex::new(String::new());
}
// data dir
lazy_static! {
static ref APP_LOG_DIR: Mutex<String> = Mutex::new(String::new());
}
// init PickleDb connection
lazy_static! {
static ref DB: Mutex<PickleDb> = Mutex::new(
@@ -58,9 +63,6 @@ lazy_static! {
}
fn main() {
// log to file
simple_logging::log_to_file(config::LOG_FILE_NAME, LevelFilter::max()).expect("Failed to start logger ... is directory writable?");
// init vosk
vosk::init_vosk();
@@ -70,6 +72,14 @@ fn main() {
std::fs::create_dir_all(app.path_resolver().app_config_dir().unwrap())?;
APP_CONFIG_DIR.lock().unwrap().push_str(app.path_resolver().app_config_dir().unwrap().to_str().unwrap());
std::fs::create_dir_all(app.path_resolver().app_log_dir().unwrap())?;
APP_LOG_DIR.lock().unwrap().push_str(app.path_resolver().app_log_dir().unwrap().to_str().unwrap());
// log to file
let log_file_path = format!("{}/{}", APP_LOG_DIR.lock().unwrap(), config::LOG_FILE_NAME);
println!("!!!===============!!!\nLOGGING TO {}\n!!!===============!!!\n", &log_file_path);
simple_logging::log_to_file(log_file_path, LevelFilter::max()).expect("Failed to start logger ... is directory writable?");
Ok(())
})
.invoke_handler(tauri::generate_handler![

139
src-tauri/src/recorder.rs
View File

@@ -1,52 +1,127 @@
use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicU32, AtomicBool, Ordering};
use pv_recorder::{Recorder, RecorderBuilder};
use log::{info};
// use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicU32, Ordering};
use log::{info, warn, error};
use atomic_enum::atomic_enum;
mod pvrecorder;
// mod cpal;
// mod portaudio;
use crate::DB;
#[atomic_enum]
#[derive(PartialEq)]
pub enum RecorderType {
Cpal,
PvRecorder,
PortAudio
}
pub static RECORDER_TYPE: AtomicRecorderType = AtomicRecorderType::new(RecorderType::PvRecorder); // use pvrecorder as default
pub static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static RECORDER: OnceCell<Recorder> = OnceCell::new();
pub static IS_RECORDING: AtomicBool = AtomicBool::new(false);
fn init_microphone() {
if RECORDER.get().is_none() {
RECORDER.get_or_init(|| RecorderBuilder::new()
.device_index(get_selected_microphone_index())
.frame_length(FRAME_LENGTH.load(Ordering::SeqCst) as i32)
.init()
.expect("Failed to initialize pvrecorder"));
info!("Microphone recorder initialized!")
pub fn init() {
match RECORDER_TYPE.load(Ordering::SeqCst) {
RecorderType::PvRecorder => {
// Init Pv Recorder
info!("Initializing Pv Recorder audio backend.");
match pvrecorder::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
false => {
// Switch to CPAL recorder
warn!("Pv Recorder audio backend failed.");
// RECORDER_TYPE.store(RecorderType::PortAudio, Ordering::SeqCst);
// init again
init();
},
_ => ()
}
},
RecorderType::PortAudio => {
// Init PortAudio
info!("Initializing PortAudio audio backend");
todo!();
// match portaudio::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
// false => {
// // Switch to PortAudio recorder
// error!("PortAudio audio backend failed.");
// },
// _ => ()
// }
},
RecorderType::Cpal => {
// Init CPAL
info!("Initializing CPAL audio backend");
todo!();
// match cpal::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
// false => {
// // Switch to CPAL recorder
// error!("CPAL audio backend failed.");
// },
// _ => ()
// }
}
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
init_microphone();
// read to frame buffer
RECORDER.get().unwrap().read(frame_buffer).expect("Failed to read audio frame");
match RECORDER_TYPE.load(Ordering::SeqCst) {
RecorderType::PvRecorder => {
pvrecorder::read_microphone(frame_buffer);
},
RecorderType::PortAudio => {
todo!();
// portaudio::read_microphone(frame_buffer);
},
RecorderType::Cpal => {
// cpal::read_microphone(frame_buffer);
panic!("Cpal should be used via callback assignment");
}
}
}
pub fn start_recording() {
// ensure microphone is initialized
init_microphone();
RECORDER.get().unwrap().start().expect("Failed to start audio recording!");
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
match RECORDER_TYPE.load(Ordering::SeqCst) {
RecorderType::PvRecorder => {
pvrecorder::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
},
RecorderType::PortAudio => {
todo!();
// portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
},
RecorderType::Cpal => {
// cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
}
}
}
pub fn stop_recording() {
// ensure microphone is initialized
init_microphone();
RECORDER.get().unwrap().start().expect("Failed to start audio recording!");
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
match RECORDER_TYPE.load(Ordering::SeqCst) {
RecorderType::PvRecorder => {
pvrecorder::stop_recording();
},
RecorderType::PortAudio => {
todo!();
// portaudio::stop_recording();
},
RecorderType::Cpal => {
// cpal::stop_recording();
}
}
}
// pub fn update_selected_microphone_index() -> i32 {
// let selected_microphone: i32 = get_selected_microphone_index();
// // store current microphone idx
// SELECTED_MICROPHONE_IDX.store(selected_microphone, Ordering::SeqCst);
// // return microphone index
// info!("Selected microphone index = {selected_microphone}");
// selected_microphone
// }
pub fn get_selected_microphone_index() -> i32 {
let selected_microphone: i32;
@@ -57,7 +132,5 @@ pub fn get_selected_microphone_index() -> i32 {
selected_microphone = -1;
}
// return microphone index
info!("Selected microphone index = {selected_microphone}");
selected_microphone
}

186
src-tauri/src/recorder/cpal.rs Normal file
View File

@@ -0,0 +1,186 @@
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{BufferSize, StreamConfig, SampleRate, Host, Device, Stream, SampleFormat};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::Arc;
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
use crate::tauri_commands::cpal_data_callback;
static HOST: OnceCell<Host> = OnceCell::new();
thread_local!(static RECORDER: OnceCell<ArcSwap<Stream>> = OnceCell::new());
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
// init host & frame buffer for the callback
if HOST.get().is_none() {
HOST.set(cpal::default_host());
// FRAME_BUFFER.set(Mutex::new(vec![0; FRAME_LENGTH.load(Ordering::SeqCst) as usize]));
}
// init microphone
RECORDER.with(|recorder| {
match recorder.get().is_none() {
true => {
if let Some(device) = get_device(device_index as usize) {
// store
recorder.set(ArcSwap::from_pointee(create_stream(device, frame_length)));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
} else {
false
}
},
false => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
stop_recording();
}
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
if let Some(device) = get_device(device_index as usize) {
// store
recorder.get().unwrap().store(Arc::new(create_stream(device, frame_length)));
// success
return true
} else {
return false
}
}
// success
true
}
}
})
}
fn create_stream(device: Device, frame_length: u32) -> Stream {
// get default input stream config
// let default_config = device.default_input_config().unwrap();
// create config for the stream
// let config: StreamConfig = StreamConfig {
// channels: default_config.channels(),
// sample_rate: SampleRate(16000),
// buffer_size: BufferSize::Fixed(frame_length)
// };
let config = device
.default_input_config()
.expect("Failed to load default input config");
let channels = config.channels();
let err_fn = move |err| {
eprintln!("an error occurred on stream: {}", err);
};
match config.sample_format() {
SampleFormat::F32 => device.build_input_stream(
&config.into(),
move |data: &[f32], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
SampleFormat::U16 => device.build_input_stream(
&config.into(),
move |data: &[u16], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
SampleFormat::I16 => device.build_input_stream(
&config.into(),
move |data: &[i16], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
_ => todo!()
}.unwrap()
}
pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
let mut result = Vec::with_capacity(input_data.len() / 2);
result.extend(
input_data
.chunks_exact(2)
.map(|chunk| chunk[0] / 2 + chunk[1] / 2),
);
result
}
fn get_device(device_index: usize) -> Option<Device> {
if let Some(device) = HOST.get().unwrap().input_devices().expect("Get devices error ...").nth(device_index) {
Some(device)
} else {
if let Some(default) = HOST.get().unwrap().default_input_device() {
Some(default)
} else {
error!("No default input device ...");
None
}
}
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.with(|recorder| {
match recorder.get().unwrap().load().play() {
Err(msg) => {
error!("[CPAL] Audio stream PLAY error ... {:?}", msg);
},
_ => ()
};
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
});
}
pub fn stop_recording() {
// ensure microphone is initialized
RECORDER.with(|recorder| {
if !recorder.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// pause instead of stop
match recorder.get().unwrap().load().pause() {
Err(msg) => {
error!("[CPAL] Audio stream PAUSE error ... {:?}", msg);
},
_ => ()
};
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
});
}

201
src-tauri/src/recorder/portaudio.rs Normal file
View File

@@ -0,0 +1,201 @@
use portaudio as pa;
use pa::{DeviceIndex, Stream};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::{Arc, Mutex};
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
thread_local!(static RECORDER: OnceCell<ArcSwap<Mutex<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>>>> = OnceCell::new());
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
const CHANNELS: i32 = 1;
const SAMPLE_RATE: f64 = 16_000.0;
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
RECORDER.with(|r| {
match r.get().is_none() {
true => {
match create_stream(device_index, frame_length) {
Ok(stream) => {
// store
r.set(ArcSwap::from_pointee(Mutex::new(stream)));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
},
Err(msg) => {
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
// fail
false
}
}
},
_ => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
// RECORDER.get().unwrap().load().stop().expect("Failed to start audio recording!");
stop_recording();
}
// store
match create_stream(device_index, frame_length) {
Ok(stream) => {
// store new stream
r.get().unwrap().store(Arc::new(Mutex::new(stream)));
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
return true
},
Err(msg) => {
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
// fail
return false
}
}
}
// success
true
}
}
})
}
fn create_stream(device_index: i32, frame_length: u32) -> Result<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>, pa::Error> {
let pa_recorder: Result<pa::PortAudio, pa::Error> = pa::PortAudio::new();
match pa_recorder {
Ok(pa) => {
let input_settings = match get_input_settings(DeviceIndex(device_index as u32), &pa, SAMPLE_RATE, frame_length, CHANNELS) {
Ok(settings) => settings,
Err(error) => panic!("{}", String::from(error))
};
// Construct a stream with input and output sample types of i16
match pa.open_blocking_stream(input_settings) {
Ok(strm) => Ok(strm),
Err(error) => panic!("{}", error.to_string()),
}
},
Err(msg) => Err(msg)
}
}
fn get_input_latency(audio_port: &pa::PortAudio, input_index: pa::DeviceIndex) -> Result<f64, String>
{
let input_device_information = audio_port.device_info(input_index).or_else(|error| Err(String::from(format!("{}", error))));
Ok(input_device_information.unwrap().default_low_input_latency)
}
fn get_input_stream_parameters(input_index: pa::DeviceIndex, latency: f64, channels: i32) -> Result<pa::StreamParameters<i16>, String>
{
const INTERLEAVED: bool = true;
Ok(pa::StreamParameters::<i16>::new(input_index, channels, INTERLEAVED, latency))
}
fn get_input_settings(input_index: pa::DeviceIndex, audio_port: &pa::PortAudio, sample_rate: f64, frames: u32, channels: i32) -> Result<pa::InputStreamSettings<i16>, String>
{
Ok(
pa::InputStreamSettings::new(
(get_input_stream_parameters(
input_index,
(get_input_latency(
&audio_port,
input_index,
))?,
channels
))?,
sample_rate,
frames,
)
)
}
// We'll use this function to wait for read/write availability.
fn wait_for_stream<F>(f: F, name: &str) -> u32
where
F: Fn() -> Result<pa::StreamAvailable, pa::error::Error>,
{
loop {
match f() {
Ok(available) => match available {
pa::StreamAvailable::Frames(frames) => return frames as u32,
pa::StreamAvailable::InputOverflowed => println!("Input stream has overflowed"),
pa::StreamAvailable::OutputUnderflowed => {
println!("Output stream has underflowed")
}
},
Err(err) => panic!(
"An error occurred while waiting for the {} stream: {}",
name, err
),
}
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
RECORDER.with(|r| {
if !r.get().is_none() {
let cell = r.get().unwrap().load();
let mut lock = cell.lock();
let stream = lock.as_mut().unwrap();
// read to frame buffer
let in_frames = wait_for_stream(|| stream.read_available(), "Read");
if in_frames > 0 {
// let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
// println!("Read {:?} frames from the input stream.", in_frames);
let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
println!("Read: {} (required {})", input_samples.len(), frame_buffer.len());
frame_buffer.copy_from_slice(input_samples.chunks(frame_buffer.len()).last().unwrap());
}
// r.get().unwrap().load().read(frame_buffer).expect("Failed to read audio frame");
}
});
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.with(|r| {
r.get().unwrap().load().lock().unwrap().start().expect("Failed to start audio recording!");
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
});
}
pub fn stop_recording() {
RECORDER.with(|r| {
if !r.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// stop recording
let pa = r.get().unwrap().load();
r.get().unwrap().load().lock().unwrap().stop().expect("Failed to stop audio recording!");
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
});
}

105
src-tauri/src/recorder/pvrecorder.rs Normal file
View File

@@ -0,0 +1,105 @@
use pv_recorder::{Recorder, RecorderBuilder};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::Arc;
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
static RECORDER: OnceCell<ArcSwap<Recorder>> = OnceCell::new();
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
match RECORDER.get().is_none() {
true => {
let pv_recorder = RecorderBuilder::new()
.device_index(device_index)
.frame_length(frame_length as i32)
.init();
match pv_recorder {
Ok(pv) => {
// store
RECORDER.set(ArcSwap::from_pointee(pv));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
},
Err(msg) => {
error!("Failed to initialize pvrecorder.\nError details: {:?}", msg);
// fail
false
}
}
},
_ => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
RECORDER.get().unwrap().load().frame_length() != frame_length as usize {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
// RECORDER.get().unwrap().load().stop().expect("Failed to start audio recording!");
stop_recording();
}
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// store
RECORDER.get().unwrap().store(Arc::new(RecorderBuilder::new()
.device_index(device_index)
.frame_length(frame_length as i32)
.init()
.expect("Failed to initialize pvrecorder")));
}
// success
true
}
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
if !RECORDER.get().is_none() {
// read to frame buffer
match RECORDER.get().unwrap().load().read(frame_buffer) {
Err(msg) => {
// @TODO: Fix somehow. PvRecorder always wait for PCM buffer size of 512.
// error!("Failed to read audio frame. {:?}", msg);
// eprintln!("Failed to read audio frame. {:?}", msg);
},
_ => ()
}
}
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.get().unwrap().load().start().expect("Failed to start audio recording!");
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
}
pub fn stop_recording() {
// ensure microphone is initialized
if !RECORDER.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// stop recording
RECORDER.get().unwrap().load().stop().expect("Failed to stop audio recording!");
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
}

4
src-tauri/src/tauri_commands.rs
View File

@@ -3,8 +3,8 @@ mod db;
pub use db::*;
// import RECORDER commands
mod recorder;
pub use recorder::*;
mod audio;
pub use audio::*;
// import PORCUPINE commands
mod listener;

0
src-tauri/src/tauri_commands/recorder.rs → src-tauri/src/tauri_commands/audio.rs
View File

340
src-tauri/src/tauri_commands/listener.rs
View File

@@ -2,19 +2,23 @@ use porcupine::{Porcupine, PorcupineBuilder};
use std::sync::atomic::{AtomicBool, Ordering};
use std::path::Path;
use log::{info, warn, error};
use rustpotter::{Rustpotter, RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig};
// use dasp::{sample::ToSample, Sample};
// use crate::events::Payload;
use tauri::Manager;
use rand::seq::SliceRandom;
use std::time::SystemTime;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
use crate::assistant_commands;
use crate::events;
use crate::config;
use crate::vosk;
use crate::recorder;
use crate::recorder::{self, FRAME_LENGTH};
use crate::COMMANDS;
use crate::DB;
@@ -25,6 +29,15 @@ static LISTENING: AtomicBool = AtomicBool::new(false);
// stop listening with Atomic flag (to make it work between different threads)
static STOP_LISTENING: AtomicBool = AtomicBool::new(false);
// store tauri app_handle
static TAURI_APP_HANDLE: OnceCell<tauri::AppHandle> = OnceCell::new();
// store porcupine instance
static PORCUPINE: OnceCell<Porcupine> = OnceCell::new();
// store rustpotter instance
static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
#[tauri::command]
pub fn is_listening() -> bool {
LISTENING.load(Ordering::SeqCst)
@@ -34,12 +47,31 @@ pub fn is_listening() -> bool {
pub fn stop_listening() {
if is_listening() {
STOP_LISTENING.store(true, Ordering::SeqCst);
stop_recording();
}
// wait until listening stops
while is_listening() {}
}
fn get_wake_word_engine() -> config::WakeWordEngine {
let selected_wake_word_engine;
if let Some(wwengine) = DB.lock().unwrap().get::<String>("selected_wake_word_engine") {
// from db
match wwengine.trim().to_lowercase().as_str() {
"rustpotter" => selected_wake_word_engine = config::WakeWordEngine::Rustpotter,
"vosk" => selected_wake_word_engine = config::WakeWordEngine::Vosk,
"picovoice" => selected_wake_word_engine = config::WakeWordEngine::Porcupine,
&_ => todo!()
}
} else {
// default
selected_wake_word_engine = config::DEFAULT_WAKE_WORD_ENGINE; // set default wake_word engine
}
selected_wake_word_engine
}
#[tauri::command(async)]
pub fn start_listening(app_handle: tauri::AppHandle) -> Result<bool, String> {
// only one listener thread is allowed
@@ -47,44 +79,29 @@ pub fn start_listening(app_handle: tauri::AppHandle) -> Result<bool, String> {
return Err("Already listening.".into());
}
// Retrieve selected wake-word engine from DB
let selected_wake_word_engine;
if let Some(wwengine) = DB.lock().unwrap().get::<String>("selected_wake_word_engine") {
// from db
selected_wake_word_engine = wwengine;
} else {
// default
selected_wake_word_engine = config::WAKE_WORD_ENGINES.first().expect("No wake-word engines found ...").to_string(); // set default wake_word engine
// keep app handle
if TAURI_APP_HANDLE.get().is_none() {
TAURI_APP_HANDLE.set(app_handle);
}
// call selected wake-word engine listener command
match selected_wake_word_engine.as_str() {
"rustpotter" => {
info!("Starting rustpotter wake-word engine ...");
return picovoice_listen(&app_handle, |_app| {
// Greet user
events::play("run", &app_handle);
}, |app, kidx| keyword_callback(app, kidx));
match get_wake_word_engine() {
config::WakeWordEngine::Rustpotter => {
info!("Starting RUSTPOTTER wake-word engine ...");
return rustpotter_init();
},
"vosk" => {
info!("Starting vosk wake-word engine ...");
return vosk_listen(&app_handle, |_app| {
// Greet user
events::play("run", &app_handle);
}, |app, kidx| keyword_callback(app, kidx));
config::WakeWordEngine::Vosk => {
info!("Starting VOSK wake-word engine ...");
return vosk_init();
},
"picovoice" => {
info!("Starting picovoice wake-word engine ...");
return picovoice_listen(&app_handle, |_app| {
// Greet user
events::play("run", &app_handle);
}, |app, kidx| keyword_callback(app, kidx));
},
_ => Err("No wake-word engine selected ...".into())
config::WakeWordEngine::Porcupine => {
info!("Starting PICOVOICE PORCUPINE wake-word engine ...");
return picovoice_init();
}
}
}
pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) {
fn keyword_callback(_keyword_index: i32) {
// vars
let mut start: SystemTime = SystemTime::now();
let mut frame_buffer = vec![0; recorder::FRAME_LENGTH.load(Ordering::SeqCst) as usize];
@@ -94,11 +111,11 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) {
config::ASSISTANT_GREET_PHRASES
.choose(&mut rand::thread_rng())
.unwrap(),
&app_handle,
TAURI_APP_HANDLE.get().unwrap(),
);
// emit assistant greet event
app_handle
TAURI_APP_HANDLE.get().unwrap()
.emit_all(events::EventTypes::AssistantGreet.get(), ())
.unwrap();
@@ -128,7 +145,7 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) {
let cmd_result = assistant_commands::execute_command(
&cmd_path,
&cmd_config,
&app_handle,
TAURI_APP_HANDLE.get().unwrap(),
);
match cmd_result {
@@ -142,7 +159,7 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) {
}
}
app_handle
TAURI_APP_HANDLE.get().unwrap()
.emit_all(events::EventTypes::AssistantWaiting.get(), ())
.unwrap();
break; // return to picovoice after command execution (no matter successfull or not)
@@ -153,7 +170,7 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) {
match start.elapsed() {
Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
// return to picovoice after N seconds
app_handle
TAURI_APP_HANDLE.get().unwrap()
.emit_all(events::EventTypes::AssistantWaiting.get(), ())
.unwrap();
break;
@@ -163,60 +180,191 @@ pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) {
}
}
pub fn vosk_listen<'s, S, K>(app_handle: &tauri::AppHandle, start_callback: S, mut keyword_callback: K) -> Result<bool, String>
where S: Fn(&tauri::AppHandle),
K: FnMut(&tauri::AppHandle, i32) {
pub fn data_callback(frame_buffer: &[i16]) {
// println!("DATA CALLBACK {}", frame_buffer.len());
match get_wake_word_engine() {
config::WakeWordEngine::Rustpotter => {
let mut lock = RUSTPOTTER.get().unwrap().lock();
let rustpotter = lock.as_mut().unwrap();
let detection = rustpotter.process_i16(&frame_buffer);
// vars
let fetch_phrase = "джарвис".chars().collect::<Vec<_>>();
let frame_length: usize = 128;
let min_ratio: f64 = 0.8;
if let Some(detection) = detection {
if detection.score > config::RUSPOTTER_MIN_SCORE {
info!("Rustpotter detection info:\n{:?}", detection);
keyword_callback(0);
} else {
info!("Rustpotter detection info:\n{:?}", detection);
}
}
},
config::WakeWordEngine::Vosk => {
// recognize & convert to sequence
let recognized_phrase = vosk::recognize(&frame_buffer, true).unwrap_or("".into());
// Start recording
let mut frame_buffer = vec![0; frame_length];
recorder::FRAME_LENGTH.store(frame_length as u32, Ordering::SeqCst);
recorder::start_recording();
LISTENING.store(true, Ordering::SeqCst);
if !recognized_phrase.trim().is_empty() {
info!("Rec: {}", recognized_phrase);
let recognized_phrases = recognized_phrase.split_whitespace();
for phrase in recognized_phrases {
let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
// compare
let compare_ratio = seqdiff::ratio(&config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(), &recognized_phrase_chars);
info!("OG phrase: {:?}", &config::VOSK_FETCH_PHRASE);
info!("Recognized phrase: {:?}", &recognized_phrase_chars);
info!("Compare ratio: {}", compare_ratio);
// run start callback
start_callback(app_handle);
// Listen until stop flag will be true
while !STOP_LISTENING.load(Ordering::SeqCst) {
recorder::read_microphone(&mut frame_buffer);
// recognize & convert to sequence
let recognized_phrase = vosk::recognize(&frame_buffer, true).unwrap_or("".into());
if !recognized_phrase.trim().is_empty() {
info!("Rec: {}", recognized_phrase);
let recognized_phrases = recognized_phrase.split_whitespace();
for phrase in recognized_phrases {
let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
// compare
if seqdiff::ratio(&fetch_phrase, &recognized_phrase_chars) >= min_ratio {
info!("Phrase: {:?}", &fetch_phrase);
info!("Compare: {:?}", &recognized_phrase_chars);
keyword_callback(&app_handle, 0);
break;
if compare_ratio >= config::VOSK_MIN_RATIO {
info!("Phrase activated.");
keyword_callback(0);
break;
}
}
}
},
config::WakeWordEngine::Porcupine => {
if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) {
if keyword_index >= 0 {
// println!("Yes, sir! {}", keyword_index);
keyword_callback(keyword_index);
}
}
}
}
// Stop listening
recorder::stop_recording();
LISTENING.store(false, Ordering::SeqCst);
STOP_LISTENING.store(false, Ordering::SeqCst);
Ok(true)
}
pub fn picovoice_listen<'s, S, K>(app_handle: &tauri::AppHandle, start_callback: S, mut keyword_callback: K) -> Result<bool, String>
where S: Fn(&tauri::AppHandle),
K: FnMut(&tauri::AppHandle, i32) {
fn start_recording() -> Result<bool, String> {
// vars
let frame_length: usize;
// idenfity frame length
match get_wake_word_engine() {
config::WakeWordEngine::Rustpotter => {
// start recording for Rustpotter
// You need a buffer of size `rustpotter.get_samples_per_frame()` when using samples.
// You need a buffer of size `rustpotter.get_bytes_per_frame()` when using bytes.
frame_length = RUSTPOTTER.get().unwrap().lock().unwrap().get_samples_per_frame();
recorder::FRAME_LENGTH.store(frame_length as u32, Ordering::SeqCst);
},
config::WakeWordEngine::Vosk => {
// start recording for Vosk
frame_length = 128;
recorder::FRAME_LENGTH.store(frame_length as u32, Ordering::SeqCst);
},
config::WakeWordEngine::Porcupine => {
// start recording for Porcupine
frame_length = PORCUPINE.get().unwrap().frame_length() as usize;
recorder::FRAME_LENGTH.store(PORCUPINE.get().unwrap().frame_length(), Ordering::SeqCst);
}
}
// define frame buffer
let mut frame_buffer: Vec<i16> = vec![0; frame_length];
// init stuff
recorder::init(); // init
recorder::start_recording(); // start
LISTENING.store(true, Ordering::SeqCst);
info!("START listening ...");
// greet user
events::play("run", TAURI_APP_HANDLE.get().unwrap());
// record
match recorder::RECORDER_TYPE.load(Ordering::SeqCst) {
recorder::RecorderType::PvRecorder => {
while !STOP_LISTENING.load(Ordering::SeqCst) {
recorder::read_microphone(&mut frame_buffer);
data_callback(&frame_buffer);
}
// stop
stop_recording();
Ok(true)
},
recorder::RecorderType::PortAudio => {
while !STOP_LISTENING.load(Ordering::SeqCst) {
recorder::read_microphone(&mut frame_buffer);
data_callback(&frame_buffer);
}
// stop
stop_recording();
Ok(true)
}
recorder::RecorderType::Cpal => {
todo!()
}
}
}
fn stop_recording() {
// Stop listening
recorder::stop_recording();
LISTENING.store(false, Ordering::SeqCst);
STOP_LISTENING.store(false, Ordering::SeqCst);
info!("STOP listening ...");
}
fn rustpotter_init() -> Result<bool, String> {
// init rustpotter
let rustpotter_config = RustpotterConfig {
fmt: WavFmt::default(),
detector: DetectorConfig {
avg_threshold: 0.,
threshold: 0.5,
min_scores: 15,
score_mode: ScoreMode::Max,
comparator_band_size: 5,
comparator_ref: 0.22
},
filters: FiltersConfig {
gain_normalizer: GainNormalizationConfig {
enabled: true,
gain_ref: None,
min_gain: 0.5,
max_gain: 1.0,
},
band_pass: BandPassConfig {
enabled: true,
low_cutoff: 80.,
high_cutoff: 400.,
}
}
};
let mut rustpotter = Rustpotter::new(&rustpotter_config).unwrap();
// load a wakeword
let rustpotter_wake_word_files: [&str; 5] = [
"rustpotter/jarvis-default.rpw",
"rustpotter/jarvis-community-1.rpw",
"rustpotter/jarvis-community-2.rpw",
"rustpotter/jarvis-community-3.rpw",
"rustpotter/jarvis-community-4.rpw",
// "rustpotter/jarvis-community-5.rpw",
];
for rpw in rustpotter_wake_word_files {
rustpotter.add_wakeword_from_file(rpw).unwrap();
}
// store rustpotter
if RUSTPOTTER.get().is_none() {
RUSTPOTTER.set(Mutex::new(rustpotter));
}
// start recording
start_recording()
}
fn vosk_init() -> Result<bool, String> {
start_recording()
}
fn picovoice_init() -> Result<bool, String> {
// VARS
let porcupine: Porcupine;
let picovoice_api_key: String;
@@ -248,31 +396,11 @@ pub fn picovoice_listen<'s, S, K>(app_handle: &tauri::AppHandle, start_callback:
}
}
// Start recording
let mut frame_buffer = vec![0; porcupine.frame_length() as usize];
recorder::FRAME_LENGTH.store(porcupine.frame_length(), Ordering::SeqCst);
recorder::start_recording();
LISTENING.store(true, Ordering::SeqCst);
// run start callback
start_callback(app_handle);
// Listen until stop flag will be true
while !STOP_LISTENING.load(Ordering::SeqCst) {
recorder::read_microphone(&mut frame_buffer);
if let Ok(keyword_index) = porcupine.process(&frame_buffer) {
if keyword_index >= 0 {
// println!("Yes, sir! {}", keyword_index);
keyword_callback(&app_handle, keyword_index);
}
}
// store
if PORCUPINE.get().is_none() {
PORCUPINE.set(porcupine);
}
// Stop listening
recorder::stop_recording();
LISTENING.store(false, Ordering::SeqCst);
STOP_LISTENING.store(false, Ordering::SeqCst);
Ok(true)
}
// start recording
start_recording()
}

3
src-tauri/tauri.conf.json
View File

@@ -8,7 +8,7 @@
},
"package": {
"productName": "jarvis-app",
"version": "0.0.1"
"version": "0.0.2"
},
"tauri": {
"allowlist": {
@@ -41,6 +41,7 @@
"sound",
"vosk/model_small",
"picovoice",
"rustpotter",
"libvosk.dll",
"libstdc++-6.dll",
"libwinpthread-1.dll",

8
src/pages/settings.svelte
View File

@@ -16,6 +16,7 @@
// VARIABLES
let available_microphones = [];
let settings_saved = false;
let save_button_disabled = false;
let assistant_voice_val = ""; // shared
let selected_microphone = "";
@@ -32,6 +33,7 @@
// FUNCTIONS
async function save_settings() {
save_button_disabled = true; // disable save button for a while
settings_saved = false; // hide alert
await invoke("db_write", {key: "assistant_voice", val: assistant_voice_val});
@@ -49,6 +51,10 @@
settings_saved = false; // hide alert again after N seconds
}, 5000);
setTimeout(() => {
save_button_disabled = false; // enable save button again
}, 1000);
// restart listening everytime new settings is saved
stopListening(() => {
startListening();
@@ -157,7 +163,7 @@
<Space h="xl" />
<Button color="lime" radius="md" size="sm" uppercase ripple fullSize on:click={save_settings}>
<Button color="lime" radius="md" size="sm" uppercase ripple fullSize on:click={save_settings} disabled={save_button_disabled}>
Сохранить
</Button>
<Space h="sm" />