GitHub/jarvis
1
0
Fork 0
mirror of https://github.com/Priler/jarvis.git synced 2026-06-07 04:49:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

App icon, logs are now stored to log.txt file, work in progress on intergration with rustpotter

Browse Source
This commit is contained in:
Abraham
2023-04-29 14:58:16 +05:00
parent 4e1413d400
commit 5f35beb5cf
72 changed files with 515 additions and 2993 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src-tauri/src/assistant_commands.rs
View File

@@ -8,7 +8,7 @@ use core::time::Duration;
use std::path::PathBuf;
use std::process::Child;
use std::process::Command;
use tauri::Manager;
// use tauri::Manager;
mod structs;
pub use structs::*;

10
src-tauri/src/config.rs
View File

@@ -1,5 +1,4 @@
use const_concat::const_concat;
use std::env::current_dir;
// use const_concat::const_concat;
// pub const IS_DEV: bool = cfg!(debug_assertions);// cfg!(debug_assertions);
// pub const PUBLIC_PATH: &str = if IS_DEV {
@@ -8,14 +7,17 @@ use std::env::current_dir;
// "./public"
// };
pub const COMMANDS_PATH: &str = "commands/";
pub const KEYWORDS_PATH: &str = "picovoice/keywords/";
pub const WAKE_WORD_ENGINES: [&str; 2] = ["rustpotter", "picovoice"];
pub const DB_FILE_NAME: &str = "app.db";
pub const LOG_FILE_NAME: &str = "log.txt";
pub const APP_VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION");
pub const AUTHOR_NAME: Option<&str> = option_env!("CARGO_PKG_AUTHORS");
pub const REPOSITORY_LINK: Option<&str> = option_env!("CARGO_PKG_REPOSITORY");
pub const COMMANDS_PATH: &str = "commands/";
pub const KEYWORDS_PATH: &str = "picovoice/keywords/";
// pub const VOSK_MODEL_PATH: &str = const_concat!(PUBLIC_PATH, "/vosk/model_small");
pub const VOSK_MODEL_PATH: &str = "vosk/model_small";

1
src-tauri/src/events.rs
View File

@@ -6,6 +6,7 @@ pub struct Payload {
pub data: String,
}
#[allow(dead_code)]
pub enum EventTypes {
AudioPlay,
AssistantWaiting,

12
src-tauri/src/main.rs
View File

@@ -4,6 +4,8 @@
#[macro_use]
extern crate lazy_static; // better switch to once_cell ?
use pickledb::{PickleDb, PickleDbDumpPolicy, SerializationMethod};
use log::{info};
use log::LevelFilter;
use std::sync::Mutex;
// expose the config
@@ -23,6 +25,9 @@ mod vosk;
// include events
mod events;
// include recorder
mod recorder;
// app dir
lazy_static! {
static ref APP_CONFIG_DIR: Mutex<String> = Mutex::new(String::new());
@@ -37,7 +42,7 @@ lazy_static! {
SerializationMethod::Json
)
.unwrap_or_else(|_x: _| {
println!("Creating new db file at {} ...", format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME));
info!("Creating new db file at {} ...", format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME));
PickleDb::new(
format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME),
PickleDbDumpPolicy::AutoDump,
@@ -53,8 +58,13 @@ lazy_static! {
}
fn main() {
// log to file
simple_logging::log_to_file(config::LOG_FILE_NAME, LevelFilter::max()).expect("Failed to start logger ... is directory writable?");
// init vosk
vosk::init_vosk();
// run the app
tauri::Builder::default()
.setup(|app| {
std::fs::create_dir_all(app.path_resolver().app_config_dir().unwrap())?;

63
src-tauri/src/recorder.rs Normal file
View File

@@ -0,0 +1,63 @@
use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicU32, AtomicBool, Ordering};
use pv_recorder::{Recorder, RecorderBuilder};
use log::{info};
use crate::DB;
pub static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static RECORDER: OnceCell<Recorder> = OnceCell::new();
pub static IS_RECORDING: AtomicBool = AtomicBool::new(false);
fn init_microphone() {
if RECORDER.get().is_none() {
RECORDER.get_or_init(|| RecorderBuilder::new()
.device_index(get_selected_microphone_index())
.frame_length(FRAME_LENGTH.load(Ordering::SeqCst) as i32)
.init()
.expect("Failed to initialize pvrecorder"));
info!("Microphone recorder initialized!")
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
init_microphone();
// read to frame buffer
RECORDER.get().unwrap().read(frame_buffer).expect("Failed to read audio frame");
}
pub fn start_recording() {
// ensure microphone is initialized
init_microphone();
RECORDER.get().unwrap().start().expect("Failed to start audio recording!");
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
}
pub fn stop_recording() {
// ensure microphone is initialized
init_microphone();
RECORDER.get().unwrap().start().expect("Failed to start audio recording!");
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
pub fn get_selected_microphone_index() -> i32 {
let selected_microphone: i32;
// Retrieve microphone index
if let Some(smic) = DB.lock().unwrap().get::<String>("selected_microphone") {
selected_microphone = smic.parse().unwrap_or(-1);
} else {
selected_microphone = -1;
}
// return microphone index
info!("Selected microphone index = {selected_microphone}");
selected_microphone
}

272
src-tauri/src/tauri_commands/listener.rs
View File

@@ -1,9 +1,9 @@
use porcupine::{BuiltinKeywords, Porcupine, PorcupineBuilder};
use pv_recorder::RecorderBuilder;
use porcupine::{Porcupine, PorcupineBuilder};
use std::sync::atomic::{AtomicBool, Ordering};
use std::path::Path;
use log::{info, warn, error};
use crate::events::Payload;
// use crate::events::Payload;
use tauri::Manager;
use rand::seq::SliceRandom;
@@ -14,6 +14,7 @@ use crate::events;
use crate::config;
use crate::vosk;
use crate::recorder;
use crate::COMMANDS;
use crate::DB;
@@ -46,154 +47,173 @@ pub fn start_listening(app_handle: tauri::AppHandle) -> Result<bool, String> {
return Err("Already listening.".into());
}
// vars
let porcupine: Porcupine;
let mut picovoice_api_key: String = String::from("");
let selected_microphone: i32;
// Retrieve selected wake-word engine from DB
let selected_wake_word_engine;
if let Some(wwengine) = DB.lock().unwrap().get::<String>("selected_wake_word_engine") {
// from db
selected_wake_word_engine = wwengine;
} else {
// default
selected_wake_word_engine = config::WAKE_WORD_ENGINES.first().expect("No wake-word engines found ...").to_string(); // set default wake_word engine
}
let mut start = SystemTime::now();
// call selected wake-word engine listener command
match selected_wake_word_engine.as_str() {
"rustpotter" => {
info!("Starting rustpotter wake-word engine ...");
return picovoice_listen(&app_handle, |_app| {
// Greet user
events::play("run", &app_handle);
}, |app, kidx| keyword_callback(app, kidx));
},
"picovoice" => {
info!("Starting picovoice wake-word engine ...");
return picovoice_listen(&app_handle, |_app| {
// Greet user
events::play("run", &app_handle);
}, |app, kidx| keyword_callback(app, kidx));
},
_ => Err("No wake-word engine selected ...".into())
}
}
pub fn keyword_callback(app_handle: &tauri::AppHandle, _keyword_index: i32) {
// vars
let mut start: SystemTime = SystemTime::now();
let mut frame_buffer = vec![0; recorder::FRAME_LENGTH.load(Ordering::SeqCst) as usize];
// play greet phrase
events::play(
config::ASSISTANT_GREET_PHRASES
.choose(&mut rand::thread_rng())
.unwrap(),
&app_handle,
);
// emit assistant greet event
app_handle
.emit_all(events::EventTypes::AssistantGreet.get(), ())
.unwrap();
// the loop
while !STOP_LISTENING.load(Ordering::SeqCst) {
recorder::read_microphone(&mut frame_buffer);
// vosk part (partials included)
if let Some(mut test) = vosk::recognize(&frame_buffer) {
if !test.is_empty() {
println!("Recognized: {}", test);
// some filtration
test = test.to_lowercase();
for tbr in config::ASSISTANT_PHRASES_TBR {
test = test.replace(tbr, "");
}
// infer command
if let Some((cmd_path, cmd_config)) =
assistant_commands::fetch_command(&test, &COMMANDS)
{
println!("Recognized (filtered): {}", test);
println!("Command found: {:?}", cmd_path);
println!("Executing ...");
let cmd_result = assistant_commands::execute_command(
&cmd_path,
&cmd_config,
&app_handle,
);
match cmd_result {
Ok(_) => {
println!("Command executed successfully!");
start = SystemTime::now(); // listen for more commands
continue;
}
Err(error_message) => {
println!("Error executing command: {}", error_message);
}
}
app_handle
.emit_all(events::EventTypes::AssistantWaiting.get(), ())
.unwrap();
break; // return to picovoice after command execution (no matter successfull or not)
}
}
}
match start.elapsed() {
Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
// return to picovoice after N seconds
app_handle
.emit_all(events::EventTypes::AssistantWaiting.get(), ())
.unwrap();
break;
}
_ => (),
}
}
}
pub fn picovoice_listen<'s, S, K>(app_handle: &tauri::AppHandle, start_callback: S, mut keyword_callback: K) -> Result<bool, String>
where S: Fn(&tauri::AppHandle),
K: FnMut(&tauri::AppHandle, i32) {
// VARS
let porcupine: Porcupine;
let picovoice_api_key: String;
// Retrieve API key from DB
if let Some(pkey) = DB.lock().unwrap().get::<String>("api_key__picovoice") {
if !pkey.is_empty() {
picovoice_api_key = pkey;
}
}
if picovoice_api_key.is_empty() {
picovoice_api_key = pkey;
} else {
warn!("Picovoice API key is not set!");
return Err("Picovoice API key is not set!".into());
}
// Create instance of Porcupine with the given API key
match PorcupineBuilder::new_with_keyword_paths(picovoice_api_key, &[Path::new(config::KEYWORDS_PATH).join("jarvis_windows.ppn")])
.sensitivities(&[1.0f32]) // max sensitivity possible
.init() {
Ok(pinstance) => {
// porcupine successfully initialized with the valid API key
println!("Porcupine successfully initialized with the valid API key ...");
porcupine = pinstance;
}
Err(e) => {
println!("Porcupine error: either API key is not valid or there is no internet connection");
println!("Error details: {}", e);
return Err(
"Porcupine error: either API key is not valid or there is no internet connection"
.into(),
);
}
.sensitivities(&[1.0f32]) // max sensitivity possible
.init() {
Ok(pinstance) => {
// porcupine successfully initialized with the valid API key
info!("Porcupine successfully initialized with the valid API key ...");
porcupine = pinstance;
}
Err(e) => {
error!("Porcupine error: either API key is not valid or there is no internet connection");
error!("Error details: {}", e);
return Err(
"Porcupine error: either API key is not valid or there is no internet connection"
.into(),
);
}
}
// Retrieve microphone index
if let Some(smic) = DB.lock().unwrap().get::<String>("selected_microphone") {
selected_microphone = smic.parse().unwrap_or(-1);
} else {
selected_microphone = -1; // use default, if not selected
}
// Create recorder instance
let recorder = RecorderBuilder::new()
.device_index(selected_microphone)
.frame_length(porcupine.frame_length() as i32)
.init()
.expect("Failed to initialize pvrecorder");
// Start recording
println!("Listening (microphone idx = {selected_microphone}) ...");
recorder.start().expect("Failed to start audio recording");
let mut frame_buffer = vec![0; porcupine.frame_length() as usize];
recorder::FRAME_LENGTH.store(porcupine.frame_length(), Ordering::SeqCst);
recorder::start_recording();
LISTENING.store(true, Ordering::SeqCst);
// Greet user
events::play("run", &app_handle);
// run start callback
start_callback(app_handle);
// Listen until stop flag will be true
let mut frame_buffer = vec![0; porcupine.frame_length() as usize];
while !STOP_LISTENING.load(Ordering::SeqCst) {
recorder
.read(&mut frame_buffer)
.expect("Failed to read audio frame");
recorder::read_microphone(&mut frame_buffer);
if let Ok(keyword_index) = porcupine.process(&frame_buffer) {
if keyword_index >= 0 {
println!("Yes, sir! {}", keyword_index);
events::play(
config::ASSISTANT_GREET_PHRASES
.choose(&mut rand::thread_rng())
.unwrap(),
&app_handle,
);
start = SystemTime::now();
app_handle
.emit_all(events::EventTypes::AssistantGreet.get(), ())
.unwrap();
loop {
recorder
.read(&mut frame_buffer)
.expect("Failed to read audio frame");
// vosk part (partials included)
if let Some(mut test) = vosk::recognize(&frame_buffer) {
if !test.is_empty() {
println!("Recognized: {}", test);
// some filtration
test = test.to_lowercase();
for tbr in config::ASSISTANT_PHRASES_TBR {
test = test.replace(tbr, "");
}
// infer command
if let Some((cmd_path, cmd_config)) =
assistant_commands::fetch_command(&test, &COMMANDS)
{
println!("Recognized (filtered): {}", test);
println!("Command found: {:?}", cmd_path);
println!("Executing ...");
let cmd_result = assistant_commands::execute_command(
&cmd_path,
&cmd_config,
&app_handle,
);
match cmd_result {
Ok(_) => {
println!("Command executed successfully!");
start = SystemTime::now(); // listen for more commands
continue;
}
Err(error_message) => {
println!("Error executing command: {}", error_message);
}
}
app_handle
.emit_all(events::EventTypes::AssistantWaiting.get(), ())
.unwrap();
break; // return to picovoice after command execution (no matter successfull or not)
}
}
}
match start.elapsed() {
Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
// return to picovoice after N seconds
app_handle
.emit_all(events::EventTypes::AssistantWaiting.get(), ())
.unwrap();
break;
}
_ => (),
}
}
// println!("Yes, sir! {}", keyword_index);
keyword_callback(&app_handle, keyword_index);
}
}
}
// Stop listening
println!("Stop listening ...");
recorder.stop().expect("Failed to stop audio recording");
recorder::stop_recording();
LISTENING.store(false, Ordering::SeqCst);
STOP_LISTENING.store(false, Ordering::SeqCst);

2
src-tauri/src/tauri_commands/sys.rs
View File

@@ -6,7 +6,7 @@ static PEAK_ALLOC: PeakAlloc = PeakAlloc;
extern crate systemstat;
use std::thread;
use std::time::Duration;
use systemstat::{saturating_sub_bytes, Platform, System};
use systemstat::{Platform, System};
lazy_static! {
static ref SYS: System = System::new();

20
src-tauri/src/vosk.rs
View File

@@ -1,5 +1,5 @@
use std::sync::Mutex;
use vosk::{CompleteResult, DecodingState, Model, Recognizer};
use vosk::{DecodingState, Model, Recognizer};
use crate::config::VOSK_MODEL_PATH;
@@ -46,13 +46,13 @@ pub fn recognize(data: &[i16]) -> Option<String> {
}
}
pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
let mut result = Vec::with_capacity(input_data.len() / 2);
result.extend(
input_data
.chunks_exact(2)
.map(|chunk| chunk[0] / 2 + chunk[1] / 2),
);
// pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
// let mut result = Vec::with_capacity(input_data.len() / 2);
// result.extend(
// input_data
// .chunks_exact(2)
// .map(|chunk| chunk[0] / 2 + chunk[1] / 2),
// );
result
}
// result
// }