GitHub/jarvis
1
0
Fork 0
mirror of https://github.com/Priler/jarvis.git synced 2026-05-26 23:19:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Latest changes

Browse Source
This commit is contained in:
Priler
2025-12-11 23:43:50 +05:00
parent 15eacbd20f
commit 3c7df5fc6e
26 changed files with 1917 additions and 709 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/.gitignore vendored
View File

@@ -1,3 +1,5 @@
# Generated by Cargo
# will have compiled files and executables
/target/
/target/
*.db
log.txt

1908
app/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

34
app/Cargo.toml
View File

@@ -10,26 +10,28 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
serde = { version = "1.0.164", features = ["derive"] }
serde_json = "1.0.96"
hound = "3.5.0"
pv_recorder = "1.1.2"
pv_porcupine = "2.2.1"
serde = { version = "1.0.197", features = ["derive"] }
serde_json = "1.0.114"
hound = "3.5.1"
pv_recorder = "1.2.2"
pv_porcupine = "3.0.2"
seqdiff = "0.3.0"
vosk = "0.2.0"
rand = "0.8.5"
rodio = "0.17.1"
rustpotter = "2.0.0"
log = "0.4.18"
once_cell = "1.18.0"
atomic_enum = "0.2.0"
rand = "0.9.0-alpha.1"
rodio = "0.17.3"
rustpotter = "3.0.2"
log = "0.4.21"
once_cell = "1.19.0"
atomic_enum = "0.3.0"
portaudio = "0.7.0"
platform-dirs = "0.3.0"
simple-log = "1.6.0"
tray-icon = { version = "0.5.1" }
winit = "0.28.6"
image = "0.24.6"
serde_yaml = "0.9.21"
kira = "0.8.3"
tray-icon = { version = "0.12.0" }
winit = "0.29.15"
image = "0.25.0"
serde_yaml = "0.9.33"
kira = "0.8.7"
[features]
default = ["jarvis_app"]
jarvis_app = [] # feature flag saying this is an app

2
app/post_build.py
View File

@@ -15,6 +15,8 @@ SOURCE = (
"vosk/",
"lib/",
"keywords/",
"rustpotter/",
"sound/",
"libgcc_s_seh-1.dll",
"libstdc++-6.dll",
"libvosk.dll",

36
app/src/app.rs
View File

@@ -1,7 +1,7 @@
use std::time::SystemTime;
use crate::{config, audio, recorder, listener, stt, commands, COMMANDS_LIST};
use rand::seq::SliceRandom;
use crate::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST};
use rand::seq::IndexedRandom;
pub fn start() -> Result<(), ()> {
// start the loop
@@ -41,7 +41,12 @@ fn main_loop() -> Result<(), ()> {
// play some greet phrase
// @TODO. Make it via commands or upcoming events system.
audio::play_sound(&sounds_directory.join(format!("{}.wav", config::ASSISTANT_GREET_PHRASES.choose(&mut rand::thread_rng()).unwrap())));
audio::play_sound(&sounds_directory.join(format!(
"{}.wav",
config::ASSISTANT_GREET_PHRASES
.choose(&mut rand::thread_rng())
.unwrap()
)));
// wait for voice commands
'voice_recognition: loop {
@@ -62,7 +67,10 @@ fn main_loop() -> Result<(), ()> {
recognized_voice = recognized_voice.trim().into();
// infer command
if let Some((cmd_path, cmd_config)) = commands::fetch_command(&recognized_voice, &COMMANDS_LIST.get().unwrap()) {
if let Some((cmd_path, cmd_config)) = commands::fetch_command(
&recognized_voice,
&COMMANDS_LIST.get().unwrap(),
) {
// some debug info
info!("Recognized voice (filtered): {}", recognized_voice);
info!("Command found: {:?}", cmd_path);
@@ -79,11 +87,13 @@ fn main_loop() -> Result<(), ()> {
start = SystemTime::now();
} else {
// skip, if chaining is not required
start = start.checked_sub(core::time::Duration::from_secs(1000)).unwrap();
start = start
.checked_sub(core::time::Duration::from_secs(1000))
.unwrap();
}
continue 'voice_recognition; // continue voice recognition
},
}
Err(msg) => {
// fail
error!("Error executing command: {}", msg);
@@ -100,23 +110,21 @@ fn main_loop() -> Result<(), ()> {
Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
// return to wake-word listening after N seconds
break 'voice_recognition;
},
_ => ()
}
_ => (),
}
}
},
None => ()
}
None => (),
}
}
Ok(())
}
fn keyword_callback(keyword_index: i32) {
}
fn keyword_callback(keyword_index: i32) {}
pub fn close(code: i32) {
info!("Closing application.");
std::process::exit(code);
}
}

32
app/src/audio.rs
View File

@@ -1,18 +1,19 @@
mod rodio;
mod kira;
mod rodio;
use once_cell::sync::OnceCell;
use std::cmp::Ordering;
use std::path::PathBuf;
use once_cell::sync::OnceCell;
use crate::{config, DB, SOUND_DIR};
use crate::config::structs::AudioType;
use crate::{config, DB, SOUND_DIR};
static AUDIO_TYPE: OnceCell<AudioType> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !AUDIO_TYPE.get().is_none() {return Ok(());} // already initialized
if !AUDIO_TYPE.get().is_none() {
return Ok(());
} // already initialized
// set default audio type
// @TODO. Make it configurable?
@@ -27,14 +28,14 @@ pub fn init() -> Result<(), ()> {
match rodio::init() {
Ok(_) => {
info!("Successfully initialized Rodio audio backend.");
},
}
Err(msg) => {
error!("Failed to initialize Rodio audio backend.");
return Err(())
return Err(());
}
}
},
}
AudioType::Kira => {
// Init Kira
info!("Initializing Kira audio backend.");
@@ -42,11 +43,11 @@ pub fn init() -> Result<(), ()> {
match kira::init() {
Ok(_) => {
info!("Successfully initialized Kira audio backend.");
},
}
Err(msg) => {
error!("Failed to initialize Kira audio backend.");
return Err(())
return Err(());
}
}
}
@@ -61,10 +62,8 @@ pub fn play_sound(filename: &PathBuf) {
match AUDIO_TYPE.get().unwrap() {
AudioType::Rodio => {
rodio::play_sound(filename, true);
},
AudioType::Kira => {
kira::play_sound(filename)
}
AudioType::Kira => kira::play_sound(filename),
}
}
@@ -79,8 +78,11 @@ pub fn get_sound_directory() -> Option<PathBuf> {
match default_voice_path.exists() {
true => Some(default_voice_path),
_ => None
_ => {
error!("No sounds found. Search path - {:?}", voice_path);
None
}
}
}
}
}
}

19
app/src/audio/kira.rs
View File

@@ -1,20 +1,19 @@
use once_cell::sync::OnceCell;
use std::path::PathBuf;
use std::sync::Mutex;
use once_cell::sync::OnceCell;
use kira::{
manager::{
AudioManager, AudioManagerSettings,
backend::DefaultBackend,
},
sound::static_sound::{StaticSoundData, StaticSoundSettings},
manager::{backend::DefaultBackend, AudioManager, AudioManagerSettings},
sound::static_sound::{StaticSoundData, StaticSoundSettings},
};
thread_local!(static MANAGER: OnceCell<Mutex<AudioManager>> = OnceCell::new());
pub fn init() -> Result<(), ()> {
MANAGER.with(|m| {
if !m.get().is_none() {return Ok(());} // already initialized
if !m.get().is_none() {
return Ok(());
} // already initialized
// Create an audio manager. This plays sounds and manages resources.
match AudioManager::<DefaultBackend>::new(AudioManagerSettings::default()) {
@@ -24,7 +23,7 @@ pub fn init() -> Result<(), ()> {
// success
Ok(())
},
}
Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg);
@@ -47,9 +46,9 @@ pub fn play_sound(filename: &PathBuf) {
let audio_manager = &mut m.get().unwrap().lock().unwrap();
audio_manager.play(sound_data.clone()).unwrap();
});
},
}
Err(msg) => {
warn!("Cannot find sound file: {}", filename.display());
}
}
}
}

18
app/src/audio/rodio.rs
View File

@@ -4,10 +4,10 @@
Possible fixes are running rodio in a separate thread or smthng.
*/
use std::fs::File;
use std::path::PathBuf;
use std::io::BufReader;
use once_cell::sync::OnceCell;
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink};
@@ -16,7 +16,9 @@ static STREAM_HANDLE: OnceCell<OutputStreamHandle> = OnceCell::new();
static SINK: OnceCell<Sink> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !STREAM_HANDLE.get().is_none() {return Ok(());} // already initialized
if !STREAM_HANDLE.get().is_none() {
return Ok(());
} // already initialized
// get output stream handle to the default physical sound device
match OutputStream::try_default() {
@@ -30,12 +32,12 @@ pub fn init() -> Result<(), ()> {
Ok(s) => {
info!("Sink initialized.");
sink = s;
},
}
Err(msg) => {
error!("Cannot create sink.\nError details: {}", msg);
// failed
return Err(())
return Err(());
}
}
@@ -46,7 +48,7 @@ pub fn init() -> Result<(), ()> {
// success
Ok(())
},
}
Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg);
@@ -73,4 +75,4 @@ pub fn play_sound(filename: &PathBuf, sleep: bool) {
// has finished playing all its queued sounds.
SINK.get().unwrap().sleep_until_end();
}
}
}

92
app/src/commands.rs
View File

@@ -1,4 +1,4 @@
use rand::seq::SliceRandom;
use rand::seq::IndexedRandom;
use seqdiff::ratio;
use serde_yaml;
use std::path::Path;
@@ -6,13 +6,13 @@ use std::{fs, fs::File};
use core::time::Duration;
use std::path::PathBuf;
use std::process::{Command, Child};
use std::process::{Child, Command};
// use tauri::Manager;
mod structs;
pub use structs::*;
use crate::{config, audio};
use crate::{audio, config};
// @TODO. Allow commands both in yaml and json format.
pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
@@ -35,9 +35,13 @@ pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
match serde_yaml::from_reader::<File, CommandsList>(cc_reader) {
Ok(parse_result) => {
cc_yaml = parse_result;
},
}
Err(msg) => {
warn!("Can't parse {}, skipping ...\nCommand parse error is: {:?}", &cc_file.display(), msg);
warn!(
"Can't parse {}, skipping ...\nCommand parse error is: {:?}",
&cc_file.display(),
msg
);
continue;
}
}
@@ -98,7 +102,10 @@ pub fn fetch_command<'a>(
if let Some((cmd_path, scmd)) = result_scmd {
println!("Ratio is: {}", current_max_ratio);
info!("CMD is: {cmd_path:?}, SCMD is: {scmd:?}, Ratio is: {}", current_max_ratio);
info!(
"CMD is: {cmd_path:?}, SCMD is: {scmd:?}, Ratio is: {}",
current_max_ratio
);
Some((&cmd_path, &scmd))
} else {
None
@@ -111,21 +118,12 @@ pub fn execute_exe(exe: &str, args: &Vec<String>) -> std::io::Result<Child> {
}
pub fn execute_cli(cmd: &str, args: &Vec<String>) -> std::io::Result<Child> {
println!("Spawning cmd as: cmd /C {} {:?}", cmd, args);
if cfg!(target_os = "windows") {
Command::new("cmd")
.arg("/C")
.arg(cmd)
.args(args)
.spawn()
Command::new("cmd").arg("/C").arg(cmd).args(args).spawn()
} else {
Command::new("sh")
.arg("-c")
.arg(cmd)
.args(args)
.spawn()
Command::new("sh").arg("-c").arg(cmd).args(args).spawn()
}
}
@@ -139,7 +137,14 @@ pub fn execute_command(
match cmd_config.command.action.as_str() {
"voice" => {
// VOICE command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
@@ -158,7 +163,14 @@ pub fn execute_command(
},
&cmd_config.command.exe_args,
) {
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
@@ -172,17 +184,21 @@ pub fn execute_command(
// CLI command type
let cli_cmd = &cmd_config.command.cli_cmd;
match execute_cli(
cli_cmd,
&cmd_config.command.cli_args,
) {
Ok(_) => {
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
match execute_cli(cli_cmd, &cmd_config.command.cli_args) {
Ok(_) => {
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(true)
},
}
Err(msg) => {
error!("CLI command error ({})", msg);
Err(format!("Shell command error ({})", msg).into())
@@ -191,7 +207,14 @@ pub fn execute_command(
}
"terminate" => {
// TERMINATE command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
@@ -200,7 +223,14 @@ pub fn execute_command(
}
"stop_chaining" => {
// STOP_CHAINING command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
@@ -209,7 +239,7 @@ pub fn execute_command(
_ => {
error!("Command type unknown");
Err("Command type unknown".into())
},
}
}
}
@@ -222,4 +252,4 @@ pub fn list(from: &[AssistantCommand]) -> Vec<String> {
}
out
}
}

3
app/src/commands/structs.rs
View File

@@ -35,12 +35,11 @@ pub struct ConfigCommandSection {
pub cli_cmd: String,
#[serde(default)]
pub cli_args: Vec<String>
pub cli_args: Vec<String>,
}
#[derive(Deserialize, Debug)]
pub struct ConfigVoiceSection {
#[serde(default)]
pub sounds: Vec<String>,
}

58
app/src/config.rs
View File

@@ -1,21 +1,25 @@
pub mod structs;
use structs::WakeWordEngine;
use structs::SpeechToTextEngine;
use structs::RecorderType;
use structs::AudioType;
use structs::RecorderType;
use structs::SpeechToTextEngine;
use structs::WakeWordEngine;
use std::fs;
use std::env;
use std::path::PathBuf;
use once_cell::sync::Lazy;
use std::env;
use std::fs;
use std::path::PathBuf;
use platform_dirs::{AppDirs};
use rustpotter::{RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig};
use platform_dirs::AppDirs;
use crate::{config, APP_DIRS, APP_CONFIG_DIR, APP_LOG_DIR};
#[cfg(feature="jarvis_app")]
use rustpotter::{
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig,
RustpotterConfig, ScoreMode,
};
use crate::{config, APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
#[allow(dead_code)]
pub fn init_dirs() -> Result<(), String> {
// infer app dirs
if APP_DIRS.get().is_some() {
@@ -23,7 +27,9 @@ pub fn init_dirs() -> Result<(), String> {
}
// cache_dir, config_dir, data_dir, state_dir
APP_DIRS.set(AppDirs::new(Some(config::BUNDLE_IDENTIFIER), false).unwrap()).unwrap();
APP_DIRS
.set(AppDirs::new(Some(config::BUNDLE_IDENTIFIER), false).unwrap())
.unwrap();
// setup directories
let mut config_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
@@ -33,7 +39,8 @@ pub fn init_dirs() -> Result<(), String> {
if !config_dir.exists() {
if fs::create_dir_all(&config_dir).is_err() {
config_dir = env::current_dir().expect("Cannot infer the config directory");
fs::create_dir_all(&config_dir).expect("Cannot create config directory, access denied?");
fs::create_dir_all(&config_dir)
.expect("Cannot create config directory, access denied?");
}
}
@@ -51,10 +58,9 @@ pub fn init_dirs() -> Result<(), String> {
Ok(())
}
/*
Defaults.
*/
Defaults.
*/
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter;
@@ -72,23 +78,29 @@ pub const TG_OFFICIAL_LINK: Option<&str> = Some("https://t.me/howdyho_official")
pub const FEEDBACK_LINK: Option<&str> = Some("https://t.me/jarvis_feedback_bot");
/*
Tray.
*/
Tray.
*/
pub const TRAY_ICON: &str = "32x32.png";
pub const TRAY_TOOLTIP: &str = "Jarvis Voice Assistant";
// RUSPOTTER
#[cfg(feature="jarvis_app")]
pub const RUSPOTTER_MIN_SCORE: f32 = 0.62;
#[cfg(feature="jarvis_app")]
pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
RustpotterConfig {
fmt: WavFmt::default(),
fmt: AudioFmt::default(),
detector: DetectorConfig {
avg_threshold: 0.,
threshold: 0.5,
min_scores: 15,
score_mode: ScoreMode::Average,
comparator_band_size: 5,
comparator_ref: 0.22
score_ref: 0.22,
band_size: 5,
vad_mode: None,
score_mode: ScoreMode::Max,
eager: false,
// comparator_band_size: 5,
// comparator_ref: 0.22
},
filters: FiltersConfig {
gain_normalizer: GainNormalizationConfig {
@@ -101,8 +113,8 @@ pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
enabled: true,
low_cutoff: 80.,
high_cutoff: 400.,
}
}
},
},
}
});

23
app/src/config/structs.rs
View File

@@ -1,30 +1,43 @@
use std::fmt;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
pub enum WakeWordEngine {
Rustpotter,
Vosk,
Porcupine
Porcupine,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[derive(Serialize, Deserialize, Debug)]
pub enum SpeechToTextEngine {
Vosk
Vosk,
}
impl fmt::Display for SpeechToTextEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[derive(PartialEq, Debug)]
pub enum RecorderType {
Cpal,
PvRecorder,
PortAudio
PortAudio,
}
#[derive(PartialEq, Debug)]
pub enum AudioType {
Rodio,
Kira
Kira,
}
// pub enum TextToSpeechEngine {}
// pub enum IntentRecognitionEngine {}
// pub enum IntentRecognitionEngine {}

19
app/src/db.rs
View File

@@ -1,22 +1,29 @@
pub mod structs;
use crate::{config, APP_CONFIG_DIR};
use std::path::PathBuf;
use log::info;
use std::fs::File;
use std::io::{BufReader, Read};
use log::info;
use std::path::PathBuf;
use serde_json;
fn get_db_file_path() -> PathBuf {
PathBuf::from(format!("{}/{}", APP_CONFIG_DIR.get().unwrap().display(), config::DB_FILE_NAME))
PathBuf::from(format!(
"{}/{}",
APP_CONFIG_DIR.get().unwrap().display(),
config::DB_FILE_NAME
))
}
pub fn init_settings() -> structs::Settings {
let mut db = None;
let db_file_path = get_db_file_path();
info!("Loading settings db file located at: {}", db_file_path.display());
info!(
"Loading settings db file located at: {}",
db_file_path.display()
);
if db_file_path.exists() {
// try load existing settings
@@ -43,9 +50,9 @@ pub fn save_settings(settings: &structs::Settings) -> Result<(), std::io::Error>
std::fs::write(
db_file_path,
serde_json::to_string_pretty(&settings).unwrap()
serde_json::to_string_pretty(&settings).unwrap(),
)?;
info!("Settings saved.");
Ok(())
}
}

14
app/src/db/structs.rs
View File

@@ -1,8 +1,8 @@
use serde::{Deserialize, Serialize};
use crate::config;
use serde::{Deserialize, Serialize};
use crate::config::structs::WakeWordEngine;
use crate::config::structs::SpeechToTextEngine;
use crate::config::structs::WakeWordEngine;
#[derive(Serialize, Deserialize, Debug)]
pub struct Settings {
@@ -12,7 +12,7 @@ pub struct Settings {
pub wake_word_engine: WakeWordEngine,
pub speech_to_text_engine: SpeechToTextEngine,
pub api_keys: ApiKeys
pub api_keys: ApiKeys,
}
impl Default for Settings {
@@ -26,8 +26,8 @@ impl Default for Settings {
api_keys: ApiKeys {
picovoice: String::from(""),
openai: String::from("")
}
openai: String::from(""),
},
}
}
}
@@ -35,5 +35,5 @@ impl Default for Settings {
#[derive(Serialize, Deserialize, Debug)]
pub struct ApiKeys {
pub picovoice: String,
pub openai: String
}
pub openai: String,
}

30
app/src/listener.rs
View File

@@ -7,8 +7,8 @@ mod vosk;
use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::{config, stt};
use crate::config::structs::WakeWordEngine;
use crate::{config, stt};
use crate::DB;
@@ -19,10 +19,14 @@ static WAKE_WORD_ENGINE: OnceCell<WakeWordEngine> = OnceCell::new();
static LISTENING: AtomicBool = AtomicBool::new(false);
pub fn init() -> Result<(), ()> {
if !WAKE_WORD_ENGINE.get().is_none() {return Ok(());} // already initialized
if !WAKE_WORD_ENGINE.get().is_none() {
return Ok(());
} // already initialized
// store current engine
WAKE_WORD_ENGINE.set(DB.get().unwrap().wake_word_engine).unwrap();
WAKE_WORD_ENGINE
.set(DB.get().unwrap().wake_word_engine)
.unwrap();
// load given wake-word engine
match WAKE_WORD_ENGINE.get().unwrap() {
@@ -31,33 +35,27 @@ pub fn init() -> Result<(), ()> {
info!("Initializing Porcupine wake-word engine.");
return porcupine::init();
},
}
WakeWordEngine::Rustpotter => {
// Init Rustpotter wake-word engine
info!("Initializing Rustpotter wake-word engine.");
return rustpotter::init();
},
}
WakeWordEngine::Vosk => {
// Init Vosk as wake-word engine (very slow, though)
info!("Initializing Vosk as wake-word engine.");
warn!("Using Vosk as wake-word engine is highly not recommended, because it's very slow for this task.");
return vosk::init();
},
}
}
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
match WAKE_WORD_ENGINE.get().unwrap() {
WakeWordEngine::Porcupine => {
porcupine::data_callback(frame_buffer)
},
WakeWordEngine::Rustpotter => {
rustpotter::data_callback(frame_buffer)
},
WakeWordEngine::Vosk => {
vosk::data_callback(frame_buffer)
}
WakeWordEngine::Porcupine => porcupine::data_callback(frame_buffer),
WakeWordEngine::Rustpotter => rustpotter::data_callback(frame_buffer),
WakeWordEngine::Vosk => vosk::data_callback(frame_buffer),
}
}
}

40
app/src/listener/porcupine.rs
View File

@@ -3,8 +3,8 @@ use std::path::Path;
use once_cell::sync::OnceCell;
use porcupine::{Porcupine, PorcupineBuilder};
use crate::DB;
use crate::config;
use crate::DB;
// store porcupine instance
static PORCUPINE: OnceCell<Porcupine> = OnceCell::new();
@@ -16,26 +16,30 @@ pub fn init() -> Result<(), ()> {
picovoice_api_key = DB.get().unwrap().api_keys.picovoice.clone();
if picovoice_api_key.trim().is_empty() {
warn!("Picovoice API key is not set.");
return Err(())
return Err(());
}
// create porcupine instance with the given API key
match PorcupineBuilder::new_with_keyword_paths(picovoice_api_key, &[Path::new(config::KEYWORDS_PATH).join(config::DEFAULT_KEYWORD)])
.sensitivities(&[config::DEFAULT_SENSITIVITY]) // set sensitivity
.init() {
Ok(pinstance) => {
// success
info!("Porcupine successfully initialized with the given API key.");
match PorcupineBuilder::new_with_keyword_paths(
picovoice_api_key,
&[Path::new(config::KEYWORDS_PATH).join(config::DEFAULT_KEYWORD)],
)
.sensitivities(&[config::DEFAULT_SENSITIVITY]) // set sensitivity
.init()
{
Ok(pinstance) => {
// success
info!("Porcupine successfully initialized with the given API key.");
// store
PORCUPINE.set(pinstance);
},
Err(msg) => {
error!("Porcupine failed to initialize, either API key is not valid or there is no internet connection.");
error!("Error details: {}", msg);
// store
PORCUPINE.set(pinstance);
}
Err(msg) => {
error!("Porcupine failed to initialize, either API key is not valid or there is no internet connection.");
error!("Error details: {}", msg);
return Err(());
}
return Err(());
}
}
Ok(())
@@ -44,9 +48,9 @@ pub fn init() -> Result<(), ()> {
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) {
if keyword_index >= 0 {
return Some(keyword_index)
return Some(keyword_index);
}
}
None
}
}

27
app/src/listener/rustpotter.rs
View File

@@ -2,10 +2,13 @@ use std::path::Path;
use std::sync::Mutex;
use once_cell::sync::OnceCell;
use rustpotter::{Rustpotter, RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig};
use rustpotter::{
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig, Rustpotter,
RustpotterConfig, ScoreMode,
};
use crate::DB;
use crate::config;
use crate::DB;
// store rustpotter instance
static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
@@ -19,23 +22,23 @@ pub fn init() -> Result<(), ()> {
// success
// wake word files list
// @TODO. Make it configurable via GUI for custom user voice.
let rustpotter_wake_word_files: [&str; 5] = [
let rustpotter_wake_word_files: [&str; 1] = [
"rustpotter/jarvis-default.rpw",
"rustpotter/jarvis-community-1.rpw",
"rustpotter/jarvis-community-2.rpw",
"rustpotter/jarvis-community-3.rpw",
"rustpotter/jarvis-community-4.rpw",
// "rustpotter/jarvis-community-1.rpw",
// "rustpotter/jarvis-community-2.rpw",
// "rustpotter/jarvis-community-3.rpw",
// "rustpotter/jarvis-community-4.rpw",
// "rustpotter/jarvis-community-5.rpw",
];
// load wake word files
for rpw in rustpotter_wake_word_files {
rinstance.add_wakeword_from_file(rpw).unwrap();
rinstance.add_wakeword_from_file(rpw, rpw).unwrap(); // @TODO: Change wakeword key to something else?
}
// store
RUSTPOTTER.set(Mutex::new(rinstance));
},
}
Err(msg) => {
error!("Rustpotter failed to initialize.\nError details: {}", msg);
@@ -49,17 +52,17 @@ pub fn init() -> Result<(), ()> {
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
let mut lock = RUSTPOTTER.get().unwrap().lock();
let rustpotter = lock.as_mut().unwrap();
let detection = rustpotter.process_i16(&frame_buffer);
let detection = rustpotter.process_samples(frame_buffer.to_vec()); // @TODO. Temp crutch. Fix optimization issue, frame_buffer should not be copied to a new vector!
if let Some(detection) = detection {
if detection.score > config::RUSPOTTER_MIN_SCORE {
info!("Rustpotter detection info:\n{:?}", detection);
return Some(0)
return Some(0);
} else {
info!("Rustpotter detection info:\n{:?}", detection)
}
}
None
}
}

9
app/src/listener/vosk.rs
View File

@@ -17,17 +17,20 @@ pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
// compare
let compare_ratio = seqdiff::ratio(&config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(), &recognized_phrase_chars);
let compare_ratio = seqdiff::ratio(
&config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(),
&recognized_phrase_chars,
);
info!("og phrase: {:?}", &config::VOSK_FETCH_PHRASE);
info!("recognized phrase: {:?}", &recognized_phrase_chars);
info!("compare ratio: {}", compare_ratio);
if compare_ratio >= config::VOSK_MIN_RATIO {
info!("Phrase activated.");
return Some(0)
return Some(0);
}
}
}
None
}
}

8
app/src/log.rs
View File

@@ -6,7 +6,11 @@ use crate::APP_LOG_DIR;
pub fn init_logging() -> Result<(), String> {
// configure logging
let config = LogConfigBuilder::builder()
.path(format!("{}/{}", APP_LOG_DIR.get().unwrap().display(), config::LOG_FILE_NAME))
.path(format!(
"{}/{}",
APP_LOG_DIR.get().unwrap().display(),
config::LOG_FILE_NAME
))
.size(1 * 100)
.roll_count(10)
.time_format("%Y-%m-%d %H:%M:%S.%f") //E.g:%H:%M:%S.%f
@@ -18,4 +22,4 @@ pub fn init_logging() -> Result<(), String> {
simple_log::new(config)?;
Ok(())
}
}

21
app/src/main.rs
View File

@@ -3,7 +3,7 @@ use std::error::Error;
use std::path::PathBuf;
use once_cell::sync::{Lazy, OnceCell};
use platform_dirs::{AppDirs};
use platform_dirs::AppDirs;
// expose the config
mod config;
@@ -35,8 +35,8 @@ mod stt;
// include commands
mod commands;
use commands::AssistantCommand;
use crate::commands::list;
use commands::AssistantCommand;
// include audio
mod audio;
@@ -45,8 +45,8 @@ mod audio;
mod listener;
// some global data
static APP_DIR: Lazy<PathBuf> = Lazy::new(|| {env::current_dir().unwrap()});
static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| {APP_DIR.clone().join("sound")});
static APP_DIR: Lazy<PathBuf> = Lazy::new(|| env::current_dir().unwrap());
static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| APP_DIR.clone().join("sound"));
static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
@@ -62,7 +62,10 @@ fn main() -> Result<(), String> {
// log some base info
info!("Starting Jarvis v{} ...", config::APP_VERSION.unwrap());
info!("Config directory is: {}", APP_CONFIG_DIR.get().unwrap().display());
info!(
"Config directory is: {}",
APP_CONFIG_DIR.get().unwrap().display()
);
info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display());
// initialize database (settings)
@@ -93,7 +96,11 @@ fn main() -> Result<(), String> {
// init commands
info!("Initializing commands.");
let commands = commands::parse_commands().unwrap();
info!("Commands initialized.\nOverall commands parsed: {}\nParsed commands: {:?}", commands.len(), commands::list(&commands));
info!(
"Commands initialized.\nOverall commands parsed: {}\nParsed commands: {:?}",
commands.len(),
commands::list(&commands)
);
COMMANDS_LIST.set(commands).unwrap();
// init audio
@@ -111,4 +118,4 @@ fn main() -> Result<(), String> {
app::start();
Ok(())
}
}

71
app/src/recorder.rs
View File

@@ -4,7 +4,7 @@ mod pvrecorder;
use once_cell::sync::OnceCell;
use crate::{DB, config, config::structs::RecorderType};
use crate::{config, config::structs::RecorderType, DB};
static RECORDER_TYPE: OnceCell<RecorderType> = OnceCell::new();
static FRAME_LENGTH: OnceCell<u32> = OnceCell::new();
@@ -14,23 +14,35 @@ pub fn init() -> Result<(), ()> {
// @TODO. Make it configurable?
RECORDER_TYPE.set(config::DEFAULT_RECORDER_TYPE).unwrap();
// some info
info!("Loading recorder ...");
info!("Available audio_devices are:\n{:?}", get_audio_devices());
// load given recorder
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
// Init Pv Recorder
info!("Initializing PvRecorder recording backend.");
FRAME_LENGTH.set(512u32).unwrap(); // pvrecorder requires frame buffer of 512
match pvrecorder::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.get().unwrap().to_owned()) {
let selected_microphone = get_selected_microphone_index();
match pvrecorder::init_microphone(
selected_microphone,
FRAME_LENGTH.get().unwrap().to_owned(),
) {
false => {
error!("Recorder initialization failed.");
return Err(())
},
return Err(());
}
_ => {
info!("Recorder initialization success.");
info!(
"Recorder initialization success. Listening to microphone ({}): {}",
selected_microphone,
get_audio_device_name(selected_microphone)
);
}
}
},
}
RecorderType::PortAudio => {
// Init PortAudio
info!("Initializing PortAudio recording backend");
@@ -42,7 +54,7 @@ pub fn init() -> Result<(), ()> {
// },
// _ => ()
// }
},
}
RecorderType::Cpal => {
// Init CPAL
info!("Initializing CPAL recording backend");
@@ -64,11 +76,11 @@ pub fn read_microphone(frame_buffer: &mut [i16]) {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
pvrecorder::read_microphone(frame_buffer);
},
}
RecorderType::PortAudio => {
todo!();
// portaudio::read_microphone(frame_buffer);
},
}
RecorderType::Cpal => {
// cpal::read_microphone(frame_buffer);
panic!("Cpal should be used via callback assignment");
@@ -79,12 +91,15 @@ pub fn read_microphone(frame_buffer: &mut [i16]) {
pub fn start_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
return pvrecorder::start_recording(get_selected_microphone_index(), FRAME_LENGTH.get().unwrap().to_owned());
},
return pvrecorder::start_recording(
get_selected_microphone_index(),
FRAME_LENGTH.get().unwrap().to_owned(),
);
}
RecorderType::PortAudio => {
todo!();
// portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
},
}
RecorderType::Cpal => {
todo!();
// cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
@@ -94,13 +109,11 @@ pub fn start_recording() -> Result<(), ()> {
pub fn stop_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
pvrecorder::stop_recording()
},
RecorderType::PvRecorder => pvrecorder::stop_recording(),
RecorderType::PortAudio => {
todo!();
// portaudio::stop_recording();
},
}
RecorderType::Cpal => {
todo!();
// cpal::stop_recording();
@@ -110,4 +123,28 @@ pub fn stop_recording() -> Result<(), ()> {
pub fn get_selected_microphone_index() -> i32 {
DB.get().unwrap().microphone
}
}
pub fn get_audio_devices() -> Vec<String> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => pvrecorder::list_audio_devices(),
RecorderType::PortAudio => {
todo!();
}
RecorderType::Cpal => {
todo!();
}
}
}
pub fn get_audio_device_name(idx: i32) -> String {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => pvrecorder::get_audio_device_name(idx),
RecorderType::PortAudio => {
todo!();
}
RecorderType::Cpal => {
todo!();
}
}
}

61
app/src/recorder/pvrecorder.rs
View File

@@ -1,16 +1,16 @@
use once_cell::sync::OnceCell;
use pv_recorder::{PvRecorder, PvRecorderBuilder};
use std::sync::atomic::{AtomicBool, Ordering};
use pv_recorder::{Recorder, RecorderBuilder};
static RECORDER: OnceCell<Recorder> = OnceCell::new();
static RECORDER: OnceCell<PvRecorder> = OnceCell::new();
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
match RECORDER.get().is_none() {
true => {
let pv_recorder = RecorderBuilder::new()
let pv_recorder = PvRecorderBuilder::new(frame_length as i32)
.device_index(device_index)
.frame_length(frame_length as i32)
// .frame_length(frame_length as i32)
.init();
match pv_recorder {
@@ -20,7 +20,7 @@ pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
// success
true
},
}
Err(msg) => {
error!("Failed to initialize pvrecorder.\nError details: {:?}", msg);
@@ -28,8 +28,8 @@ pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
false
}
}
},
_ => true // already initialized
}
_ => true, // already initialized
}
}
@@ -37,12 +37,17 @@ pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
if !RECORDER.get().is_none() {
// read to frame buffer
match RECORDER.get().unwrap().read(frame_buffer) {
let frame = RECORDER.get().unwrap().read();
match frame {
Ok(f) => {
frame_buffer.copy_from_slice(f.as_slice());
}
Err(msg) => {
// @TODO: Fix? PvRecorder always wait for PCM buffer size of 512.
error!("Failed to read audio frame. {:?}", msg);
},
_ => ()
}
}
}
}
@@ -61,7 +66,7 @@ pub fn start_recording(device_index: i32, frame_length: u32) -> Result<(), ()> {
// success
Ok(())
},
}
Err(msg) => {
error!("Failed to start audio recording!");
@@ -83,16 +88,42 @@ pub fn stop_recording() -> Result<(), ()> {
IS_RECORDING.store(false, Ordering::SeqCst);
// success
return Ok(())
},
return Ok(());
}
Err(msg) => {
error!("Failed to stop audio recording!");
// fail
return Err(())
return Err(());
}
}
}
Ok(()) // if already stopped or not yet initialized
}
}
pub fn list_audio_devices() -> Vec<String> {
let audio_devices = PvRecorderBuilder::default().get_available_devices();
match audio_devices {
Ok(audio_devices) => audio_devices,
Err(err) => panic!("Failed to get audio devices: {}", err),
}
}
pub fn get_audio_device_name(idx: i32) -> String {
let audio_devices = list_audio_devices();
let mut first_device: String = String::new();
for (_idx, device) in audio_devices.iter().enumerate() {
if idx as usize == _idx {
return device.to_string();
}
if _idx == 0 {
first_device = device.to_string()
}
}
// return first device as default, if none were matched
first_device
}

12
app/src/stt.rs
View File

@@ -1,14 +1,16 @@
mod vosk;
use once_cell::sync::OnceCell;
use crate::config;
use once_cell::sync::OnceCell;
use crate::config::structs::SpeechToTextEngine;
static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !STT_TYPE.get().is_none() {return Ok(());} // already initialized
if !STT_TYPE.get().is_none() {
return Ok(());
} // already initialized
// set default stt type
// @TODO. Make it configurable?
@@ -29,8 +31,6 @@ pub fn init() -> Result<(), ()> {
pub fn recognize(data: &[i16], partial: bool) -> Option<String> {
match STT_TYPE.get().unwrap() {
SpeechToTextEngine::Vosk => {
vosk::recognize(data, partial)
}
SpeechToTextEngine::Vosk => vosk::recognize(data, partial),
}
}
}

28
app/src/stt/vosk.rs
View File

@@ -9,7 +9,9 @@ static MODEL: OnceCell<Model> = OnceCell::new();
static RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
pub fn init_vosk() {
if !RECOGNIZER.get().is_none() {return;} // already initialized
if !RECOGNIZER.get().is_none() {
return;
} // already initialized
let model = Model::new(VOSK_MODEL_PATH).unwrap();
let mut recognizer = Recognizer::new(&model, 16000.0).unwrap();
@@ -23,12 +25,26 @@ pub fn init_vosk() {
}
pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
let state = RECOGNIZER.get().unwrap().lock().unwrap().accept_waveform(data);
let state = RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.accept_waveform(data);
match state {
DecodingState::Running => {
if include_partial {
Some(RECOGNIZER.get().unwrap().lock().unwrap().partial_result().partial.into())
Some(
RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.partial_result()
.partial
.into(),
)
} else {
None
}
@@ -36,7 +52,11 @@ pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
DecodingState::Finalized => {
// Result will always be multiple because we called set_max_alternatives
Some(
RECOGNIZER.get().unwrap().lock().unwrap()
RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.result()
.multiple()
.unwrap()

51
app/src/tray.rs
View File

@@ -1,11 +1,11 @@
mod menu;
use image;
use tray_icon::{
menu::{AboutMetadata, Menu, MenuEvent, MenuItem, PredefinedMenuItem},
TrayEvent, TrayIconBuilder,
TrayIconBuilder, TrayIconEvent,
};
use winit::event_loop::{ControlFlow, EventLoopBuilder};
use image;
use winit::platform::windows::EventLoopBuilderExtWindows;
use crate::config;
@@ -15,7 +15,6 @@ pub fn init() {
// New thread will prevent tray icon to work on MacOS
// @TODO: MacOS support.
std::thread::spawn(|| {
// load tray icon
let icon_path = format!("{}/icons/{}", env!("CARGO_MANIFEST_DIR"), config::TRAY_ICON);
let icon = load_icon(std::path::Path::new(&icon_path));
@@ -24,8 +23,9 @@ pub fn init() {
let tray_menu = Menu::with_items(&[
&MenuItem::new("Перезапуск", true, None),
&MenuItem::new("Настройки", true, None),
&MenuItem::new("Выход", true, None)
]);
&MenuItem::new("Выход", true, None),
])
.unwrap();
#[cfg(not(target_os = "linux"))]
let mut tray_icon = Some(
@@ -34,7 +34,7 @@ pub fn init() {
.with_tooltip(config::TRAY_TOOLTIP)
.with_icon(icon)
.build()
.unwrap()
.unwrap(),
);
// Since winit doesn't use gtk on Linux, and we need gtk for
@@ -55,33 +55,37 @@ pub fn init() {
}
// run the event loop
let event_loop = EventLoopBuilder::new().with_any_thread(true).build();
let event_loop = EventLoopBuilder::new()
.with_any_thread(true)
.build()
.unwrap();
let menu_channel = MenuEvent::receiver();
let tray_channel = TrayEvent::receiver();
let tray_channel = TrayIconEvent::receiver();
event_loop.run(move |_event, _, control_flow| {
*control_flow = ControlFlow::Poll;
event_loop
.run(move |_event, event_loop| {
event_loop.set_control_flow(ControlFlow::Poll);
//if let Ok(event) = tray_channel.try_recv() {
// println!("tray event: {event:?}");
//}
//if let Ok(event) = tray_channel.try_recv() {
// println!("tray event: {event:?}");
//}
if let Ok(event) = menu_channel.try_recv() {
println!("menu event: {:?}", event);
if let Ok(event) = menu_channel.try_recv() {
println!("menu event: {:?}", event);
if event.id == 1002 {
std::process::exit(0);
if event.id == "1002" {
std::process::exit(0);
}
}
}
});
})
.expect("Tray event loop run error");
});
info!("Tray initialized.");
}
fn load_icon(path: &std::path::Path) -> tray_icon::icon::Icon {
fn load_icon(path: &std::path::Path) -> tray_icon::Icon {
let (icon_rgba, icon_width, icon_height) = {
let image = image::open(path)
.expect("Failed to open icon path")
@@ -90,6 +94,5 @@ fn load_icon(path: &std::path::Path) -> tray_icon::icon::Icon {
let rgba = image.into_raw();
(rgba, width, height)
};
tray_icon::icon::Icon::from_rgba(icon_rgba, icon_width, icon_height)
.expect("Failed to open icon")
}
tray_icon::Icon::from_rgba(icon_rgba, icon_width, icon_height).expect("Failed to open icon")
}

6
app/src/tray/menu.rs
View File

@@ -1,7 +1,7 @@
pub enum TrayMenuItem {
Restart,
Settings,
Exit
Exit,
}
impl TrayMenuItem {
@@ -9,7 +9,7 @@ impl TrayMenuItem {
match *self {
TrayMenuItem::Restart => "Перезапустить",
TrayMenuItem::Settings => "Настройки",
TrayMenuItem::Exit => "Выход"
TrayMenuItem::Exit => "Выход",
}
}
}
}