GitHub/jarvis
1
0
Fork 0
mirror of https://github.com/Priler/jarvis.git synced 2026-05-26 07:08:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

intent recognition added (first implementation)

Browse Source
This commit is contained in:
Priler
2026-01-05 01:22:45 +05:00
parent 0c28304840
commit 5f1f9ce321
20 changed files with 625 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

290
Cargo.lock generated
View File

@@ -261,7 +261,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "241b621213072e993be4f6f3a9e4b45f65b7e6faad43001be957184b7bb1824b"
dependencies = [
"atk-sys",
"glib",
"glib 0.18.5",
"libc",
]
@@ -271,10 +271,10 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5e48b684b0ca77d2bbadeef17424c2ea3c897d44d566a1617e7e8f30614d086"
dependencies = [
"glib-sys",
"gobject-sys",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -495,7 +495,7 @@ checksum = "8ca26ef0159422fb77631dc9d17b102f253b876fe1586b03b803e63a309b4ee2"
dependencies = [
"bitflags 2.10.0",
"cairo-sys-rs",
"glib",
"glib 0.18.5",
"libc",
"once_cell",
"thiserror 1.0.69",
@@ -507,9 +507,9 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "685c9fa8e590b8b3d678873528d83411db17242a73fccaed827770ea0fedda51"
dependencies = [
"glib-sys",
"glib-sys 0.18.1",
"libc",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -653,7 +653,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02"
dependencies = [
"smallvec",
"target-lexicon",
"target-lexicon 0.12.16",
]
[[package]]
name = "cfg-expr"
version = "0.20.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21be0e1ce6cdb2ee7fff840f922fb04ead349e5cfb1e750b769132d44ce04720"
dependencies = [
"smallvec",
"target-lexicon 0.13.3",
]
[[package]]
@@ -675,8 +685,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2"
dependencies = [
"iana-time-zone",
"js-sys",
"num-traits",
"serde",
"wasm-bindgen",
"windows-link 0.2.1",
]
@@ -1017,6 +1029,19 @@ dependencies = [
"syn 2.0.113",
]
[[package]]
name = "dashmap"
version = "5.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
dependencies = [
"cfg-if",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
]
[[package]]
name = "dasp_sample"
version = "0.11.0"
@@ -1640,7 +1665,7 @@ dependencies = [
"gdk-pixbuf",
"gdk-sys",
"gio",
"glib",
"glib 0.18.5",
"libc",
"pango",
]
@@ -1653,7 +1678,7 @@ checksum = "50e1f5f1b0bfb830d6ccc8066d18db35c487b1b2b1e8589b5dfe9f07e8defaec"
dependencies = [
"gdk-pixbuf-sys",
"gio",
"glib",
"glib 0.18.5",
"libc",
"once_cell",
]
@@ -1664,11 +1689,11 @@ version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9839ea644ed9c97a34d129ad56d38a25e6756f99f3a88e15cd39c20629caf7"
dependencies = [
"gio-sys",
"glib-sys",
"gobject-sys",
"gio-sys 0.18.1",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -1679,13 +1704,13 @@ checksum = "5c2d13f38594ac1e66619e188c6d5a1adb98d11b2fcf7894fc416ad76aa2f3f7"
dependencies = [
"cairo-sys-rs",
"gdk-pixbuf-sys",
"gio-sys",
"glib-sys",
"gobject-sys",
"gio-sys 0.18.1",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"pango-sys",
"pkg-config",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -1695,11 +1720,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "140071d506d223f7572b9f09b5e155afbd77428cd5cc7af8f2694c41d98dfe69"
dependencies = [
"gdk-sys",
"glib-sys",
"gobject-sys",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"pkg-config",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -1711,7 +1736,7 @@ dependencies = [
"gdk",
"gdkx11-sys",
"gio",
"glib",
"glib 0.18.5",
"libc",
"x11",
]
@@ -1723,9 +1748,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e2e7445fe01ac26f11601db260dd8608fe172514eb63b3b5e261ea6b0f4428d"
dependencies = [
"gdk-sys",
"glib-sys",
"glib-sys 0.18.1",
"libc",
"system-deps",
"system-deps 6.2.2",
"x11",
]
@@ -2040,8 +2065,8 @@ dependencies = [
"futures-core",
"futures-io",
"futures-util",
"gio-sys",
"glib",
"gio-sys 0.18.1",
"glib 0.18.5",
"libc",
"once_cell",
"pin-project-lite",
@@ -2055,13 +2080,26 @@ version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37566df850baf5e4cb0dfb78af2e4b9898d817ed9263d1090a2df958c64737d2"
dependencies = [
"glib-sys",
"gobject-sys",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"system-deps",
"system-deps 6.2.2",
"winapi",
]
[[package]]
name = "gio-sys"
version = "0.21.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0071fe88dba8e40086c8ff9bbb62622999f49628344b1d1bf490a48a29d80f22"
dependencies = [
"glib-sys 0.21.5",
"gobject-sys 0.21.5",
"libc",
"system-deps 7.0.7",
"windows-sys 0.61.2",
]
[[package]]
name = "glam"
version = "0.30.9"
@@ -2083,10 +2121,10 @@ dependencies = [
"futures-executor",
"futures-task",
"futures-util",
"gio-sys",
"glib-macros",
"glib-sys",
"gobject-sys",
"gio-sys 0.18.1",
"glib-macros 0.18.5",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"memchr",
"once_cell",
@@ -2094,6 +2132,27 @@ dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "glib"
version = "0.21.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16de123c2e6c90ce3b573b7330de19be649080ec612033d397d72da265f1bd8b"
dependencies = [
"bitflags 2.10.0",
"futures-channel",
"futures-core",
"futures-executor",
"futures-task",
"futures-util",
"gio-sys 0.21.5",
"glib-macros 0.21.5",
"glib-sys 0.21.5",
"gobject-sys 0.21.5",
"libc",
"memchr",
"smallvec",
]
[[package]]
name = "glib-macros"
version = "0.18.5"
@@ -2108,6 +2167,19 @@ dependencies = [
"syn 2.0.113",
]
[[package]]
name = "glib-macros"
version = "0.21.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf59b675301228a696fe01c3073974643365080a76cc3ed5bc2cbc466ad87f17"
dependencies = [
"heck 0.5.0",
"proc-macro-crate 3.4.0",
"proc-macro2",
"quote",
"syn 2.0.113",
]
[[package]]
name = "glib-sys"
version = "0.18.1"
@@ -2115,7 +2187,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "063ce2eb6a8d0ea93d2bf8ba1957e78dbab6be1c2220dd3daca57d5a9d869898"
dependencies = [
"libc",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
name = "glib-sys"
version = "0.21.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d95e1a3a19ae464a7286e14af9a90683c64d70c02532d88d87ce95056af3e6c"
dependencies = [
"libc",
"system-deps 7.0.7",
]
[[package]]
@@ -2130,9 +2212,20 @@ version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0850127b514d1c4a4654ead6dedadb18198999985908e6ffe4436f53c785ce44"
dependencies = [
"glib-sys",
"glib-sys 0.18.1",
"libc",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
name = "gobject-sys"
version = "0.21.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dca35da0d19a18f4575f3cb99fe1c9e029a2941af5662f326f738a21edaf294"
dependencies = [
"glib-sys 0.21.5",
"libc",
"system-deps 7.0.7",
]
[[package]]
@@ -2148,7 +2241,7 @@ dependencies = [
"gdk",
"gdk-pixbuf",
"gio",
"glib",
"glib 0.18.5",
"gtk-sys",
"gtk3-macros",
"libc",
@@ -2166,12 +2259,12 @@ dependencies = [
"cairo-sys-rs",
"gdk-pixbuf-sys",
"gdk-sys",
"gio-sys",
"glib-sys",
"gobject-sys",
"gio-sys 0.18.1",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"pango-sys",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -2208,6 +2301,12 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
version = "0.16.1"
@@ -2560,6 +2659,25 @@ dependencies = [
"cfb",
]
[[package]]
name = "intent-classifier"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d4532decd2fe3806d9e068b57ceee827c7f1ab7b4ce4b2ed4c8c80679b7eefa"
dependencies = [
"ahash",
"chrono",
"dashmap",
"regex",
"serde",
"serde_json",
"thiserror 1.0.69",
"tokio",
"tracing",
"unicode-segmentation",
"uuid",
]
[[package]]
name = "interpolate_name"
version = "0.2.4"
@@ -2625,15 +2743,17 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
name = "jarvis-app"
version = "0.1.0"
dependencies = [
"glib",
"glib 0.21.5",
"gtk",
"image",
"jarvis-core",
"log",
"once_cell",
"parking_lot",
"platform-dirs",
"rand 0.8.5",
"simple-log",
"tokio",
"tray-icon",
"winapi",
"winit",
@@ -2644,6 +2764,7 @@ name = "jarvis-core"
version = "0.1.0"
dependencies = [
"hound",
"intent-classifier",
"kira",
"log",
"once_cell",
@@ -2657,6 +2778,9 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml",
"sha2",
"tokio",
"toml 0.9.10+spec-1.1.0",
"vosk",
]
@@ -2689,7 +2813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca5671e9ffce8ffba57afc24070e906da7fc4b1ba66f2cabebf61bf2ea257fcc"
dependencies = [
"bitflags 1.3.2",
"glib",
"glib 0.18.5",
"javascriptcore-rs-sys",
]
@@ -2699,10 +2823,10 @@ version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af1be78d14ffa4b75b66df31840478fef72b51f8c2465d4ca7c194da9f7a5124"
dependencies = [
"glib-sys",
"gobject-sys",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -2827,7 +2951,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03589b9607c868cc7ae54c0b2a22c8dc03dd41692d48f2d7df73615c6a95dc0a"
dependencies = [
"glib",
"glib 0.18.5",
"gtk",
"gtk-sys",
"libappindicator-sys",
@@ -3884,7 +4008,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ca27ec1eb0457ab26f3036ea52229edbdb74dee1edd29063f5b9b010e7ebee4"
dependencies = [
"gio",
"glib",
"glib 0.18.5",
"libc",
"once_cell",
"pango-sys",
@@ -3896,10 +4020,10 @@ version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "436737e391a843e5933d6d9aa102cb126d501e815b83601365a948a518555dc5"
dependencies = [
"glib-sys",
"gobject-sys",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -4775,8 +4899,8 @@ dependencies = [
"ashpd",
"block2 0.6.2",
"dispatch2",
"glib-sys",
"gobject-sys",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"gtk-sys",
"js-sys",
"log",
@@ -5421,7 +5545,7 @@ checksum = "471f924a40f31251afc77450e781cb26d55c0b650842efafc9c6cbd2f7cc4f9f"
dependencies = [
"futures-channel",
"gio",
"glib",
"glib 0.18.5",
"libc",
"soup3-sys",
]
@@ -5432,11 +5556,11 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ebe8950a680a12f24f15ebe1bf70db7af98ad242d9db43596ad3108aab86c27"
dependencies = [
"gio-sys",
"glib-sys",
"gobject-sys",
"gio-sys 0.18.1",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"libc",
"system-deps",
"system-deps 6.2.2",
]
[[package]]
@@ -5728,13 +5852,26 @@ version = "6.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349"
dependencies = [
"cfg-expr",
"cfg-expr 0.15.8",
"heck 0.5.0",
"pkg-config",
"toml 0.8.2",
"version-compare",
]
[[package]]
name = "system-deps"
version = "7.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c8f33736f986f16d69b6cb8b03f55ddcad5c41acc4ccc39dd88e84aa805e7f"
dependencies = [
"cfg-expr 0.20.5",
"heck 0.5.0",
"pkg-config",
"toml 0.9.10+spec-1.1.0",
"version-compare",
]
[[package]]
name = "systemstat"
version = "0.2.5"
@@ -5806,6 +5943,12 @@ version = "0.12.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
[[package]]
name = "target-lexicon"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c"
[[package]]
name = "tauri"
version = "2.9.5"
@@ -6262,13 +6405,26 @@ dependencies = [
"bytes",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"tracing",
"windows-sys 0.61.2",
]
[[package]]
name = "tokio-macros"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.113",
]
[[package]]
name = "tokio-util"
version = "0.7.17"
@@ -6976,10 +7132,10 @@ dependencies = [
"gdk",
"gdk-sys",
"gio",
"gio-sys",
"glib",
"glib-sys",
"gobject-sys",
"gio-sys 0.18.1",
"glib 0.18.5",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"gtk",
"gtk-sys",
"javascriptcore-rs",
@@ -6998,15 +7154,15 @@ dependencies = [
"bitflags 1.3.2",
"cairo-sys-rs",
"gdk-sys",
"gio-sys",
"glib-sys",
"gobject-sys",
"gio-sys 0.18.1",
"glib-sys 0.18.1",
"gobject-sys 0.18.0",
"gtk-sys",
"javascriptcore-rs-sys",
"libc",
"pkg-config",
"soup3-sys",
"system-deps",
"system-deps 6.2.2",
]
[[package]]

4
Cargo.toml
View File

@@ -29,4 +29,6 @@ pv_recorder = { git = "https://github.com/Priler/pvrecorder" }
vosk = "0.3"
rustpotter = { git = "https://github.com/Priler/rustpotter" }
image = "0.25"
parking_lot = "0.12.5"
parking_lot = "0.12.5"
toml = "0.9.8"
sha2 = "0.10"

5
crates/jarvis-app/Cargo.toml
View File

@@ -16,6 +16,9 @@ winit = "0.30"
image.workspace = true
platform-dirs.workspace = true
rand.workspace = true
parking_lot.workspace = true
tokio = { version = "1", features = ["rt-multi-thread"] }
[target.'cfg(windows)'.dependencies.winit]
version = "0.30"
@@ -26,4 +29,4 @@ winapi = { version = "0.3", features = ["winuser"] }
[target.'cfg(target_os = "linux")'.dependencies]
gtk = "0.18"
glib = "0.18"
glib = "0.21.5"

24
crates/jarvis-app/src/app.rs
View File

@@ -1,6 +1,6 @@
use std::time::SystemTime;
use jarvis_core::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST};
use jarvis_core::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST, intent};
use rand::prelude::*;
pub fn start() -> Result<(), ()> {
@@ -9,6 +9,7 @@ pub fn start() -> Result<(), ()> {
}
fn main_loop() -> Result<(), ()> {
let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime");
let mut start: SystemTime;
let sounds_directory = audio::get_sound_directory().unwrap();
let frame_length: usize = 512; // default for every wake-word engine
@@ -34,7 +35,7 @@ fn main_loop() -> Result<(), ()> {
// recognize wake-word
match listener::data_callback(&frame_buffer) {
Some(keyword_index) => {
Some(_keyword_index) => {
// wake-word activated, process further commands
// capture current time
start = SystemTime::now();
@@ -66,13 +67,18 @@ fn main_loop() -> Result<(), ()> {
}
recognized_voice = recognized_voice.trim().into();
// infer command
if let Some((cmd_path, cmd_config)) = commands::fetch_command(
&recognized_voice,
&COMMANDS_LIST.get().unwrap(),
) {
// some debug info
info!("Recognized voice (filtered): {}", recognized_voice);
// infer command (try intent recognition first, fallback to levenshtein)
let cmd_result = if let Some((intent_id, confidence)) =
rt.block_on(intent::classify(&recognized_voice))
{
info!("Intent recognized: {} (confidence: {:.2})", intent_id, confidence);
intent::get_command_by_intent(COMMANDS_LIST.get().unwrap(), &intent_id)
} else {
info!("Intent not recognized, trying levenshtein fallback...");
commands::fetch_command(&recognized_voice, COMMANDS_LIST.get().unwrap())
};
if let Some((cmd_path, cmd_config)) = cmd_result {
info!("Command found: {:?}", cmd_path);
info!("Executing!");

25
crates/jarvis-app/src/main.rs
View File

@@ -1,8 +1,9 @@
use std::path::PathBuf;
use parking_lot::RwLock;
use std::sync::Arc;
// include core
use jarvis_core::{
audio, commands, config, db, listener, recorder, stt,
audio, commands, config, db, listener, recorder, stt, intent,
APP_CONFIG_DIR, APP_LOG_DIR, COMMANDS_LIST, DB,
};
@@ -32,7 +33,8 @@ fn main() -> Result<(), String> {
info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display());
// initialize database (settings)
let _ = DB.set(db::init_settings());
DB.set(Arc::new(RwLock::new(db::init_settings())))
.expect("DB already initialized");
// initialize tray
// @TODO. macOS currently not supported for tray functionality,
@@ -58,7 +60,13 @@ fn main() -> Result<(), String> {
// init commands
info!("Initializing commands.");
let cmds = commands::parse_commands().unwrap();
let cmds = match commands::parse_commands() {
Ok(c) => c,
Err(e) => {
warn!("Failed to parse commands: {}. Starting with empty command list.", e);
Vec::new()
}
};
info!("Commands initialized. Count: {}, List: {:?}", cmds.len(), commands::list(&cmds));
COMMANDS_LIST.set(cmds).unwrap();
@@ -73,6 +81,15 @@ fn main() -> Result<(), String> {
app::close(1); // cannot continue without wake-word engine
}
// init intent-recognition engine
let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime");
rt.block_on(async {
if intent::init(COMMANDS_LIST.get().unwrap()).await.is_err() {
error!("Failed to initialize intent classifier");
app::close(1);
}
});
// start the app (in the background thread)
std::thread::spawn(|| {
let _ = app::start();

8
crates/jarvis-core/Cargo.toml
View File

@@ -21,12 +21,16 @@ kira.workspace = true
pv_recorder.workspace = true
rustpotter.workspace = true
parking_lot.workspace = true
toml.workspace = true
sha2.workspace = true
# pv_recorder = { workspace = true, optional = true }
vosk = { version = "0.3.1", optional = true }
intent-classifier = { version = "0.1.0", optional = true }
# rustpotter = { workspace = true, optional = true }
tokio = { version = "1", features = ["sync"], optional = true }
[features]
default = ["jarvis_app"]
jarvis_app = ["vosk"]
jarvis_app = ["vosk", "intent-classifier", "tokio"]

145
crates/jarvis-core/src/commands.rs
View File

@@ -12,72 +12,96 @@ use std::process::{Child, Command};
mod structs;
pub use structs::*;
use std::collections::HashMap;
use crate::{audio, config};
// @TODO. Allow commands both in yaml and json format.
pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
pub fn parse_commands() -> Result<Vec<JCommandsList>, String> {
// collect commands
let mut commands: Vec<AssistantCommand> = vec![];
let mut commands: Vec<JCommandsList> = Vec::new();
// read commands directories first
if let Ok(cpaths) = fs::read_dir(config::COMMANDS_PATH) {
for cpath in cpaths {
// validate this command, check if required files exists
let _cpath = match cpath {
Ok(entry) => entry.path(),
Err(e) => {
warn!("Failed to read command directory entry: {}", e);
continue;
}
};
let cc_file = Path::new(&_cpath).join("command.yaml");
let cmd_dirs = fs::read_dir(config::COMMANDS_PATH)
.map_err(|e| format!("Error reading commands directory: {}", e))?;
if cc_file.exists() {
// try parse config files
let cc_reader = std::fs::File::open(&cc_file).unwrap();
let cc_yaml: CommandsList;
// try parse command.yaml
match serde_yaml::from_reader::<File, CommandsList>(cc_reader) {
Ok(parse_result) => {
cc_yaml = parse_result;
}
Err(msg) => {
warn!(
"Can't parse {}, skipping ...\nCommand parse error is: {:?}",
&cc_file.display(),
msg
);
continue;
}
}
// everything seems to be Ok
commands.push(AssistantCommand {
path: _cpath,
commands: cc_yaml,
});
for entry in cmd_dirs {
let entry = match entry {
Ok(e) => e,
Err(e) => {
warn!("Failed to read command directory entry: {}", e);
continue;
}
};
let cmd_path = entry.path();
let toml_file = cmd_path.join("command.toml");
if !toml_file.exists() {
continue;
}
// read and parse TOML
let content = match fs::read_to_string(&toml_file) {
Ok(c) => c,
Err(e) => {
warn!("Failed to read {}: {}", toml_file.display(), e);
continue;
}
};
if !commands.is_empty() {
Ok(commands)
} else {
error!("No commands were found");
Err("No commands were found".into())
}
let file: JCommandsList = match toml::from_str(&content) {
Ok(f) => f,
Err(e) => {
warn!("Failed to parse {}: {}", toml_file.display(), e);
continue;
}
};
commands.push(JCommandsList {
path: cmd_path,
commands: file.commands,
});
}
if commands.is_empty() {
Err("No commands found".into())
} else {
error!("Error reading commands directory");
return Err("Error reading commands directory".into());
info!("Loaded {} commands", commands.len());
Ok(commands)
}
}
// Commands hash generation for cache invalidation (deterministi c)
pub fn commands_hash(commands: &Vec<JCommandsList>) -> String {
use sha2::{Sha256, Digest};
let mut hasher = Sha256::new();
// collect all command ids and phrases, sorted
let mut all_ids: Vec<_> = commands.iter()
.flat_map(|ac| ac.commands.iter().map(|c| (&c.id, &c.phrases)))
.collect();
all_ids.sort_by_key(|(id, _)| *id);
for (id, phrases) in all_ids {
hasher.update(id.as_bytes());
for phrase in phrases {
hasher.update(phrase.as_bytes());
}
}
format!("{:x}", hasher.finalize())
}
// @TODO. NLU or smthng else is required, in order to infer commands with highest accuracy possible.
pub fn fetch_command<'a>(
phrase: &str,
commands: &'a Vec<AssistantCommand>,
) -> Option<(&'a PathBuf, &'a Config)> {
commands: &'a Vec<JCommandsList>,
) -> Option<(&'a PathBuf, &'a JCommand)> {
// result scmd
let mut result_scmd: Option<(&PathBuf, &Config)> = None;
let mut result_scmd: Option<(&PathBuf, &JCommand)> = None;
let mut current_max_ratio = config::CMD_RATIO_THRESHOLD;
// convert fetch phrase to sequence
@@ -86,7 +110,7 @@ pub fn fetch_command<'a>(
// list all the commands
for cmd in commands {
// list all subcommands
for scmd in &cmd.commands.list {
for scmd in &cmd.commands {
// list all phrases in command
for cmd_phrase in &scmd.phrases {
// convert cmd phrase to sequence
@@ -135,18 +159,17 @@ pub fn execute_cli(cmd: &str, args: &Vec<String>) -> std::io::Result<Child> {
pub fn execute_command(
cmd_path: &PathBuf,
cmd_config: &Config,
cmd_config: &JCommand,
// app_handle: &tauri::AppHandle,
) -> Result<bool, String> {
let sounds_directory = audio::get_sound_directory().unwrap();
match cmd_config.command.action.as_str() {
match cmd_config.action.as_str() {
"voice" => {
// VOICE command type
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@@ -158,8 +181,8 @@ pub fn execute_command(
}
"ahk" => {
// AutoHotkey command type
let exe_path_absolute = Path::new(&cmd_config.command.exe_path);
let exe_path_local = Path::new(&cmd_path).join(&cmd_config.command.exe_path);
let exe_path_absolute = Path::new(&cmd_config.exe_path);
let exe_path_local = Path::new(&cmd_path).join(&cmd_config.exe_path);
if let Ok(_) = execute_exe(
if exe_path_absolute.exists() {
@@ -167,12 +190,11 @@ pub fn execute_command(
} else {
exe_path_local.to_str().unwrap()
},
&cmd_config.command.exe_args,
&cmd_config.exe_args,
) {
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@@ -188,14 +210,13 @@ pub fn execute_command(
}
"cli" => {
// CLI command type
let cli_cmd = &cmd_config.command.cli_cmd;
let cli_cmd = &cmd_config.cli_cmd;
match execute_cli(cli_cmd, &cmd_config.command.cli_args) {
match execute_cli(cli_cmd, &cmd_config.cli_args) {
Ok(_) => {
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@@ -216,7 +237,6 @@ pub fn execute_command(
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@@ -232,7 +252,6 @@ pub fn execute_command(
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@@ -249,7 +268,7 @@ pub fn execute_command(
}
}
pub fn list(from: &[AssistantCommand]) -> Vec<String> {
pub fn list(from: &Vec<JCommandsList>) -> Vec<String> {
let mut out: Vec<String> = vec![];
for x in from.iter() {

45
crates/jarvis-core/src/commands/structs.rs
View File

@@ -1,45 +1,38 @@
use serde::Deserialize;
use std::path::PathBuf;
use serde::Deserialize;
#[derive(Debug)]
pub struct AssistantCommand {
#[derive(Deserialize, Debug)]
pub struct JCommandsList {
#[serde(skip)]
pub path: PathBuf,
pub commands: CommandsList,
pub commands: Vec<JCommand>,
}
#[derive(Deserialize, Debug)]
pub struct CommandsList {
pub list: Vec<Config>,
}
#[derive(Deserialize, Debug)]
pub struct Config {
pub command: ConfigCommandSection,
pub voice: ConfigVoiceSection,
pub phrases: Vec<String>,
}
#[derive(Deserialize, Debug)]
pub struct ConfigCommandSection {
#[derive(Deserialize, Debug, Clone)]
pub struct JCommand {
pub id: String,
pub action: String,
#[serde(default)]
pub description: String,
#[serde(default)]
pub exe_path: String,
#[serde(default)]
pub exe_args: Vec<String>,
#[serde(default)]
pub cli_cmd: String,
#[serde(default)]
pub cli_args: Vec<String>,
}
#[derive(Deserialize, Debug)]
pub struct ConfigVoiceSection {
#[serde(default)]
pub sounds: Vec<String>,
pub phrases: Vec<String>,
}

5
crates/jarvis-core/src/config.rs
View File

@@ -17,6 +17,7 @@ use rustpotter::{
RustpotterConfig, ScoreMode,
};
use crate::IntentRecognitionEngine;
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
#[allow(dead_code)]
@@ -64,6 +65,7 @@ pub fn init_dirs() -> Result<(), String> {
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter;
pub const DEFAULT_INTENT_RECOGNITION_ENGINE: IntentRecognitionEngine = IntentRecognitionEngine::IntentClassifier;
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
pub const DEFAULT_VOICE: &str = "jarvis-og";
@@ -130,6 +132,9 @@ pub const VOSK_FETCH_PHRASE: &str = "джарвис";
pub const VOSK_MODEL_PATH: &str = "vosk/model_small";
pub const VOSK_MIN_RATIO: f64 = 70.0;
// IRE (intents recognition)
pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.5;
// ETC
pub const CMD_RATIO_THRESHOLD: f64 = 65f64;
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);

6
crates/jarvis-core/src/config/structs.rs
View File

@@ -8,6 +8,12 @@ pub enum WakeWordEngine {
Porcupine,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
pub enum IntentRecognitionEngine {
IntentClassifier,
Rasa,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)

3
crates/jarvis-core/src/db/structs.rs
View File

@@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize};
use crate::config::structs::SpeechToTextEngine;
use crate::config::structs::WakeWordEngine;
use crate::config::structs::IntentRecognitionEngine;
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Settings {
@@ -10,6 +11,7 @@ pub struct Settings {
pub voice: String,
pub wake_word_engine: WakeWordEngine,
pub intent_recognition_engine: IntentRecognitionEngine,
pub speech_to_text_engine: SpeechToTextEngine,
pub api_keys: ApiKeys,
@@ -22,6 +24,7 @@ impl Default for Settings {
voice: String::from(""),
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
intent_recognition_engine: config::DEFAULT_INTENT_RECOGNITION_ENGINE,
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
api_keys: ApiKeys {

62
crates/jarvis-core/src/intent.rs Normal file
View File

@@ -0,0 +1,62 @@
mod intentclassifier;
use std::path::PathBuf;
use crate::{JCommandsList, commands::JCommand, config};
use once_cell::sync::OnceCell;
use crate::config::structs::IntentRecognitionEngine;
static IRE_TYPE: OnceCell<IntentRecognitionEngine> = OnceCell::new();
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
if IRE_TYPE.get().is_some() {
return Ok(());
} // already initialized
// set default ire type
// @TODO. Make it configurable?
IRE_TYPE.set(config::DEFAULT_INTENT_RECOGNITION_ENGINE).unwrap();
// load given recorder
match IRE_TYPE.get().unwrap() {
IntentRecognitionEngine::IntentClassifier => {
info!("Initializing IRE backend.");
intentclassifier::init(&commands).await?;
info!("IRE backend initialized.");
},
IntentRecognitionEngine::Rasa => todo!(),
}
Ok(())
}
pub async fn classify(text: &str) -> Option<(String, f64)> {
match IRE_TYPE.get()? {
IntentRecognitionEngine::IntentClassifier => {
match intentclassifier::classify(text).await {
Ok(prediction) => {
let confidence = prediction.confidence.value();
if confidence >= config::INTENT_CLASSIFIER_MIN_CONFIDENCE {
Some((prediction.intent.to_string(), confidence))
} else {
None
}
}
Err(e) => {
error!("Intent classification error: {}", e);
None
}
}
}
IntentRecognitionEngine::Rasa => todo!(),
}
}
pub fn get_command_by_intent(commands: &'static Vec<JCommandsList>, intent_id: &str) -> Option<(&'static PathBuf, &'static JCommand)> {
match IRE_TYPE.get()? {
IntentRecognitionEngine::IntentClassifier => {
intentclassifier::get_command(commands, intent_id)
}
IntentRecognitionEngine::Rasa => todo!(),
}
}

107
crates/jarvis-core/src/intent/intentclassifier.rs Normal file
View File

@@ -0,0 +1,107 @@
use intent_classifier::{
IntentClassifier, IntentPrediction, IntentError,
TrainingExample, TrainingSource, IntentId
};
use tokio::sync::OnceCell;
use std::path::PathBuf;
use std::fs;
use crate::commands::{self, JCommand, JCommandsList};
use crate::{APP_CONFIG_DIR};
static CLASSIFIER: OnceCell<IntentClassifier> = OnceCell::const_new();
// static COMMANDS_MAP: OnceCell<Vec<JCommandsList>> = OnceCell::const_new();
const TRAINING_CACHE_FILE: &str = "intent_training.json";
const COMMANDS_HASH_FILE: &str = "commands_hash.txt";
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
// parse commands first
// let commands = commands::parse_commands()?;
let current_hash = commands::commands_hash(&commands); // regen hash for current commands set
// init classifier
let classifier = IntentClassifier::new().await
.map_err(|e| format!("Failed to init IntentClassifier: {}", e))?;
// check if we can use cached training data
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
let hash_path = config_dir.join(COMMANDS_HASH_FILE);
let cache_path = config_dir.join(TRAINING_CACHE_FILE);
let should_retrain = if hash_path.exists() && cache_path.exists() {
let stored_hash = fs::read_to_string(&hash_path).unwrap_or_default();
stored_hash.trim() != current_hash
} else {
true
};
if should_retrain {
info!("Training intent classifier with {} commands...", commands.len());
train_classifier(&classifier, &commands).await?;
// save training data and hash
if let Ok(export) = classifier.export_training_data().await {
let _ = fs::write(&cache_path, export);
let _ = fs::write(&hash_path, &current_hash);
info!("Training data cached.");
}
} else {
info!("Loading cached training data...");
if let Ok(data) = fs::read_to_string(&cache_path) {
classifier.import_training_data(&data).await
.map_err(|e| format!("Failed to import training data: {}", e))?;
}
}
// store data
CLASSIFIER.set(classifier).map_err(|_| "Classifier already set")?;
// COMMANDS_MAP.set(commands).map_err(|_| "Commands map already set")?;
Ok(())
}
pub async fn classify(text: &str) -> Result<IntentPrediction, IntentError> {
let classifier = CLASSIFIER.get().expect("IntentClassifier not initialized");
classifier.predict_intent(text).await
}
// get command by intent ID
pub fn get_command(commands: &'static Vec<JCommandsList>, intent_id: &str) -> Option<(&'static PathBuf, &'static JCommand)> {
// let commands = COMMANDS_MAP.get()?;
for assistant_cmd in commands {
for cmd in &assistant_cmd.commands {
if cmd.id == intent_id {
return Some((&assistant_cmd.path, cmd));
}
}
}
None
}
// based on: https://github.com/ciresnave/intent-classifier/blob/main/examples/basic_usage.rs
async fn train_classifier(
classifier: &IntentClassifier,
commands: &Vec<JCommandsList>
) -> Result<(), String> {
for assistant_cmd in commands {
for cmd in &assistant_cmd.commands {
for phrase in &cmd.phrases {
let example = TrainingExample {
text: phrase.clone(),
intent: IntentId::from(cmd.id.as_str()),
confidence: 1.0,
source: TrainingSource::Programmatic,
};
classifier.add_training_example(example).await
.map_err(|e| format!("Failed to add training example: {}", e))?;
}
}
}
Ok(())
}

13
crates/jarvis-core/src/lib.rs
View File

@@ -1,6 +1,6 @@
use once_cell::sync::{Lazy, OnceCell};
use parking_lot::RwLock;
use std::sync::Arc;
use std::{sync::Arc};
use platform_dirs::AppDirs;
use std::path::PathBuf;
@@ -20,6 +20,9 @@ pub mod recorder;
#[cfg(feature = "jarvis_app")]
pub mod stt;
#[cfg(feature = "jarvis_app")]
pub mod intent;
// shared statics
pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| std::env::current_dir().unwrap());
pub static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| APP_DIR.clone().join("sound"));
@@ -27,9 +30,11 @@ pub static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
pub static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
pub static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
pub static DB: OnceCell<Arc<RwLock<db::structs::Settings>>> = OnceCell::new();
pub static COMMANDS_LIST: OnceCell<Vec<commands::AssistantCommand>> = OnceCell::new();
pub static COMMANDS_LIST: OnceCell<Vec<JCommandsList>> = OnceCell::new();
// re-exports
pub use commands::AssistantCommand;
pub use commands::JCommandsList;
pub use config::structs::*;
pub use db::structs::Settings;
pub use db::structs::Settings;
// use crate::commands::{JComandsList, JCommand};

2
crates/jarvis-core/src/listener.rs
View File

@@ -25,7 +25,7 @@ pub fn init() -> Result<(), ()> {
// store current engine
WAKE_WORD_ENGINE
.set(DB.get().unwrap().wake_word_engine)
.set(DB.get().unwrap().read().wake_word_engine)
.unwrap();
// load given wake-word engine

8
crates/jarvis-gui/src/tauri_commands/db.rs
View File

@@ -9,6 +9,7 @@ pub fn db_read(state: tauri::State<'_, AppState>, key: &str) -> String {
"selected_microphone" => settings.microphone.to_string(),
"assistant_voice" => settings.voice.clone(),
"selected_wake_word_engine" => format!("{:?}", settings.wake_word_engine),
"selected_intent_recognition_engine" => format!("{:?}", settings.intent_recognition_engine),
"speech_to_text_engine" => format!("{:?}", settings.speech_to_text_engine),
"api_key__picovoice" => settings.api_keys.picovoice.clone(),
"api_key__openai" => settings.api_keys.openai.clone(),
@@ -41,6 +42,13 @@ pub fn db_write(state: tauri::State<'_, AppState>, key: &str, val: &str) -> bool
_ => return false,
}
}
"selected_intent_recognition_engine" => {
match val.to_lowercase().as_str() {
"intentclassifier" => settings.intent_recognition_engine = jarvis_core::config::structs::IntentRecognitionEngine::IntentClassifier,
"rasa" => settings.intent_recognition_engine = jarvis_core::config::structs::IntentRecognitionEngine::Rasa,
_ => return false,
}
}
"api_key__picovoice" => {
settings.api_keys.picovoice = val.to_string();
}

18
frontend/src/routes/settings/index.svelte
View File

@@ -46,6 +46,7 @@
let voiceVal = ""
let selectedMicrophone = ""
let selectedWakeWordEngine = ""
let selectedIntentRecognitionEngine = ""
let apiKeyPicovoice = ""
let apiKeyOpenai = ""
@@ -71,6 +72,7 @@
invoke("db_write", { key: "assistant_voice", val: voiceVal }),
invoke("db_write", { key: "selected_microphone", val: selectedMicrophone }),
invoke("db_write", { key: "selected_wake_word_engine", val: selectedWakeWordEngine }),
invoke("db_write", { key: "selected_intent_recognition_engine", val: selectedIntentRecognitionEngine }),
invoke("db_write", { key: "api_key__picovoice", val: apiKeyPicovoice }),
invoke("db_write", { key: "api_key__openai", val: apiKeyOpenai })
])
@@ -106,15 +108,17 @@
}))
// load settings from db
const [mic, wakeWord, pico, openai] = await Promise.all([
const [mic, wakeWord, intentReco, pico, openai] = await Promise.all([
invoke<string>("db_read", { key: "selected_microphone" }),
invoke<string>("db_read", { key: "selected_wake_word_engine" }),
invoke<string>("db_read", { key: "selected_intent_recognition_engine" }),
invoke<string>("db_read", { key: "api_key__picovoice" }),
invoke<string>("db_read", { key: "api_key__openai" })
])
selectedMicrophone = mic
selectedWakeWordEngine = wakeWord
selectedIntentRecognitionEngine = intentReco
apiKeyPicovoice = pico
apiKeyOpenai = openai
} catch (err) {
@@ -228,6 +232,18 @@
</Alert>
{/if}
<Space h="xl" />
<NativeSelect
data={[
{ label: "Intent Classifier", value: "IntentClassifier" },
{ label: "Rasa", value: "Rasa" }
]}
label="Распознавание команд (Intent Recognition)"
description="Выберите, какая нейросеть будет отвечать за распознавание команд."
variant="filled"
bind:value={selectedIntentRecognitionEngine}
/>
<Space h="xl" />
<InputWrapper label="Ключ OpenAI">

BIN
jarvis-app.zip Normal file
View File

Binary file not shown.

34
resources/commands/browser/command.toml Normal file
View File

@@ -0,0 +1,34 @@
[[commands]]
id = "browser_open"
action = "ahk"
exe_path = "ahk/Run browser.exe"
sounds = ["ok1", "ok2", "ok3"]
phrases = [
"открой браузер",
"открой хром",
"гугл хром",
]
[[commands]]
id = "browser_close"
action = "ahk"
exe_path = "ahk/Close browser.exe"
sounds = ["ok1", "ok2", "ok3", "ok4"]
phrases = [
"закрой все браузеры",
"закрой браузер",
"выключи браузер",
"убери браузер",
]
[[commands]]
id = "open_google"
action = "ahk"
exe_path = "ahk/Run website.exe"
exe_args = ["http://google.com"]
sounds = ["ok1", "ok2", "ok3", "ok4"]
phrases = [
"открой гугл",
"запусти гугл",
"перейди в гугл",
]

76
resources/commands/browser/command.yaml
View File

@@ -1,76 +0,0 @@
list:
- command:
action: ahk
exe_path: ahk/Run browser.exe
voice:
sounds:
- ok1
- ok2
- ok3
phrases:
- открой браузер
- открой хром
- гугл хром
- command:
action: ahk
exe_path: ahk/Close browser.exe
voice:
sounds:
- ok1
- ok2
- ok3
- ok4
phrases:
- закрой все браузеры
- закрой браузер
- выключи браузер
- убери браузер
- command:
action: ahk
exe_path: ahk/Run website.exe
exe_args:
- http://google.com
voice:
sounds:
- ok1
- ok2
- ok3
- ok4
phrases:
- открой гугл
- запусти гугл
- перейди в гугл
- command:
action: ahk
exe_path: ahk/Run website.exe
exe_args:
- http://youtube.com
voice:
sounds:
- ok1
- ok2
- ok3
- ok4
phrases:
- открой ютуб
- ютуб
- запусти ютуб
- command:
action: ahk
exe_path: ahk/Run website.exe
exe_args:
- https://translate.google.com
voice:
sounds:
- ok1
- ok2
- ok3
- ok4
phrases:
- открой переводчик
- переводчик
- запусти переводчик