feat: Quick ask window (#458)

* - feature: added poc for shinkai spotlight

* - improve: prevent app closing

* - refactor: windows management

* - fix: command to find shinkai

* wip

* wip

* feat: quick ask base version

* add hide spotlight command

* feat: shortcuts

* improvements

* more improvements

* - fix: added target-os to reopen event

---------

Co-authored-by: Alfredo Gallardo <[email protected]>

apps/shinkai-desktop/src-tauri/Cargo.toml

@@ -36,6 +36,7 @@ hex = "0.4.3"
 base64 = "0.22.1"
 time = "^0.3.36"
 listeners = "0.2"
+log = "0.4"
 
 tauri-plugin-global-shortcut = "2.0.0-rc.2"
 tauri-plugin-shell = "2.0.0-rc.3"
@@ -45,6 +46,7 @@ tauri-plugin-dialog = "2.0.0-rc.7"
 tauri-plugin-fs="2.0.0-rc.5"
 tauri-plugin-os = "2.0.0-rc.1"
 tauri-plugin-process = "2.0.0-rc.1"
+tauri-plugin-log = "2.0.0-rc"
 
 [features]
 # this feature is used for production builds or when `devPath` points to the filesystem

apps/shinkai-desktop/src-tauri/build.rs

@@ -1,7 +1,7 @@
 fn main() {
-  // fix for crashing bundle dmg/app on MACOS
-  println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.13");
+    // fix for crashing bundle dmg/app on MACOS
+    println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.13");
 
-  // triggers tauri build
-  tauri_build::build()
+    // triggers tauri build
+    tauri_build::build()
 }

apps/shinkai-desktop/src-tauri/capabilities/migrated.json

@@ -2,7 +2,9 @@
   "identifier": "migrated",
   "description": "permissions that were migrated from v1",
   "local": true,
-  "windows": ["main"],
+  "windows": [
+    "main"
+  ],
   "permissions": [
     "core:default",
     "core:webview:allow-create-webview-window",
@@ -14,7 +16,8 @@
     "updater:allow-download",
     "updater:allow-install",
     "updater:allow-download-and-install",
-    "process:allow-restart"
+    "process:allow-restart",
+    "log:default"
   ],
   "commands.allow": [
     "shinkai_node_is_running",
@@ -30,4 +33,4 @@
     "hardware_get_summary",
     "galxe_generate_proof"
   ]
-}
+}

apps/shinkai-desktop/src-tauri/src/audio/mod.rs

@@ -1 +1 @@
-pub mod transcribe;
+pub mod transcribe;

apps/shinkai-desktop/src-tauri/src/audio/transcribe.rs

@@ -28,7 +28,9 @@ pub fn run<T>(
     let buffer = Arc::new(Mutex::new(Vec::new()));
 
     // Initialize the VAD
-    let vad = Arc::new(Mutex::new(Vad::new(config.sample_rate.0.try_into().unwrap()).unwrap()));
+    let vad = Arc::new(Mutex::new(
+        Vad::new(config.sample_rate.0.try_into().unwrap()).unwrap(),
+    ));
     vad.lock()
         .unwrap()
         .fvad_set_mode(webrtc_vad::VadMode::VeryAggressive)
@@ -49,14 +51,20 @@ pub fn run<T>(
 
                 let mut is_activated = is_activated.lock().unwrap();
                 let mut vad = Vad::new(config.sample_rate.0.try_into().unwrap()).unwrap();
-                vad.fvad_set_mode(webrtc_vad::VadMode::VeryAggressive).unwrap();
+                vad.fvad_set_mode(webrtc_vad::VadMode::VeryAggressive)
+                    .unwrap();
 
                 // print buffer length
                 // println!("Buffer length: {}", buffer.len());
                 if !buffer.is_empty() && buffer.len() >= 480 {
                     // println!("Buffer length: {}", buffer.len());
-                    let buffer_i16: Vec<i16> = buffer.iter().map(|&f| (f * i16::MAX as f32) as i16).collect();
-                    let is_voice = vad.is_voice_segment(&buffer_i16[(buffer.len() - 480)..]).unwrap();
+                    let buffer_i16: Vec<i16> = buffer
+                        .iter()
+                        .map(|&f| (f * i16::MAX as f32) as i16)
+                        .collect();
+                    let is_voice = vad
+                        .is_voice_segment(&buffer_i16[(buffer.len() - 480)..])
+                        .unwrap();
                     // println!("Is voice: {}", is_voice);
 
                     if is_voice {
@@ -68,7 +76,9 @@ pub fn run<T>(
                             println!("Started recording");
                         }
                     } else if let Some(last_voice_activity) = last_voice_activity {
-                        if last_voice_activity.elapsed() > Duration::from_secs_f32(1.0) && *is_activated {
+                        if last_voice_activity.elapsed() > Duration::from_secs_f32(1.0)
+                            && *is_activated
+                        {
                             // Stop recording
                             *is_activated = false;
                             if let Some(start_time) = start_time {
@@ -84,20 +94,28 @@ pub fn run<T>(
                                 let ctx = ctx.lock().unwrap();
                                 let mut state = ctx.create_state().expect("failed to create state");
 
-                                let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
+                                let mut params =
+                                    FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
                                 params.set_n_threads(2);
                                 params.set_print_special(false);
                                 params.set_print_progress(false);
                                 params.set_print_realtime(false);
                                 params.set_print_timestamps(false);
 
-                                let audio_data = whisper_rs::convert_stereo_to_mono_audio(&buffer).unwrap();
+                                let audio_data =
+                                    whisper_rs::convert_stereo_to_mono_audio(&buffer).unwrap();
 
-                                state.full(params, &audio_data).expect("failed to run model");
+                                state
+                                    .full(params, &audio_data)
+                                    .expect("failed to run model");
 
-                                let num_segments = state.full_n_segments().expect("failed to get number of segments");
+                                let num_segments = state
+                                    .full_n_segments()
+                                    .expect("failed to get number of segments");
                                 for i in 0..num_segments {
-                                    let segment = state.full_get_segment_text(i).expect("failed to get segment");
+                                    let segment = state
+                                        .full_get_segment_text(i)
+                                        .expect("failed to get segment");
                                     println!("Segment {}: {}", i, segment);
 
                                     let start_timestamp = state
@@ -106,7 +124,10 @@ pub fn run<T>(
                                     let end_timestamp = state
                                         .full_get_segment_t1(i)
                                         .expect("failed to get segment end timestamp");
-                                    println!("[{} - {}]: {}", start_timestamp, end_timestamp, segment);
+                                    println!(
+                                        "[{} - {}]: {}",
+                                        start_timestamp, end_timestamp, segment
+                                    );
                                 }
 
                                 // Clear the buffer

apps/shinkai-desktop/src-tauri/src/commands/galxe.rs

@@ -1,6 +1,9 @@
 use crate::galxe;
 
 #[tauri::command]
-pub fn galxe_generate_proof(node_signature: &str, payload: &str) -> Result<(String, String), String> {
+pub fn galxe_generate_proof(
+    node_signature: &str,
+    payload: &str,
+) -> Result<(String, String), String> {
     galxe::generate_proof(node_signature.to_string(), payload.to_string())
 }

apps/shinkai-desktop/src-tauri/src/commands/mod.rs

@@ -1,3 +1,4 @@
-pub mod shinkai_node_manager_commands;
+pub mod galxe;
 pub mod hardware;
-pub mod galxe;
+pub mod shinkai_node_manager_commands;
+pub mod spotlight_commands;

apps/shinkai-desktop/src-tauri/src/commands/shinkai_node_manager_commands.rs

@@ -1,6 +1,12 @@
 use crate::globals::SHINKAI_NODE_MANAGER_INSTANCE;
 use crate::local_shinkai_node::process_handlers::logger::LogEntry;
 use crate::local_shinkai_node::shinkai_node_options::ShinkaiNodeOptions;
+use crate::windows::{show_or_recreate_window, Window};
+
+#[tauri::command]
+pub async fn show_shinkai_node_manager_window(app_handle: tauri::AppHandle) {
+    show_or_recreate_window(app_handle, Window::ShinkaiNodeManager);
+}
 
 #[tauri::command]
 pub async fn shinkai_node_is_running() -> Result<bool, String> {

apps/shinkai-desktop/src-tauri/src/commands/spotlight_commands.rs

@@ -0,0 +1,6 @@
+use crate::windows::hide_spotlight_window;
+
+#[tauri::command]
+pub async fn hide_spotlight_window_app(app_handle: tauri::AppHandle) {
+    hide_spotlight_window(app_handle)
+}

apps/shinkai-desktop/src-tauri/src/galxe.rs

@@ -11,44 +11,43 @@ pub fn unsafe_deterministic_signature_keypair(bytes: &[u8]) -> (SigningKey, Veri
     (secret_key, public_key)
 }
 
-pub fn generate_proof(
-  node_signature: String,
-  payload: String,
-) -> Result<(String, String), String> {
-  let secret_desktop_key: &str =
-      option_env!("SECRET_DESKTOP_INSTALLATION_PROOF_KEY").unwrap_or("Dc9{3R9JmXe7£w9Fs](7");
-  let (secret_key, public_key) =
-      unsafe_deterministic_signature_keypair(node_signature.as_bytes());
-  // Convert the public key to hex
-  let public_key_hex = hex::encode(public_key.to_bytes());
+pub fn generate_proof(node_signature: String, payload: String) -> Result<(String, String), String> {
+    let secret_desktop_key: &str =
+        option_env!("SECRET_DESKTOP_INSTALLATION_PROOF_KEY").unwrap_or("Dc9{3R9JmXe7£w9Fs](7");
+    let (secret_key, public_key) =
+        unsafe_deterministic_signature_keypair(node_signature.as_bytes());
+    // Convert the public key to hex
+    let public_key_hex = hex::encode(public_key.to_bytes());
 
-  // Combine the public key hex and the secret desktop key
-  let combined = format!("{}{}", public_key_hex, secret_desktop_key);
+    // Combine the public key hex and the secret desktop key
+    let combined = format!("{}{}", public_key_hex, secret_desktop_key);
 
-  // Hash the combined value and take the last 4 characters
-  let mut hasher = Hasher::new();
-  hasher.update(combined.as_bytes());
-  let hash_result = hasher.finalize();
-  let hash_str = hex::encode(hash_result.as_bytes());
-  let last_8_chars = &hash_str[hash_str.len() - 8..];
+    // Hash the combined value and take the last 4 characters
+    let mut hasher = Hasher::new();
+    hasher.update(combined.as_bytes());
+    let hash_result = hasher.finalize();
+    let hash_str = hex::encode(hash_result.as_bytes());
+    let last_8_chars = &hash_str[hash_str.len() - 8..];
 
-  // Concatenate the public key hex with the last 4 characters and JSON string using :::
-  let concatenated = format!(
-      "{}:::{}:::{}",
-      public_key_hex, last_8_chars, base64::encode(payload.as_bytes())
-  );
+    // Concatenate the public key hex with the last 4 characters and JSON string using :::
+    let concatenated = format!(
+        "{}:::{}:::{}",
+        public_key_hex,
+        last_8_chars,
+        base64::encode(payload.as_bytes())
+    );
 
-  // Hash the concatenated string
-  let mut hasher = Hasher::new();
-  hasher.update(concatenated.as_bytes());
-  let final_hash_result = hasher.finalize();
-  let final_hash_bytes = final_hash_result.as_bytes();
+    // Hash the concatenated string
+    let mut hasher = Hasher::new();
+    hasher.update(concatenated.as_bytes());
+    let final_hash_result = hasher.finalize();
+    let final_hash_bytes = final_hash_result.as_bytes();
 
-  // Sign the final hash
-  let signature = secret_key.sign(final_hash_bytes);
+    // Sign the final hash
+    let signature = secret_key.sign(final_hash_bytes);
 
-  // Return the signature as a hexadecimal string and the concatenated string
-  Ok((hex::encode(signature.to_bytes()), concatenated))
+    // Return the signature as a hexadecimal string and the concatenated string
+    Ok((hex::encode(signature.to_bytes()), concatenated))
 }
 
 #[cfg(test)]
@@ -58,7 +57,8 @@ mod tests {
     #[test]
     fn test_generate_proof() {
         let node_signature = "test_node_signature".to_string();
-        let json_string = r#"{"number_of_qa_subscriptions":3, "number_of_subscriptions":5}"#.to_string();
+        let json_string =
+            r#"{"number_of_qa_subscriptions":3, "number_of_subscriptions":5}"#.to_string();
         let result = generate_proof(node_signature, json_string);
         assert!(result.is_ok());
         let expected_signature = "5d59de6abc995c26e95cd88cb78983200e7ef0984761eeae2fad311e500adfcdf95c15b11e55b673052269a3b46e075a760a8a952521dd36057574f55f7e8d0f";

apps/shinkai-desktop/src-tauri/src/global_shortcuts/create_chat.rs

@@ -0,0 +1,16 @@
+use tauri::{Emitter, Manager};
+use tauri_plugin_global_shortcut::{Shortcut, ShortcutEvent};
+
+use crate::windows::{show_or_recreate_window, Window};
+
+pub fn create_chat(app: &tauri::AppHandle, _: Shortcut, _: ShortcutEvent) {
+    show_or_recreate_window(app.clone(), Window::Main);
+    if let Some(window) = app.get_webview_window("main") {
+        if let Err(e) = app.emit("create-chat", ()) {
+            log::error!("failed to emit 'create-chat': {}", e);
+        }
+        if let Err(e) = window.set_focus() {
+            log::error!("failed to set focus: {}", e);
+        }
+    }
+}

apps/shinkai-desktop/src-tauri/src/global_shortcuts/mod.rs

@@ -0,0 +1,21 @@
+use tauri_plugin_global_shortcut::{Code, Modifiers, Shortcut, ShortcutEvent, ShortcutState};
+
+mod create_chat;
+mod toggle_spotlight;
+
+pub fn global_shortcut_handler(app: &tauri::AppHandle, shortcut: Shortcut, event: ShortcutEvent) {
+    if event.state != ShortcutState::Pressed {
+        return;
+    }
+    match shortcut {
+        s if s.matches(Modifiers::SUPER | Modifiers::SHIFT, Code::KeyI) => {
+            create_chat::create_chat(app, shortcut, event);
+        }
+        s if s.matches(Modifiers::SUPER | Modifiers::SHIFT, Code::KeyJ) => {
+            toggle_spotlight::toggle_spotlight(app, shortcut, event);
+        }
+        _ => {
+          log::warn!("unhandled shortcut: {:?}", shortcut);
+        }
+    }
+}

apps/shinkai-desktop/src-tauri/src/global_shortcuts/toggle_spotlight.rs

@@ -0,0 +1,14 @@
+use tauri::Manager;
+use tauri_plugin_global_shortcut::{Shortcut, ShortcutEvent};
+
+use crate::windows::{show_or_recreate_window, Window};
+
+pub fn toggle_spotlight(app: &tauri::AppHandle, _: Shortcut, _: ShortcutEvent) {
+    if let Some(spotlight_window) = app.get_webview_window(Window::Spotlight.as_str()) {
+        if spotlight_window.is_visible().unwrap_or(false) && spotlight_window.is_focused().unwrap_or(false) {
+            let _ = spotlight_window.hide();
+            return;
+        }
+    }
+    show_or_recreate_window(app.clone(), Window::Spotlight)
+}

apps/shinkai-desktop/src-tauri/src/globals.rs

@@ -4,4 +4,5 @@ use crate::local_shinkai_node::shinkai_node_manager::ShinkaiNodeManager;
 use once_cell::sync::OnceCell;
 use tokio::sync::Mutex;
 
-pub static SHINKAI_NODE_MANAGER_INSTANCE: OnceCell<Arc<Mutex<ShinkaiNodeManager>>> = OnceCell::new();
+pub static SHINKAI_NODE_MANAGER_INSTANCE: OnceCell<Arc<Mutex<ShinkaiNodeManager>>> =
+    OnceCell::new();

apps/shinkai-desktop/src-tauri/src/hardware.rs

@@ -104,7 +104,7 @@ pub fn hardware_get_summary() -> HardwareSummary {
     let cpus = sys.cpus().len();
 
     let requirement_status;
-    if  is_macos || (cpus >= RECOMMENDED_CPUS && memory >= RECOMMENDED_MEMORY && discrete_gpu) {
+    if is_macos || (cpus >= RECOMMENDED_CPUS && memory >= RECOMMENDED_MEMORY && discrete_gpu) {
         requirement_status = RequirementsStatus::Optimal;
     } else if cpus >= RECOMMENDED_CPUS && memory >= RECOMMENDED_MEMORY {
         requirement_status = RequirementsStatus::Recommended;

apps/shinkai-desktop/src-tauri/src/local_shinkai_node/mod.rs

@@ -1,4 +1,4 @@
+pub mod ollama_api;
 pub mod process_handlers;
 pub mod shinkai_node_manager;
 pub mod shinkai_node_options;
-pub mod ollama_api;

apps/shinkai-desktop/src-tauri/src/local_shinkai_node/ollama_api/mod.rs

@@ -1,2 +1,2 @@
-pub mod ollama_api_types;
 pub mod ollama_api_client;
+pub mod ollama_api_types;