Added a preview of the document analysis assistant (#561)

Co-authored-by: Peer Schütt <20603780+peerschuett@users.noreply.github.com>

runtime/src/app_window.rs

@@ -1,14 +1,16 @@
 use std::sync::Mutex;
 use std::time::Duration;
-use log::{error, info, trace, warn};
+use log::{debug, error, info, trace, warn};
 use once_cell::sync::Lazy;
 use rocket::{get, post};
+use rocket::response::stream::TextStream;
 use rocket::serde::json::Json;
 use rocket::serde::Serialize;
 use serde::Deserialize;
 use tauri::updater::UpdateResponse;
-use tauri::{Manager, PathResolver, Window};
+use tauri::{FileDropEvent, UpdaterEvent, RunEvent, Manager, PathResolver, Window, WindowEvent};
 use tauri::api::dialog::blocking::FileDialogBuilder;
+use tokio::sync::broadcast;
 use tokio::time;
 use crate::api_token::APIToken;
 use crate::dotnet::stop_dotnet_server;
@@ -22,18 +24,70 @@ static MAIN_WINDOW: Lazy<Mutex<Option<Window>>> = Lazy::new(|| Mutex::new(None))
 /// The update response coming from the Tauri updater.
 static CHECK_UPDATE_RESPONSE: Lazy<Mutex<Option<UpdateResponse<tauri::Wry>>>> = Lazy::new(|| Mutex::new(None));
 
+/// The event broadcast sender for Tauri events.
+static EVENT_BROADCAST: Lazy<Mutex<Option<broadcast::Sender<Event>>>> = Lazy::new(|| Mutex::new(None));
+
 /// Starts the Tauri app.
 pub fn start_tauri() {
     info!("Starting Tauri app...");
+
+    // Create the event broadcast channel:
+    let (event_sender, root_event_receiver) = broadcast::channel(100);
+
+    // Save a copy of the event broadcast sender for later use:
+    *EVENT_BROADCAST.lock().unwrap() = Some(event_sender.clone());
+
+    // When the last receiver is dropped, we lose the ability to send events.
+    // Therefore, we spawn a task that keeps the root receiver alive:
+    tauri::async_runtime::spawn(async move {
+        let mut root_receiver = root_event_receiver;
+        loop {
+            match root_receiver.recv().await {
+                Ok(event) => {
+                    debug!(Source = "Tauri"; "Tauri event received: location=root receiver       , event={event:?}");
+                },
+
+                Err(broadcast::error::RecvError::Lagged(skipped)) => {
+                    warn!(Source = "Tauri"; "Root event receiver lagged, skipped {skipped} messages.");
+                },
+
+                Err(broadcast::error::RecvError::Closed) => {
+                    warn!(Source = "Tauri"; "Root event receiver channel closed.");
+                    return;
+                },
+            }
+        }
+    });
+
     let app = tauri::Builder::default()
         .setup(move |app| {
+
+            // Get the main window:
             let window = app.get_window("main").expect("Failed to get main window.");
+
+            // Register a callback for window events, such as file drops. We have to use
+            // this handler in addition to the app event handler, because file drop events
+            // are only available in the window event handler (is a bug, cf. https://github.com/tauri-apps/tauri/issues/14338):
+            window.on_window_event(move |event| {
+                debug!(Source = "Tauri"; "Tauri event received: location=window event handler, event={event:?}");
+                let event_to_send = Event::from_window_event(event);
+                let sender = event_sender.clone();
+                tauri::async_runtime::spawn(async move {
+                    match sender.send(event_to_send) {
+                        Ok(_) => {},
+                        Err(error) => error!(Source = "Tauri"; "Failed to channel window event: {error}"),
+                    }
+                });
+            });
+
+            // Save the main window for later access:
             *MAIN_WINDOW.lock().unwrap() = Some(window);
 
             info!(Source = "Bootloader Tauri"; "Setup is running.");
             let data_path = app.path_resolver().app_local_data_dir().unwrap();
             let data_path = data_path.join("data");
 
+            // Get and store the data and config directories:
             DATA_DIRECTORY.set(data_path.to_str().unwrap().to_string()).map_err(|_| error!("Was not abe to set the data directory.")).unwrap();
             CONFIG_DIRECTORY.set(app.path_resolver().app_config_dir().unwrap().to_str().unwrap().to_string()).map_err(|_| error!("Was not able to set the config directory.")).unwrap();
 
@@ -46,84 +100,84 @@ pub fn start_tauri() {
         .build(tauri::generate_context!())
         .expect("Error while running Tauri application");
 
-    app.run(|app_handle, event| match event {
-
-        tauri::RunEvent::WindowEvent { event, label, .. } => {
-            match event {
-                tauri::WindowEvent::CloseRequested { .. } => {
-                    warn!(Source = "Tauri"; "Window '{label}': close was requested.");
-                }
-
-                tauri::WindowEvent::Destroyed => {
-                    warn!(Source = "Tauri"; "Window '{label}': was destroyed.");
-                }
-
-                tauri::WindowEvent::FileDrop(files) => {
-                    info!(Source = "Tauri"; "Window '{label}': files were dropped: {files:?}");
-                }
-
-                _ => (),
-            }
+    // The app event handler:
+    app.run(|app_handle, event| {
+        if !matches!(event, RunEvent::MainEventsCleared) {
+            debug!(Source = "Tauri"; "Tauri event received: location=app event handler   , event={event:?}");
         }
-
-        tauri::RunEvent::Updater(updater_event) => {
-            match updater_event {
-
-                tauri::UpdaterEvent::UpdateAvailable { body, date, version } => {
-                    let body_len = body.len();
-                    info!(Source = "Tauri"; "Updater: update available: body size={body_len} time={date:?} version={version}");
-                }
-
-                tauri::UpdaterEvent::Pending => {
-                    info!(Source = "Tauri"; "Updater: update is pending!");
-                }
-
-                tauri::UpdaterEvent::DownloadProgress { chunk_length, content_length: _ } => {
-                    trace!(Source = "Tauri"; "Updater: downloading chunk of {chunk_length} bytes");
-                }
-
-                tauri::UpdaterEvent::Downloaded => {
-                    info!(Source = "Tauri"; "Updater: update has been downloaded!");
-                    warn!(Source = "Tauri"; "Try to stop the .NET server now...");
-
-                    if is_prod() {
-                        stop_dotnet_server();
-                    } else {
-                        warn!(Source = "Tauri"; "Development environment detected; do not stop the .NET server.");
-                    }
-                }
-
-                tauri::UpdaterEvent::Updated => {
-                    info!(Source = "Tauri"; "Updater: app has been updated");
-                    warn!(Source = "Tauri"; "Try to restart the app now...");
-
-                    if is_prod() {
-                        app_handle.restart();
-                    } else {
-                        warn!(Source = "Tauri"; "Development environment detected; do not restart the app.");
+        
+        match event {
+            RunEvent::WindowEvent { event, label, .. } => {
+                match event {
+                    WindowEvent::CloseRequested { .. } => {
+                        warn!(Source = "Tauri"; "Window '{label}': close was requested.");
                     }
 
-                }
+                    WindowEvent::Destroyed => {
+                        warn!(Source = "Tauri"; "Window '{label}': was destroyed.");
+                    }
 
-                tauri::UpdaterEvent::AlreadyUpToDate => {
-                    info!(Source = "Tauri"; "Updater: app is already up to date");
-                }
-
-                tauri::UpdaterEvent::Error(error) => {
-                    warn!(Source = "Tauri"; "Updater: failed to update: {error}");
+                    _ => (),
                 }
             }
-        }
 
-        tauri::RunEvent::ExitRequested { .. } => {
-            warn!(Source = "Tauri"; "Run event: exit was requested.");
-        }
+            RunEvent::Updater(updater_event) => {
+                match updater_event {
+                    UpdaterEvent::UpdateAvailable { body, date, version } => {
+                        let body_len = body.len();
+                        info!(Source = "Tauri"; "Updater: update available: body size={body_len} time={date:?} version={version}");
+                    }
 
-        tauri::RunEvent::Ready => {
-            info!(Source = "Tauri"; "Run event: Tauri app is ready.");
-        }
+                    UpdaterEvent::Pending => {
+                        info!(Source = "Tauri"; "Updater: update is pending!");
+                    }
 
-        _ => {}
+                    UpdaterEvent::DownloadProgress { chunk_length, content_length: _ } => {
+                        trace!(Source = "Tauri"; "Updater: downloading chunk of {chunk_length} bytes");
+                    }
+
+                    UpdaterEvent::Downloaded => {
+                        info!(Source = "Tauri"; "Updater: update has been downloaded!");
+                        warn!(Source = "Tauri"; "Try to stop the .NET server now...");
+
+                        if is_prod() {
+                            stop_dotnet_server();
+                        } else {
+                            warn!(Source = "Tauri"; "Development environment detected; do not stop the .NET server.");
+                        }
+                    }
+
+                    UpdaterEvent::Updated => {
+                        info!(Source = "Tauri"; "Updater: app has been updated");
+                        warn!(Source = "Tauri"; "Try to restart the app now...");
+
+                        if is_prod() {
+                            app_handle.restart();
+                        } else {
+                            warn!(Source = "Tauri"; "Development environment detected; do not restart the app.");
+                        }
+                    }
+
+                    UpdaterEvent::AlreadyUpToDate => {
+                        info!(Source = "Tauri"; "Updater: app is already up to date");
+                    }
+
+                    UpdaterEvent::Error(error) => {
+                        warn!(Source = "Tauri"; "Updater: failed to update: {error}");
+                    }
+                }
+            }
+
+            RunEvent::ExitRequested { .. } => {
+                warn!(Source = "Tauri"; "Run event: exit was requested.");
+            }
+
+            RunEvent::Ready => {
+                info!(Source = "Tauri"; "Run event: Tauri app is ready.");
+            }
+
+            _ => {}
+        }
     });
 
     warn!(Source = "Tauri"; "Tauri app was stopped.");
@@ -133,6 +187,144 @@ pub fn start_tauri() {
     }
 }
 
+/// Our event API endpoint for Tauri events. We try to send an endless stream of events to the client.
+/// If no events are available for a certain time, we send a ping event to keep the connection alive.
+/// When the client disconnects, the stream is closed. But we try to not lose events in between.
+/// The client is expected to reconnect automatically when the connection is closed and continue
+/// listening for events.
+#[get("/events")]
+pub async fn get_event_stream(_token: APIToken) -> TextStream![String] {
+    // Get the lock to the event broadcast sender:
+    let event_broadcast_lock = EVENT_BROADCAST.lock().unwrap();
+
+    // Get and subscribe to the event receiver:
+    let mut event_receiver = event_broadcast_lock.as_ref()
+        .expect("Event sender not initialized.")
+        .subscribe();
+
+    // Drop the lock to allow other access to the sender:
+    drop(event_broadcast_lock);
+
+    // Create the event stream:
+    TextStream! {
+        loop {
+            // Wait at most 3 seconds for an event:
+            match time::timeout(Duration::from_secs(3), event_receiver.recv()).await {
+
+                // Case: we received an event
+                Ok(Ok(event)) => {
+                    // Serialize the event to JSON. Important is that the entire event
+                    // is serialized as a single line so that the client can parse it
+                    // correctly:
+                    let event_json = serde_json::to_string(&event).unwrap();
+                    yield event_json;
+
+                    // The client expects a newline after each event because we are using
+                    // a method to read the stream line-by-line:
+                    yield "\n".to_string();
+                },
+
+                // Case: we lagged behind and missed some events
+                Ok(Err(broadcast::error::RecvError::Lagged(skipped))) => {
+                    warn!(Source = "Tauri"; "Event receiver lagged, skipped {skipped} messages.");
+                },
+
+                // Case: the event channel was closed
+                Ok(Err(broadcast::error::RecvError::Closed)) => {
+                    warn!(Source = "Tauri"; "Event receiver channel closed.");
+                    return;
+                },
+
+                // Case: timeout. We will send a ping event to keep the connection alive.
+                Err(_) => {
+                    let ping_event = Event::new(TauriEventType::Ping, Vec::new());
+
+                    // Again, we have to serialize the event as a single line:
+                    let event_json = serde_json::to_string(&ping_event).unwrap();
+                    yield event_json;
+
+                    // The client expects a newline after each event because we are using
+                    // a method to read the stream line-by-line:
+                    yield "\n".to_string();
+                },
+            }
+        }
+    }
+}
+
+/// Data structure representing a Tauri event for our event API.
+#[derive(Debug, Clone, Serialize)]
+pub struct Event {
+    pub event_type: TauriEventType,
+    pub payload: Vec<String>,
+}
+
+/// Implementation of the Event struct.
+impl Event {
+
+    /// Creates a new Event instance.
+    pub fn new(event_type: TauriEventType, payload: Vec<String>) -> Self {
+        Event {
+            payload,
+            event_type,
+        }
+    }
+
+    /// Creates an Event instance from a Tauri WindowEvent.
+    pub fn from_window_event(window_event: &WindowEvent) -> Self {
+        match window_event {
+            WindowEvent::FileDrop(drop_event) => {
+                match drop_event {
+                    FileDropEvent::Hovered(files) => Event::new(TauriEventType::FileDropHovered,
+                                                                files.iter().map(|f| f.to_string_lossy().to_string()).collect(),
+                    ),
+
+                    FileDropEvent::Dropped(files) => Event::new(TauriEventType::FileDropDropped,
+                                                                files.iter().map(|f| f.to_string_lossy().to_string()).collect(),
+                    ),
+
+                    FileDropEvent::Cancelled => Event::new(TauriEventType::FileDropCanceled,
+                                                           Vec::new(),
+                    ),
+
+                    _ => Event::new(TauriEventType::Unknown,
+                                    Vec::new(),
+                    ),
+                }
+            },
+
+            WindowEvent::Focused(state) => if *state {
+                Event::new(TauriEventType::WindowFocused,
+                           Vec::new(),
+                )
+            } else {
+                Event::new(TauriEventType::WindowNotFocused,
+                           Vec::new(),
+                )
+            },
+
+            _ => Event::new(TauriEventType::Unknown,
+                            Vec::new(),
+            ),
+        }
+    }
+}
+
+/// The types of Tauri events we can send through our event API.
+#[derive(Debug, Serialize, Clone)]
+pub enum TauriEventType {
+    None,
+    Ping,
+    Unknown,
+
+    WindowFocused,
+    WindowNotFocused,
+
+    FileDropHovered,
+    FileDropDropped,
+    FileDropCanceled,
+}
+
 /// Changes the location of the main window to the given URL.
 pub async fn change_location_to(url: &str) {
     // Try to get the main window. If it is not available yet, wait for it:

runtime/src/file_data.rs

@@ -27,6 +27,7 @@ pub struct Chunk {
     pub stream_id: String,
     pub metadata: Metadata,
 }
+
 impl Chunk {
     pub fn new(content: String, metadata: Metadata) -> Self {
         Chunk { content, stream_id: String::new(), metadata }

runtime/src/runtime_api.rs

@@ -68,6 +68,7 @@ pub fn start_runtime_api() {
                 crate::dotnet::dotnet_port,
                 crate::dotnet::dotnet_ready,
                 crate::clipboard::set_clipboard,
+                crate::app_window::get_event_stream,
                 crate::app_window::check_for_update,
                 crate::app_window::install_update,
                 crate::app_window::select_directory,

runtime/tauri.conf.json

@@ -40,7 +40,8 @@
         "resizable": true,
         "title": "MindWork AI Studio",
         "width": 1920,
-        "height": 1080
+        "height": 1080,
+        "fileDropEnabled": true
       }
     ],
     "security": {