Added a preview of the document analysis assistant (#561)

Co-authored-by: Peer Schütt <20603780+peerschuett@users.noreply.github.com>

app/MindWork AI Studio/Tools/Components.cs

@@ -18,6 +18,7 @@ public enum Components
     JOB_POSTING_ASSISTANT,
     BIAS_DAY_ASSISTANT,
     ERI_ASSISTANT,
+    DOCUMENT_ANALYSIS_ASSISTANT,
     
     // ReSharper disable InconsistentNaming
     I18N_ASSISTANT,

app/MindWork AI Studio/Tools/ComponentsExtensions.cs

@@ -42,10 +42,11 @@ public static class ComponentsExtensions
         Components.JOB_POSTING_ASSISTANT => TB("Job Posting Assistant"),
         Components.ERI_ASSISTANT => TB("ERI Server"),
         Components.I18N_ASSISTANT => TB("Localization Assistant"),
+        Components.DOCUMENT_ANALYSIS_ASSISTANT => TB("Document Analysis Assistant"),
         
         Components.CHAT => TB("New Chat"),
         
-        _ => Enum.GetName(typeof(Components), component)!,
+        _ => Enum.GetName(component)!,
     };
 
     public static ComponentsData GetData(this Components destination) => destination switch
@@ -62,6 +63,7 @@ public static class ComponentsExtensions
         Components.SYNONYMS_ASSISTANT => new(Event.SEND_TO_SYNONYMS_ASSISTANT, Routes.ASSISTANT_SYNONYMS),
         Components.MY_TASKS_ASSISTANT => new(Event.SEND_TO_MY_TASKS_ASSISTANT, Routes.ASSISTANT_MY_TASKS),
         Components.JOB_POSTING_ASSISTANT => new(Event.SEND_TO_JOB_POSTING_ASSISTANT, Routes.ASSISTANT_JOB_POSTING),
+        Components.DOCUMENT_ANALYSIS_ASSISTANT => new(Event.SEND_TO_DOCUMENT_ANALYSIS_ASSISTANT, Routes.ASSISTANT_DOCUMENT_ANALYSIS),
         
         Components.CHAT => new(Event.SEND_TO_CHAT, Routes.CHAT),
         
@@ -84,6 +86,9 @@ public static class ComponentsExtensions
         Components.JOB_POSTING_ASSISTANT => settingsManager.ConfigurationData.JobPostings.PreselectOptions ? settingsManager.ConfigurationData.JobPostings.MinimumProviderConfidence : default,
         Components.BIAS_DAY_ASSISTANT => settingsManager.ConfigurationData.BiasOfTheDay.PreselectOptions ? settingsManager.ConfigurationData.BiasOfTheDay.MinimumProviderConfidence : default,
         Components.ERI_ASSISTANT => settingsManager.ConfigurationData.ERI.PreselectOptions ? settingsManager.ConfigurationData.ERI.MinimumProviderConfidence : default,
+        
+        #warning Add minimum confidence for DOCUMENT_ANALYSIS_ASSISTANT:
+        //Components.DOCUMENT_ANALYSIS_ASSISTANT => settingsManager.ConfigurationData.DocumentAnalysis.PreselectOptions ? settingsManager.ConfigurationData.DocumentAnalysis.MinimumProviderConfidence : default,
 
         _ => default,
     };
@@ -108,6 +113,9 @@ public static class ComponentsExtensions
             Components.BIAS_DAY_ASSISTANT => settingsManager.ConfigurationData.BiasOfTheDay.PreselectOptions ? settingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.BiasOfTheDay.PreselectedProvider) : null,
             Components.ERI_ASSISTANT => settingsManager.ConfigurationData.ERI.PreselectOptions ? settingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.ERI.PreselectedProvider) : null,
             Components.I18N_ASSISTANT => settingsManager.ConfigurationData.I18N.PreselectOptions ? settingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.I18N.PreselectedProvider) : null,
+            
+            #warning Add preselected provider for DOCUMENT_ANALYSIS_ASSISTANT:
+            //Components.DOCUMENT_ANALYSIS_ASSISTANT => settingsManager.ConfigurationData.DocumentAnalysis.PreselectOptions ? settingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.DocumentAnalysis.PreselectedProvider) : null,
 
             Components.CHAT => settingsManager.ConfigurationData.Chat.PreselectOptions ? settingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.Chat.PreselectedProvider) : null,
 
@@ -123,6 +131,8 @@ public static class ComponentsExtensions
 
     public static Profile PreselectedProfile(this Components component, SettingsManager settingsManager) => component switch
     {
+        #warning Add preselected profile for DOCUMENT_ANALYSIS_ASSISTANT:
+        // Components.DOCUMENT_ANALYSIS_ASSISTANT => settingsManager.ConfigurationData.DocumentAnalysis.PreselectOptions ? settingsManager.ConfigurationData.Profiles.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.DocumentAnalysis.PreselectedProfile) : default,
         Components.AGENDA_ASSISTANT => settingsManager.ConfigurationData.Agenda.PreselectOptions ? settingsManager.ConfigurationData.Profiles.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.Agenda.PreselectedProfile) ?? Profile.NO_PROFILE : Profile.NO_PROFILE,
         Components.CODING_ASSISTANT => settingsManager.ConfigurationData.Coding.PreselectOptions ? settingsManager.ConfigurationData.Profiles.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.Coding.PreselectedProfile) ?? Profile.NO_PROFILE : Profile.NO_PROFILE,
         Components.EMAIL_ASSISTANT => settingsManager.ConfigurationData.EMail.PreselectOptions ? settingsManager.ConfigurationData.Profiles.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.EMail.PreselectedProfile) ?? Profile.NO_PROFILE : Profile.NO_PROFILE,

app/MindWork AI Studio/Tools/Event.cs

@@ -14,6 +14,7 @@ public enum Event
     SHOW_ERROR,
     SHOW_WARNING,
     SHOW_SUCCESS,
+    TAURI_EVENT_RECEIVED,
     
     // Update events:
     USER_SEARCH_FOR_UPDATE,
@@ -47,4 +48,5 @@ public enum Event
     SEND_TO_SYNONYMS_ASSISTANT,
     SEND_TO_MY_TASKS_ASSISTANT,
     SEND_TO_JOB_POSTING_ASSISTANT,
+    SEND_TO_DOCUMENT_ANALYSIS_ASSISTANT,
 }

app/MindWork AI Studio/Tools/EventHandlers.cs

@@ -0,0 +1,14 @@
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Add handling for more DOM events to Blazor components.
+/// </summary>
+/// <remarks>
+/// See https://learn.microsoft.com/en-us/aspnet/core/blazor/components/event-handling. It is important
+/// that this class is named EventHandlers.
+/// </remarks>
+[EventHandler("onmouseenter", typeof(EventArgs), enableStopPropagation: true, enablePreventDefault: true)]
+[EventHandler("onmouseleave", typeof(EventArgs), enableStopPropagation: true, enablePreventDefault: true)]
+public static class EventHandlers;

app/MindWork AI Studio/Tools/Rust/FileTypeFilter.cs

@@ -19,7 +19,9 @@ public readonly record struct FileTypeFilter(string FilterName, string[] FilterE
     
     public static FileTypeFilter AllOffice => new(TB("All Office Files"), ["docx", "xlsx", "pptx", "doc", "xls", "ppt", "pdf"]);
     
-    public static FileTypeFilter AllImages => new(TB("All Image Files"), ["jpg", "jpeg", "png", "gif", "bmp", "tiff"]);
+    public static FileTypeFilter AllImages => new(TB("All Image Files"), ["jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic"]);
+    
+    public static FileTypeFilter AllVideos => new(TB("All Video Files"), ["mp4", "avi", "mkv", "mov", "wmv", "flv", "webm"]);
     
     public static FileTypeFilter Executables => new(TB("Executable Files"), ["exe", "app", "bin", "appimage"]);
 }

app/MindWork AI Studio/Tools/Rust/TauriEvent.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// The data structure for a Tauri event sent from the Rust backend to the C# frontend.
+/// </summary>
+/// <param name="EventType">The type of the Tauri event.</param>
+/// <param name="Payload">The payload of the Tauri event.</param>
+public readonly record struct TauriEvent(TauriEventType EventType, List<string> Payload);

app/MindWork AI Studio/Tools/Rust/TauriEventType.cs

@@ -0,0 +1,18 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// The type of Tauri events we can receive.
+/// </summary>
+public enum TauriEventType
+{
+    NONE,
+    PING,
+    UNKNOWN,
+    
+    WINDOW_FOCUSED,
+    WINDOW_NOT_FOCUSED,
+    
+    FILE_DROP_HOVERED,
+    FILE_DROP_DROPPED,
+    FILE_DROP_CANCELED,
+}

app/MindWork AI Studio/Tools/Services/RustService.Events.cs

@@ -0,0 +1,77 @@
+using System.Text.Json;
+
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools.Services;
+
+public partial class RustService
+{
+    /// <summary>
+    /// Consume the Tauri event stream and forward relevant events to the message bus.
+    /// </summary>
+    /// <param name="stopToken">Cancellation token to stop the stream.</param>
+    private async Task StartStreamTauriEvents(CancellationToken stopToken)
+    {
+        // Outer try-catch to handle cancellation:
+        try
+        {
+            while (!stopToken.IsCancellationRequested)
+            {
+                // Inner try-catch to handle streaming issues:
+                try
+                {
+                    // Open the event stream:
+                    await using var stream = await this.http.GetStreamAsync("/events", stopToken);
+                    
+                    
+                    // Read events line by line:
+                    using var reader = new StreamReader(stream);
+                    
+                    // Read until the end of the stream or cancellation:
+                    while(!reader.EndOfStream && !stopToken.IsCancellationRequested)
+                    {
+                        // Read the next line of JSON from the stream:
+                        var line = await reader.ReadLineAsync(stopToken);
+                        
+                        // Skip empty lines:
+                        if (string.IsNullOrWhiteSpace(line))
+                            continue;
+                        
+                        // Deserialize the Tauri event:
+                        var tauriEvent = JsonSerializer.Deserialize<TauriEvent>(line, this.jsonRustSerializerOptions);
+                        
+                        // Forward relevant events to the message bus:
+                        if (tauriEvent != default && tauriEvent.EventType
+                                is not TauriEventType.NONE
+                                and not TauriEventType.UNKNOWN
+                                and not TauriEventType.PING)
+                        {
+                            this.logger!.LogDebug("Received Tauri event {EventType} with {NumPayloadItems} payload items.", tauriEvent.EventType, tauriEvent.Payload.Count);
+                            await MessageBus.INSTANCE.SendMessage(null, Event.TAURI_EVENT_RECEIVED, tauriEvent);
+                        }
+                    }
+                }
+                
+                // The cancellation token was triggered, exit the loop:
+                catch (OperationCanceledException)
+                {
+                    break;
+                }
+                
+                // Some other error occurred, log it and retry after a delay:
+                catch (Exception e)
+                {
+                    this.logger!.LogError("Error while streaming Tauri events: {Message}", e.Message);
+                    await Task.Delay(TimeSpan.FromSeconds(3), stopToken);
+                }
+            }
+        }
+        
+        // The cancellation token was triggered, exit the method:
+        catch (OperationCanceledException)
+        {
+        }
+        
+        this.logger!.LogWarning("Stopped streaming Tauri events.");
+    }
+}

app/MindWork AI Studio/Tools/Services/RustService.cs

@@ -1,6 +1,10 @@
 using System.Security.Cryptography;
 using System.Text.Json;
 
+using AIStudio.Settings;
+
+using Version = System.Version;
+
 // ReSharper disable NotAccessedPositionalProperty.Local
 
 namespace AIStudio.Tools.Services;
@@ -8,13 +12,14 @@ namespace AIStudio.Tools.Services;
 /// <summary>
 /// Calling Rust functions.
 /// </summary>
-public sealed partial class RustService : IDisposable
+public sealed partial class RustService : BackgroundService
 {
     private readonly HttpClient http;
 
     private readonly JsonSerializerOptions jsonRustSerializerOptions = new()
     {
         PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
+        Converters = { new TolerantEnumConverter() },
     };
     
     private ILogger<RustService>? logger;
@@ -59,11 +64,24 @@ public sealed partial class RustService : IDisposable
         this.encryptor = encryptionService;
     }
 
-    #region IDisposable
+    #region Overrides of BackgroundService
 
-    public void Dispose()
+    /// <summary>
+    /// The main execution loop of the Rust service as a background thread.
+    /// </summary>
+    /// <param name="stopToken">The cancellation token to stop the service.</param>
+    protected override async Task ExecuteAsync(CancellationToken stopToken)
+    {
+        this.logger?.LogInformation("The Rust service was initialized.");
+        
+        // Start consuming Tauri events:
+        await this.StartStreamTauriEvents(stopToken);
+    }
+    
+    public override void Dispose()
     {
         this.http.Dispose();
+        base.Dispose();
     }
 
     #endregion

app/MindWork AI Studio/Tools/Services/TemporaryChatService.cs

@@ -24,24 +24,24 @@ public sealed class TemporaryChatService(ILogger<TemporaryChatService> logger, S
             return;
         }
 
-        await this.StartMaintenance();
+        this.StartMaintenance();
         while (!stoppingToken.IsCancellationRequested)
         {
             await Task.Delay(CHECK_INTERVAL, stoppingToken);
-            await this.StartMaintenance();
+            this.StartMaintenance();
         }
     }
 
     #endregion
 
-    private Task StartMaintenance()
+    private void StartMaintenance()
     {
         logger.LogInformation("Starting maintenance of temporary chat storage.");
         var temporaryDirectories = Path.Join(SettingsManager.DataDirectory, "tempChats");
         if(!Directory.Exists(temporaryDirectories))
         {
             logger.LogWarning("Temporary chat storage directory does not exist. End maintenance.");
-            return Task.CompletedTask;
+            return;
         }
 
         foreach (var tempChatDirPath in Directory.EnumerateDirectories(temporaryDirectories))
@@ -71,7 +71,6 @@ public sealed class TemporaryChatService(ILogger<TemporaryChatService> logger, S
         }
 
         logger.LogInformation("Finished maintenance of temporary chat storage.");
-        return Task.CompletedTask;
     }
     
     public static void Initialize()