Added Rust availability monitor and improved error handling

app/MindWork AI Studio/Program.cs

@@ -134,6 +134,7 @@ internal sealed class Program
         builder.Services.AddHostedService<TemporaryChatService>();
         builder.Services.AddHostedService<EnterpriseEnvironmentService>();
         builder.Services.AddHostedService<GlobalShortcutService>();
+        builder.Services.AddHostedService<RustAvailabilityMonitorService>();
         
         // ReSharper disable AccessToDisposedClosure
         builder.Services.AddHostedService<RustService>(_ => rust);
@@ -230,4 +231,4 @@ internal sealed class Program
         PluginFactory.Dispose();
         programLogger.LogInformation("The AI Studio server was stopped.");
     }
-}
+}

app/MindWork AI Studio/Tools/Event.cs

@@ -15,6 +15,7 @@ public enum Event
     SHOW_WARNING,
     SHOW_SUCCESS,
     TAURI_EVENT_RECEIVED,
+    RUST_SERVICE_UNAVAILABLE,
     
     // Update events:
     USER_SEARCH_FOR_UPDATE,
@@ -53,4 +54,4 @@ public enum Event
     SEND_TO_MY_TASKS_ASSISTANT,
     SEND_TO_JOB_POSTING_ASSISTANT,
     SEND_TO_DOCUMENT_ANALYSIS_ASSISTANT,
-}
+}

app/MindWork AI Studio/Tools/Services/EnterpriseEnvironmentService.cs

@@ -42,6 +42,7 @@ public sealed class EnterpriseEnvironmentService(ILogger<EnterpriseEnvironmentSe
             catch (Exception e)
             {
                 logger.LogError(e, "Failed to fetch the enterprise remove configuration ID from the Rust service.");
+                await MessageBus.INSTANCE.SendMessage(null, Event.RUST_SERVICE_UNAVAILABLE, "EnterpriseEnvRemoveConfigId failed");
                 return;
             }
             
@@ -60,6 +61,7 @@ public sealed class EnterpriseEnvironmentService(ILogger<EnterpriseEnvironmentSe
             catch (Exception e)
             {
                 logger.LogError(e, "Failed to fetch the enterprise configuration server URL from the Rust service.");
+                await MessageBus.INSTANCE.SendMessage(null, Event.RUST_SERVICE_UNAVAILABLE, "EnterpriseEnvConfigServerUrl failed");
                 return;
             }
 
@@ -71,6 +73,7 @@ public sealed class EnterpriseEnvironmentService(ILogger<EnterpriseEnvironmentSe
             catch (Exception e)
             {
                 logger.LogError(e, "Failed to fetch the enterprise configuration ID from the Rust service.");
+                await MessageBus.INSTANCE.SendMessage(null, Event.RUST_SERVICE_UNAVAILABLE, "EnterpriseEnvConfigId failed");
                 return;
             }
             

app/MindWork AI Studio/Tools/Services/RustAvailabilityMonitorService.cs

@@ -0,0 +1,75 @@
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Tools.Services;
+
+public sealed class RustAvailabilityMonitorService : BackgroundService, IMessageBusReceiver
+{
+    private const int UNAVAILABLE_EVENT_THRESHOLD = 2;
+
+    private readonly ILogger<RustAvailabilityMonitorService> logger;
+    private readonly MessageBus messageBus;
+    private readonly IHostApplicationLifetime appLifetime;
+
+    private int rustUnavailableCount;
+    
+    // To prevent multiple shutdown triggers. We use int instead of bool for Interlocked operations.
+    private int shutdownTriggered;
+
+    public RustAvailabilityMonitorService(
+        ILogger<RustAvailabilityMonitorService> logger,
+        MessageBus messageBus,
+        IHostApplicationLifetime appLifetime)
+    {
+        this.logger = logger;
+        this.messageBus = messageBus;
+        this.appLifetime = appLifetime;
+
+        this.messageBus.RegisterComponent(this);
+        this.ApplyFilters([], [Event.RUST_SERVICE_UNAVAILABLE]);
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        this.logger.LogInformation("The Rust availability monitor service was initialized.");
+        await Task.Delay(Timeout.InfiniteTimeSpan, stoppingToken);
+    }
+
+    public override async Task StopAsync(CancellationToken cancellationToken)
+    {
+        this.messageBus.Unregister(this);
+        await base.StopAsync(cancellationToken);
+    }
+
+    public Task ProcessMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data)
+    {
+        if (triggeredEvent is not Event.RUST_SERVICE_UNAVAILABLE)
+            return Task.CompletedTask;
+
+        var reason = data switch
+        {
+            string s when !string.IsNullOrWhiteSpace(s) => s,
+            _ => "unknown reason",
+        };
+        
+        // Thread-safe incrementation of the unavailable count and check against the threshold:
+        var numEvents = Interlocked.Increment(ref this.rustUnavailableCount);
+        if (numEvents <= UNAVAILABLE_EVENT_THRESHOLD)
+        {
+            this.logger.LogWarning("Rust service unavailable (num repeats={NumRepeats}, threshold={Threshold}). Reason = '{Reason}'. Waiting for more occurrences before shutting down the server.", numEvents, UNAVAILABLE_EVENT_THRESHOLD, reason);
+            return Task.CompletedTask;
+        }
+
+        // Ensure shutdown is only triggered once:
+        if (Interlocked.Exchange(ref this.shutdownTriggered, 1) != 0)
+            return Task.CompletedTask;
+
+        this.logger.LogError("Rust service unavailable (num repeats={NumRepeats}, threshold={Threshold}). Reason = '{Reason}'. Shutting down the server.", numEvents, UNAVAILABLE_EVENT_THRESHOLD, reason);
+        this.appLifetime.StopApplication();
+        return Task.CompletedTask;
+    }
+
+    public Task<TResult?> ProcessMessageWithResult<TPayload, TResult>(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data)
+    {
+        return Task.FromResult<TResult?>(default);
+    }
+}

app/MindWork AI Studio/Tools/Services/RustService.Events.cs

@@ -62,6 +62,7 @@ public partial class RustService
                 catch (Exception e)
                 {
                     this.logger!.LogError("Error while streaming Tauri events: {Message}", e.Message);
+                    await this.ReportRustServiceUnavailable("Tauri event stream error");
                     await Task.Delay(TimeSpan.FromSeconds(3), stopToken);
                 }
             }
@@ -74,4 +75,4 @@ public partial class RustService
         
         this.logger!.LogWarning("Stopped streaming Tauri events.");
     }
-}
+}

app/MindWork AI Studio/Tools/Services/RustService.cs

@@ -69,6 +69,8 @@ public sealed partial class RustService : BackgroundService
         this.encryptor = encryptionService;
     }
 
+    private Task ReportRustServiceUnavailable(string reason) => MessageBus.INSTANCE.SendMessage(null, Event.RUST_SERVICE_UNAVAILABLE, reason);
+
     #region Overrides of BackgroundService
 
     /// <summary>
@@ -90,4 +92,4 @@ public sealed partial class RustService : BackgroundService
     }
 
     #endregion
-}
+}

app/MindWork AI Studio/wwwroot/changelog/v26.1.2.md

@@ -10,6 +10,7 @@
 - Improved the developer experience by detecting incorrect CPU architecture metadata when checking and installing the Pandoc dependency.
 - Improved the error messages for failed communication with AI servers.
 - Improved the error handling for the enterprise environment service regarding the communication with our Rust layer.
+- Improved the error handling when continuous networking issues towards our Rust layer occur.
 - Fixed a logging bug that prevented log events from being recorded in some cases.
 - Fixed a bug that allowed adding a provider (LLM, embedding, or transcription) without selecting a model.
 - Fixed a bug with local transcription providers by handling errors correctly when the local provider is unavailable.