From cbda2bdf649c18aca8e11da242eda70ea53a004d Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sat, 24 Jan 2026 21:09:20 +0100 Subject: [PATCH] Added Rust availability monitor and improved error handling --- app/MindWork AI Studio/Program.cs | 3 +- app/MindWork AI Studio/Tools/Event.cs | 3 +- .../Services/EnterpriseEnvironmentService.cs | 3 + .../RustAvailabilityMonitorService.cs | 75 +++++++++++++++++++ .../Tools/Services/RustService.Events.cs | 3 +- .../Tools/Services/RustService.cs | 4 +- .../wwwroot/changelog/v26.1.2.md | 1 + 7 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Services/RustAvailabilityMonitorService.cs diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index c577b2f4..9c0e2bcd 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -134,6 +134,7 @@ internal sealed class Program builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); // ReSharper disable AccessToDisposedClosure builder.Services.AddHostedService(_ => rust); @@ -230,4 +231,4 @@ internal sealed class Program PluginFactory.Dispose(); programLogger.LogInformation("The AI Studio server was stopped."); } -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Tools/Event.cs b/app/MindWork AI Studio/Tools/Event.cs index 0590c5c7..b3d3628f 100644 --- a/app/MindWork AI Studio/Tools/Event.cs +++ b/app/MindWork AI Studio/Tools/Event.cs @@ -15,6 +15,7 @@ public enum Event SHOW_WARNING, SHOW_SUCCESS, TAURI_EVENT_RECEIVED, + RUST_SERVICE_UNAVAILABLE, // Update events: USER_SEARCH_FOR_UPDATE, @@ -53,4 +54,4 @@ public enum Event SEND_TO_MY_TASKS_ASSISTANT, SEND_TO_JOB_POSTING_ASSISTANT, SEND_TO_DOCUMENT_ANALYSIS_ASSISTANT, -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Tools/Services/EnterpriseEnvironmentService.cs b/app/MindWork AI Studio/Tools/Services/EnterpriseEnvironmentService.cs index ff727129..44645dc7 100644 --- a/app/MindWork AI Studio/Tools/Services/EnterpriseEnvironmentService.cs +++ b/app/MindWork AI Studio/Tools/Services/EnterpriseEnvironmentService.cs @@ -42,6 +42,7 @@ public sealed class EnterpriseEnvironmentService(ILogger logger; + private readonly MessageBus messageBus; + private readonly IHostApplicationLifetime appLifetime; + + private int rustUnavailableCount; + + // To prevent multiple shutdown triggers. We use int instead of bool for Interlocked operations. + private int shutdownTriggered; + + public RustAvailabilityMonitorService( + ILogger logger, + MessageBus messageBus, + IHostApplicationLifetime appLifetime) + { + this.logger = logger; + this.messageBus = messageBus; + this.appLifetime = appLifetime; + + this.messageBus.RegisterComponent(this); + this.ApplyFilters([], [Event.RUST_SERVICE_UNAVAILABLE]); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + this.logger.LogInformation("The Rust availability monitor service was initialized."); + await Task.Delay(Timeout.InfiniteTimeSpan, stoppingToken); + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + this.messageBus.Unregister(this); + await base.StopAsync(cancellationToken); + } + + public Task ProcessMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) + { + if (triggeredEvent is not Event.RUST_SERVICE_UNAVAILABLE) + return Task.CompletedTask; + + var reason = data switch + { + string s when !string.IsNullOrWhiteSpace(s) => s, + _ => "unknown reason", + }; + + // Thread-safe incrementation of the unavailable count and check against the threshold: + var numEvents = Interlocked.Increment(ref this.rustUnavailableCount); + if (numEvents <= UNAVAILABLE_EVENT_THRESHOLD) + { + this.logger.LogWarning("Rust service unavailable (num repeats={NumRepeats}, threshold={Threshold}). Reason = '{Reason}'. Waiting for more occurrences before shutting down the server.", numEvents, UNAVAILABLE_EVENT_THRESHOLD, reason); + return Task.CompletedTask; + } + + // Ensure shutdown is only triggered once: + if (Interlocked.Exchange(ref this.shutdownTriggered, 1) != 0) + return Task.CompletedTask; + + this.logger.LogError("Rust service unavailable (num repeats={NumRepeats}, threshold={Threshold}). Reason = '{Reason}'. Shutting down the server.", numEvents, UNAVAILABLE_EVENT_THRESHOLD, reason); + this.appLifetime.StopApplication(); + return Task.CompletedTask; + } + + public Task ProcessMessageWithResult(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data) + { + return Task.FromResult(default); + } +} diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Events.cs b/app/MindWork AI Studio/Tools/Services/RustService.Events.cs index e4d72a0f..62538938 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Events.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Events.cs @@ -62,6 +62,7 @@ public partial class RustService catch (Exception e) { this.logger!.LogError("Error while streaming Tauri events: {Message}", e.Message); + await this.ReportRustServiceUnavailable("Tauri event stream error"); await Task.Delay(TimeSpan.FromSeconds(3), stopToken); } } @@ -74,4 +75,4 @@ public partial class RustService this.logger!.LogWarning("Stopped streaming Tauri events."); } -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Tools/Services/RustService.cs b/app/MindWork AI Studio/Tools/Services/RustService.cs index 6272378c..5d4e2b08 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.cs @@ -69,6 +69,8 @@ public sealed partial class RustService : BackgroundService this.encryptor = encryptionService; } + private Task ReportRustServiceUnavailable(string reason) => MessageBus.INSTANCE.SendMessage(null, Event.RUST_SERVICE_UNAVAILABLE, reason); + #region Overrides of BackgroundService /// @@ -90,4 +92,4 @@ public sealed partial class RustService : BackgroundService } #endregion -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.1.2.md b/app/MindWork AI Studio/wwwroot/changelog/v26.1.2.md index 2199c74e..14629b44 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.1.2.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.1.2.md @@ -10,6 +10,7 @@ - Improved the developer experience by detecting incorrect CPU architecture metadata when checking and installing the Pandoc dependency. - Improved the error messages for failed communication with AI servers. - Improved the error handling for the enterprise environment service regarding the communication with our Rust layer. +- Improved the error handling when continuous networking issues towards our Rust layer occur. - Fixed a logging bug that prevented log events from being recorded in some cases. - Fixed a bug that allowed adding a provider (LLM, embedding, or transcription) without selecting a model. - Fixed a bug with local transcription providers by handling errors correctly when the local provider is unavailable.