Hardened background work against disconnected circuits (#937)

app/MindWork AI Studio/Tools/Services/AIStudioCircuitHandler.cs

@@ -0,0 +1,61 @@
+using Microsoft.AspNetCore.Components.Server.Circuits;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Follows the life of one circuit, so the rest of the app knows when its browser is unreachable.
+/// </summary>
+/// <remarks>
+/// The app keeps disconnected circuits for a long time on purpose, cf. the retention settings in
+/// Program.cs. That is what lets a user return to a working app after the machine woke up — but it also
+/// means that the components of reloaded or sleeping windows stay alive and keep receiving events. They
+/// may keep working: everything they do on the server is fine. Only JavaScript interop is impossible
+/// while the connection is gone. So this handler does two things, and deliberately nothing more:
+/// it publishes the connection state, and it cleans up once a circuit is truly over.
+/// </remarks>
+public sealed class AIStudioCircuitHandler(CircuitStateService circuitState, MessageBus messageBus, ILogger<AIStudioCircuitHandler> logger)
+    : CircuitHandler
+{
+    #region Overrides of CircuitHandler
+
+    public override Task OnCircuitOpenedAsync(Circuit circuit, CancellationToken cancellationToken)
+    {
+        circuitState.AssignCircuit(circuit.Id);
+        logger.LogInformation("The circuit '{CircuitId}' was opened.", circuit.Id);
+
+        return Task.CompletedTask;
+    }
+
+    public override Task OnConnectionUpAsync(Circuit circuit, CancellationToken cancellationToken)
+    {
+        circuitState.MarkAsConnected();
+        logger.LogInformation("The browser connection of the circuit '{CircuitId}' is up.", circuit.Id);
+
+        return Task.CompletedTask;
+    }
+
+    public override Task OnConnectionDownAsync(Circuit circuit, CancellationToken cancellationToken)
+    {
+        circuitState.MarkAsDisconnected();
+        logger.LogInformation("The browser connection of the circuit '{CircuitId}' is down. Its JavaScript interop is paused until it returns.", circuit.Id);
+
+        return Task.CompletedTask;
+    }
+
+    public override Task OnCircuitClosedAsync(Circuit circuit, CancellationToken cancellationToken)
+    {
+        circuitState.MarkAsDisconnected();
+
+        //
+        // The components of this circuit will not come back, so nobody would ever deregister them:
+        // Blazor disposes components of a retained circuit without giving them a chance to run their
+        // disposal in every case. Without this, the message bus would keep and serve them forever.
+        //
+        var numRemovedReceivers = messageBus.UnregisterCircuit(circuitState);
+        logger.LogInformation("The circuit '{CircuitId}' was closed. Removed {NumReceivers} message bus receiver(s) of that circuit.", circuit.Id, numRemovedReceivers);
+
+        return Task.CompletedTask;
+    }
+
+    #endregion
+}

app/MindWork AI Studio/Tools/Services/CircuitStateService.cs

@@ -0,0 +1,46 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Knows whether the browser connection of one circuit is currently up.
+/// </summary>
+/// <remarks>
+/// There is one instance of this service per circuit, i.e. per browser window. It exists because the app
+/// keeps disconnected circuits around for a long time, cf. the retention settings in Program.cs: after a
+/// reload or while the machine sleeps, the components of the old circuit are still alive and still receive
+/// events. They may do their work as before — only JavaScript interop is impossible while the connection
+/// is gone. This is what tells them apart. Only the circuit handler changes this state.
+/// </remarks>
+public sealed class CircuitStateService
+{
+    private volatile bool isConnected = true;
+
+    /// <summary>
+    /// True, as long as the browser of this circuit is reachable, and thus JS interop is possible.
+    /// </summary>
+    /// <remarks>
+    /// This starts as true: a circuit is created for a connected browser, and the handler reports the
+    /// first connection only afterwards. Starting as false would block the interop of the first render.
+    /// </remarks>
+    public bool IsConnected => this.isConnected;
+
+    /// <summary>
+    /// The ID of this circuit, for logging purposes. It is "n/a" until the circuit was opened.
+    /// </summary>
+    public string CircuitId { get; private set; } = "n/a";
+
+    /// <summary>
+    /// Called by the circuit handler when the circuit was opened.
+    /// </summary>
+    /// <param name="circuitId">The ID of the opened circuit.</param>
+    public void AssignCircuit(string circuitId) => this.CircuitId = circuitId;
+
+    /// <summary>
+    /// Called by the circuit handler when the browser connection was established or restored.
+    /// </summary>
+    public void MarkAsConnected() => this.isConnected = true;
+
+    /// <summary>
+    /// Called by the circuit handler when the browser connection was lost or the circuit ended.
+    /// </summary>
+    public void MarkAsDisconnected() => this.isConnected = false;
+}

app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs

@@ -173,7 +173,7 @@ public sealed class MediaTranscriptionService(
         }
 
         this.UpdateImportState(target, Path.GetFileName(mediaPaths[0]), MediaTranscriptionPhase.QUEUED, null, MediaImportStatus.QUEUED);
-        _ = Task.Run(() => this.RunAttachmentBatchAsync(mediaPaths, target, ownerChat));
+        Task.Run(() => this.RunAttachmentBatchAsync(mediaPaths, target, ownerChat)).Observe($"{nameof(MediaTranscriptionService)}: running an attachment batch");
         return true;
     }
 
@@ -191,7 +191,7 @@ public sealed class MediaTranscriptionService(
         }
 
         this.UpdateImportState(target, Path.GetFileName(mediaPath), MediaTranscriptionPhase.QUEUED, null, MediaImportStatus.QUEUED);
-        _ = Task.Run(() => this.RunTextImportAsync(mediaPath, target));
+        Task.Run(() => this.RunTextImportAsync(mediaPath, target)).Observe($"{nameof(MediaTranscriptionService)}: running a text import");
         return true;
     }
 

app/MindWork AI Studio/Tools/Services/RustAvailabilityMonitorService.cs

@@ -72,8 +72,8 @@ public sealed class RustAvailabilityMonitorService : BackgroundService, IMessage
             // be a transient issue.
             //
             
-            _ = this.VerifyRustAvailability();
-            _ = this.VerifyRustAvailability();
+            this.VerifyRustAvailability().Observe($"{nameof(RustAvailabilityMonitorService)}: verifying the Rust availability");
+            this.VerifyRustAvailability().Observe($"{nameof(RustAvailabilityMonitorService)}: verifying the Rust availability");
         }
 
         if (numEvents <= UNAVAILABLE_EVENT_THRESHOLD)

app/MindWork AI Studio/Tools/Services/RustService.Log.cs

@@ -26,7 +26,12 @@ public sealed partial class RustService
     {
         try
         {
-            // Fire-and-forget the log event to avoid blocking:
+            //
+            // Fire-and-forget the log event to avoid blocking. This is the one place which deliberately
+            // discards its task instead of observing it: observing means logging the failure, and logging
+            // means sending another log event through this very method. A broken connection to Rust would
+            // feed itself. The unobserved task exception handler in Program.cs remains the safety net here.
+            //
             var request = new LogEventRequest(timestamp, level, category, message, exception, stackTrace);
             _ = this.http.PostAsJsonAsync("/log/event", request, this.jsonRustSerializerOptions);
         }