Group the indexing failures by cause

app/MindWork AI Studio/Pages/Embeddings.razor.cs

@@ -1,5 +1,7 @@
 using AIStudio.Components;
+using AIStudio.Provider;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Rust;
 using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
@@ -8,12 +10,20 @@ namespace AIStudio.Pages;
 
 public partial class Embeddings : MSGComponentBase
 {
+    private static readonly int[] PAGE_SIZE_OPTIONS = [10, 25, 50, 100];
+
     [Inject]
     private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
 
     [Inject]
     private NavigationManager NavigationManager { get; init; } = null!;
 
+    [Inject]
+    private RustService RustService { get; init; } = null!;
+
+    [Inject]
+    private ILogger<Embeddings> Logger { get; init; } = null!;
+
     private IReadOnlyList<DataSourceEmbeddingStatus> Statuses { get; set; } = [];
 
     private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles);
@@ -73,19 +83,104 @@ public partial class Embeddings : MSGComponentBase
     };
 
     /// <summary>
-    /// Names the list of failures for what it holds.
+    /// What a group of failures has in common.
     /// </summary>
     /// <remarks>
-    /// A data source whose files were all skipped for good has nothing wrong with it, so calling
-    /// the list failures would contradict the green state right above it.
+    /// Also the key the failures are grouped by: two of them belong together exactly when the
+    /// list would say the same thing about both.
     /// </remarks>
-    private string GetFailureListHeader(DataSourceEmbeddingStatus status) => status.FailedFiles > 0
-        ? string.Format(T("Failure details ({0})"), status.Failures.Count)
-        : string.Format(T("Skipped files ({0})"), status.Failures.Count);
+    private sealed record FailureCause(int Priority, string Title, string Icon, Color Color, bool IsPermanent, bool NeedsProviderSettings, bool ShowsMessagePerFile);
 
-    private static string GetFailureListIcon(DataSourceEmbeddingStatus status) => status.FailedFiles > 0
-        ? Icons.Material.Filled.ReportProblem
-        : Icons.Material.Filled.SkipNext;
+    private sealed record FailureGroup(FailureCause Cause, string EmbeddingProviderName, IReadOnlyList<DataSourceEmbeddingFailure> Failures);
+
+    /// <summary>
+    /// Puts the failures of a data source into one group per cause, the most pressing one first.
+    /// </summary>
+    /// <remarks>
+    /// Within a priority, the largest group comes first: it is the one telling the user the most
+    /// about their folder.
+    /// </remarks>
+    private IReadOnlyList<FailureGroup> GetFailureGroups(DataSourceEmbeddingStatus status) => status.Failures
+        .GroupBy(this.GetFailureCause)
+        .Select(group => new FailureGroup(group.Key, GetEmbeddingProviderName(group), group.OrderBy(failure => failure.FilePath, StringComparer.OrdinalIgnoreCase).ToList()))
+        .OrderBy(group => group.Cause.Priority)
+        .ThenByDescending(group => group.Failures.Count)
+        .ThenBy(group => group.Cause.Title, StringComparer.OrdinalIgnoreCase)
+        .ToList();
+
+    private FailureCause GetFailureCause(DataSourceEmbeddingFailure failure)
+    {
+        //
+        // Anything the provider answered comes first: it stops the entire data source, while a
+        // file nobody can read costs that one file:
+        //
+        if (failure.FailureReason is not ProviderRequestFailureReason.NONE)
+        {
+            var isFixedInSettings = failure.FailureReason.IsFixedInProviderSettings();
+            return new FailureCause(isFixedInSettings ? 0 : 1, failure.FailureReason.GetName(), isFixedInSettings ? Icons.Material.Filled.Key : Icons.Material.Filled.CloudOff, Color.Error, false, isFixedInSettings, true);
+        }
+
+        //
+        // Codes without a name of their own carry everything they know in the message of the
+        // single file, which is why those groups show that message per file:
+        //
+        var causeName = failure.ExtractionCode.GetIndexingCauseName();
+        var hasCauseName = !string.IsNullOrWhiteSpace(causeName);
+        var title = hasCauseName ? causeName : T("Other cause");
+
+        return failure.IsPermanent
+            ? new FailureCause(3, title, Icons.Material.Filled.SkipNext, Color.Default, true, false, !hasCauseName)
+            : new FailureCause(2, title, Icons.Material.Filled.ReportProblem, Color.Warning, false, false, !hasCauseName);
+    }
+
+    private static string GetEmbeddingProviderName(IEnumerable<DataSourceEmbeddingFailure> failures) => failures
+        .Select(failure => failure.EmbeddingProviderName)
+        .FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)) ?? string.Empty;
+
+    private static string GetGroupHeader(FailureGroup group) => $"{group.Cause.Title} ({group.Failures.Count})";
+
+    private static string GetFileName(string filePath)
+    {
+        var fileName = Path.GetFileName(filePath);
+        return string.IsNullOrWhiteSpace(fileName) ? filePath : fileName;
+    }
+
+    private static string GetOccurrenceText(DataSourceEmbeddingFailure failure) => failure.OccurredAtUtc > DateTimeOffset.MinValue ? failure.OccurredAtUtc.ToLocalTime().ToString("g") : string.Empty;
+
+    /// <remarks>
+    /// A failure which was not about one file, such as a folder which is gone, carries the name of
+    /// the data source instead of a path. There is nothing to show for those.
+    /// </remarks>
+    private static bool CanShowInFileManager(DataSourceEmbeddingFailure failure) => !string.IsNullOrWhiteSpace(failure.FilePath) && Path.IsPathRooted(failure.FilePath);
+
+    /// <summary>
+    /// Opens the file browser of the system and selects the file in it.
+    /// </summary>
+    /// <remarks>
+    /// Reading that a file could not be indexed is where the work starts, not where it ends: the
+    /// file has to be opened, replaced, or run through an OCR. This is the same way out the log
+    /// viewer offers for the log files.
+    /// </remarks>
+    private async Task ShowInFileManager(DataSourceEmbeddingFailure failure)
+    {
+        OpenPathResponse response;
+        try
+        {
+            response = await this.RustService.TryOpenPathInRuntimeFileManager(failure.FilePath);
+        }
+        catch (Exception e)
+        {
+            this.Logger.LogWarning(e, "Could not show a file of the embedding failure list in the file manager.");
+            await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, T("Could not open the file location.")));
+            return;
+        }
+
+        if (response.Success)
+            return;
+
+        var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue;
+        await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the file location: {0}"), issue)));
+    }
 
     private bool CanRefresh(DataSourceEmbeddingStatus status)
     {