MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-13 11:53:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Group the indexing failures by cause

Browse Source
This commit is contained in:
Thorsten Sommer committed 2026-09-09 13:07:59 +02:00
1 parent f9a989076d
commit 2b18d32bf2
6 files changed
+420 -61
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Pages/Embeddings.razor
+63 -40
View File
@@ -1,6 +1,5 @@
@attribute [Route(Routes.EMBEDDINGS)]
@inherits MSGComponentBase
@using AIStudio.Provider
<MudStack Spacing="3" Class="pr-2 pb-4" Style="height: 100%; min-height: 0; overflow-y: auto;">
<MudPaper Class="pa-4 border-dashed border rounded-lg" Elevation="0">
@@ -68,48 +67,72 @@
</MudText>
}
@*
One panel per cause, the most pressing one first. A folder in which nine
hundred scanned documents were skipped otherwise buries the handful of
files somebody has to look at.
*@
@if (status.Failures.Count > 0)
{
<MudExpansionPanels MultiExpansion="@false" Class="mt-2">
<ExpansionPanel HeaderIcon="@GetFailureListIcon(status)" HeaderText="@this.GetFailureListHeader(status)" MaxHeight="360">
<MudStack Spacing="2">
@foreach (var failure in status.Failures)
{
<MudStack Spacing="1" Class="py-1">
<MudText Typo="Typo.body2" Style="word-break: break-word;">
@failure.FilePath
</MudText>
<MudText Typo="Typo.caption" Color="@(failure.IsPermanent ? Color.Default : Color.Warning)" Style="word-break: break-word;">
@failure.Reason
</MudText>
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="1" AlignItems="AlignItems.Center">
@if (failure.IsPermanent)
{
<MudChip T="string" Size="Size.Small" Color="Color.Default" Variant="Variant.Outlined">@T("Skipped until the file changes")</MudChip>
}
@if (failure.FailureReason is not ProviderRequestFailureReason.NONE)
{
<MudChip T="string" Size="Size.Small" Color="Color.Warning" Variant="Variant.Outlined">@failure.FailureReason.GetName()</MudChip>
}
@if (!string.IsNullOrWhiteSpace(failure.EmbeddingProviderName))
{
<MudText Typo="Typo.caption">@string.Format(T("Embedding provider: {0}"), failure.EmbeddingProviderName)</MudText>
}
@if (failure.OccurredAtUtc > DateTimeOffset.MinValue)
{
<MudText Typo="Typo.caption">@failure.OccurredAtUtc.ToLocalTime().ToString("g")</MudText>
}
@if (failure.FailureReason.IsFixedInProviderSettings())
{
<MudButton Variant="Variant.Text" Size="Size.Small" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Settings" OnClick="@this.OpenEmbeddingProviderSettings">
@T("Open the settings")
</MudButton>
}
</MudStack>
<MudExpansionPanels MultiExpansion="@true" Class="mt-2">
@foreach (var group in this.GetFailureGroups(status))
{
<ExpansionPanel HeaderIcon="@group.Cause.Icon" IconColor="@group.Cause.Color" IconSize="Size.Small" HeaderText="@GetGroupHeader(group)">
<MudStack Spacing="3">
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="2" AlignItems="AlignItems.Center">
<MudChip T="string" Size="Size.Small" Color="@group.Cause.Color" Variant="Variant.Outlined">
@(group.Cause.IsPermanent ? T("Skipped until the file changes") : T("Tried again during the next run"))
</MudChip>
@if (!string.IsNullOrWhiteSpace(group.EmbeddingProviderName))
{
<MudText Typo="Typo.caption">@string.Format(T("Embedding provider: {0}"), group.EmbeddingProviderName)</MudText>
}
@if (group.Cause.NeedsProviderSettings)
{
<MudButton Variant="Variant.Text" Size="Size.Small" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Settings" OnClick="@this.OpenEmbeddingProviderSettings">
@T("Open the settings")
</MudButton>
}
</MudStack>
}
</MudStack>
</ExpansionPanel>
<MudTable Items="@group.Failures" Dense="@true" Hover="@true" Elevation="0" Breakpoint="Breakpoint.None" RowsPerPage="10" Class="border-dashed border rounded-lg">
<ColGroup>
<col/>
<col style="width: 11em;"/>
<col style="width: 7em;"/>
</ColGroup>
<HeaderContent>
<MudTh>@T("File")</MudTh>
<MudTh>@T("Noticed")</MudTh>
<MudTh>@T("Actions")</MudTh>
</HeaderContent>
<RowTemplate>
<MudTd Style="white-space: normal; overflow-wrap: anywhere;">
<MudText Typo="Typo.body2">@GetFileName(context.FilePath)</MudText>
<MudText Typo="Typo.caption" Class="mud-text-secondary">@context.FilePath</MudText>
@if (group.Cause.ShowsMessagePerFile)
{
<MudText Typo="Typo.caption" Color="Color.Warning">@context.Reason</MudText>
}
</MudTd>
<MudTd Style="white-space: nowrap;">
<MudText Typo="Typo.caption">@GetOccurrenceText(context)</MudText>
</MudTd>
<MudTd>
@if (CanShowInFileManager(context))
{
<MudTooltip Text="@T("Show this file in the file browser of your system")">
<MudIconButton Icon="@Icons.Material.Filled.FolderOpen" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.ShowInFileManager(context))"/>
</MudTooltip>
}
</MudTd>
</RowTemplate>
<PagerContent>
<MudTablePager PageSizeOptions="@PAGE_SIZE_OPTIONS"/>
</PagerContent>
</MudTable>
</MudStack>
</ExpansionPanel>
}
</MudExpansionPanels>
}
app/MindWork AI Studio/Pages/Embeddings.razor.cs
+104 -9
View File
@@ -1,5 +1,7 @@
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
@@ -8,12 +10,20 @@ namespace AIStudio.Pages;
public partial class Embeddings : MSGComponentBase
{
private static readonly int[] PAGE_SIZE_OPTIONS = [10, 25, 50, 100];
[Inject]
private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
[Inject]
private NavigationManager NavigationManager { get; init; } = null!;
[Inject]
private RustService RustService { get; init; } = null!;
[Inject]
private ILogger<Embeddings> Logger { get; init; } = null!;
private IReadOnlyList<DataSourceEmbeddingStatus> Statuses { get; set; } = [];
private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles);
@@ -73,19 +83,104 @@ public partial class Embeddings : MSGComponentBase
};
/// <summary>
/// Names the list of failures for what it holds.
/// What a group of failures has in common.
/// </summary>
/// <remarks>
/// A data source whose files were all skipped for good has nothing wrong with it, so calling
/// the list failures would contradict the green state right above it.
/// Also the key the failures are grouped by: two of them belong together exactly when the
/// list would say the same thing about both.
/// </remarks>
private string GetFailureListHeader(DataSourceEmbeddingStatus status) => status.FailedFiles > 0
? string.Format(T("Failure details ({0})"), status.Failures.Count)
: string.Format(T("Skipped files ({0})"), status.Failures.Count);
private sealed record FailureCause(int Priority, string Title, string Icon, Color Color, bool IsPermanent, bool NeedsProviderSettings, bool ShowsMessagePerFile);
private static string GetFailureListIcon(DataSourceEmbeddingStatus status) => status.FailedFiles > 0
? Icons.Material.Filled.ReportProblem
: Icons.Material.Filled.SkipNext;
private sealed record FailureGroup(FailureCause Cause, string EmbeddingProviderName, IReadOnlyList<DataSourceEmbeddingFailure> Failures);
/// <summary>
/// Puts the failures of a data source into one group per cause, the most pressing one first.
/// </summary>
/// <remarks>
/// Within a priority, the largest group comes first: it is the one telling the user the most
/// about their folder.
/// </remarks>
private IReadOnlyList<FailureGroup> GetFailureGroups(DataSourceEmbeddingStatus status) => status.Failures
.GroupBy(this.GetFailureCause)
.Select(group => new FailureGroup(group.Key, GetEmbeddingProviderName(group), group.OrderBy(failure => failure.FilePath, StringComparer.OrdinalIgnoreCase).ToList()))
.OrderBy(group => group.Cause.Priority)
.ThenByDescending(group => group.Failures.Count)
.ThenBy(group => group.Cause.Title, StringComparer.OrdinalIgnoreCase)
.ToList();
private FailureCause GetFailureCause(DataSourceEmbeddingFailure failure)
{
//
// Anything the provider answered comes first: it stops the entire data source, while a
// file nobody can read costs that one file:
//
if (failure.FailureReason is not ProviderRequestFailureReason.NONE)
{
var isFixedInSettings = failure.FailureReason.IsFixedInProviderSettings();
return new FailureCause(isFixedInSettings ? 0 : 1, failure.FailureReason.GetName(), isFixedInSettings ? Icons.Material.Filled.Key : Icons.Material.Filled.CloudOff, Color.Error, false, isFixedInSettings, true);
}
//
// Codes without a name of their own carry everything they know in the message of the
// single file, which is why those groups show that message per file:
//
var causeName = failure.ExtractionCode.GetIndexingCauseName();
var hasCauseName = !string.IsNullOrWhiteSpace(causeName);
var title = hasCauseName ? causeName : T("Other cause");
return failure.IsPermanent
? new FailureCause(3, title, Icons.Material.Filled.SkipNext, Color.Default, true, false, !hasCauseName)
: new FailureCause(2, title, Icons.Material.Filled.ReportProblem, Color.Warning, false, false, !hasCauseName);
}
private static string GetEmbeddingProviderName(IEnumerable<DataSourceEmbeddingFailure> failures) => failures
.Select(failure => failure.EmbeddingProviderName)
.FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)) ?? string.Empty;
private static string GetGroupHeader(FailureGroup group) => $"{group.Cause.Title} ({group.Failures.Count})";
private static string GetFileName(string filePath)
{
var fileName = Path.GetFileName(filePath);
return string.IsNullOrWhiteSpace(fileName) ? filePath : fileName;
}
private static string GetOccurrenceText(DataSourceEmbeddingFailure failure) => failure.OccurredAtUtc > DateTimeOffset.MinValue ? failure.OccurredAtUtc.ToLocalTime().ToString("g") : string.Empty;
/// <remarks>
/// A failure which was not about one file, such as a folder which is gone, carries the name of
/// the data source instead of a path. There is nothing to show for those.
/// </remarks>
private static bool CanShowInFileManager(DataSourceEmbeddingFailure failure) => !string.IsNullOrWhiteSpace(failure.FilePath) && Path.IsPathRooted(failure.FilePath);
/// <summary>
/// Opens the file browser of the system and selects the file in it.
/// </summary>
/// <remarks>
/// Reading that a file could not be indexed is where the work starts, not where it ends: the
/// file has to be opened, replaced, or run through an OCR. This is the same way out the log
/// viewer offers for the log files.
/// </remarks>
private async Task ShowInFileManager(DataSourceEmbeddingFailure failure)
{
OpenPathResponse response;
try
{
response = await this.RustService.TryOpenPathInRuntimeFileManager(failure.FilePath);
}
catch (Exception e)
{
this.Logger.LogWarning(e, "Could not show a file of the embedding failure list in the file manager.");
await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, T("Could not open the file location.")));
return;
}
if (response.Success)
return;
var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue;
await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the file location: {0}"), issue)));
}
private bool CanRefresh(DataSourceEmbeddingStatus status)
{