MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 02:53:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Show permanently skipped files as their own category

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-09 11:30:45 +02:00
parent 2eb619fb04
commit 59e278086c
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
6 changed files with 64 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/MindWork AI Studio/Components/DataSourceManagement.razor
View File

@ -40,7 +40,7 @@
<MudTd>
@if (context is IInternalDataSource)
{
<MudTooltip Text="@(embeddingStatus?.StateLabel ?? T("Waiting for indexing status"))">
<MudTooltip Text="@this.GetIndexingStatusTooltip(embeddingStatus)">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1">
<MudIcon Icon="@Icons.Material.Filled.Circle" Color="@GetIndexingStatusColor(embeddingStatus)" Size="Size.Small" Style="width: 0.75rem; height: 0.75rem; flex-shrink: 0;"/>
<MudText Typo="Typo.body2">@(embeddingStatus is null ? T("Not available") : string.Format(T("{0} of {1}"), embeddingStatus.IndexedFiles, embeddingStatus.TotalFiles))</MudText>

23
app/MindWork AI Studio/Components/DataSourceManagement.razor.cs
View File

@ -68,6 +68,10 @@ public partial class DataSourceManagement : MSGComponentBase
this.availableEmbeddingProviders.Add(new (provider.Name, provider.Id));
}
/// <remarks>
/// Files which were skipped for good are none of the failed ones, so a data source made of
/// scanned documents stays green: there is nothing here for the user to fix.
/// </remarks>
private static Color GetIndexingStatusColor(DataSourceEmbeddingStatus? status)
{
if (status is null || status.State is DataSourceEmbeddingState.IDLE or DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.RUNNING)
@ -78,6 +82,25 @@ public partial class DataSourceManagement : MSGComponentBase
: Color.Success;
}
/// <summary>
/// Explains the indexing dot, including the files which stay out of the index.
/// </summary>
/// <remarks>
/// The column shows the indexed files against the total, which reads as unfinished for a data
/// source whose remaining files were skipped for good. The tooltip is where that gap gets its
/// explanation.
/// </remarks>
private string GetIndexingStatusTooltip(DataSourceEmbeddingStatus? status)
{
if (status is null)
return T("Waiting for indexing status");
if (status.PermanentlySkippedFiles == 0)
return status.StateLabel;
return $"{status.StateLabel} — {string.Format(T("{0} files were skipped because they contain no readable text. AI Studio reads them again once they change."), status.PermanentlySkippedFiles)}";
}
private string GetEmbeddingName(IDataSource dataSource)
{
if(dataSource is IInternalDataSource internalDataSource)

18
app/MindWork AI Studio/Pages/Embeddings.razor
View File

@ -6,11 +6,12 @@
<MudPaper Class="pa-4 border-dashed border rounded-lg" Elevation="0">
<MudText Typo="Typo.h5">@T("Background embeddings")</MudText>
<MudText Typo="Typo.body1" Class="mt-2">
@T("AI Studio indexes local RAG data sources in the background. Finished files stay recorded so unchanged files can be skipped after a restart, while added or deleted files are detected during the next run.")
@T("AI Studio indexes local RAG data sources in the background. Finished files stay recorded so unchanged files can be skipped after a restart, while added or deleted files are detected during the next run. The same applies to documents without readable text, such as scanned pages: AI Studio remembers them and reads them again only once they change.")
</MudText>
<MudStack Row="true" Class="mt-3" Wrap="Wrap.Wrap" Spacing="2">
<MudChip T="string" Color="Color.Success" Variant="Variant.Filled">@string.Format(T("Indexed files: {0}"), this.TotalIndexedFiles)</MudChip>
<MudChip T="string" Color="Color.Info" Variant="Variant.Filled">@string.Format(T("Pending files: {0}"), this.TotalPendingFiles)</MudChip>
<MudChip T="string" Color="Color.Default" Variant="Variant.Filled">@string.Format(T("Skipped files: {0}"), this.TotalPermanentlySkippedFiles)</MudChip>
<MudChip T="string" Color="Color.Error" Variant="Variant.Filled">@string.Format(T("Failed files: {0}"), this.TotalFailedFiles)</MudChip>
</MudStack>
</MudPaper>
@ -46,6 +47,13 @@
@string.Format(T("{0} of {1} files are indexed."), status.IndexedFiles, status.TotalFiles)
</MudText>
@if (status.PermanentlySkippedFiles > 0)
{
<MudText Typo="Typo.body2">
@string.Format(T("Skipped files: {0}. AI Studio reads them again once they change."), status.PermanentlySkippedFiles)
</MudText>
}
@if (status.FailedFiles > 0)
{
<MudText Typo="Typo.body2">
@ -63,7 +71,7 @@
@if (status.Failures.Count > 0)
{
<MudExpansionPanels MultiExpansion="@false" Class="mt-2">
<ExpansionPanel HeaderIcon="@Icons.Material.Filled.ReportProblem" HeaderText="@string.Format(T("Failure details ({0})"), status.Failures.Count)" MaxHeight="360">
<ExpansionPanel HeaderIcon="@GetFailureListIcon(status)" HeaderText="@this.GetFailureListHeader(status)" MaxHeight="360">
<MudStack Spacing="2">
@foreach (var failure in status.Failures)
{
@ -71,10 +79,14 @@
<MudText Typo="Typo.body2" Style="word-break: break-word;">
@failure.FilePath
</MudText>
<MudText Typo="Typo.caption" Color="Color.Warning" Style="word-break: break-word;">
<MudText Typo="Typo.caption" Color="@(failure.IsPermanent ? Color.Default : Color.Warning)" Style="word-break: break-word;">
@failure.Reason
</MudText>
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="1" AlignItems="AlignItems.Center">
@if (failure.IsPermanent)
{
<MudChip T="string" Size="Size.Small" Color="Color.Default" Variant="Variant.Outlined">@T("Skipped until the file changes")</MudChip>
}
@if (failure.FailureReason is not ProviderRequestFailureReason.NONE)
{
<MudChip T="string" Size="Size.Small" Color="Color.Warning" Variant="Variant.Outlined">@failure.FailureReason.GetName()</MudChip>

19
app/MindWork AI Studio/Pages/Embeddings.razor.cs
View File

@ -18,10 +18,12 @@ public partial class Embeddings : MSGComponentBase
private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles);
private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles));
private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles - status.PermanentlySkippedFiles));
private int TotalFailedFiles => this.Statuses.Sum(status => status.FailedFiles);
private int TotalPermanentlySkippedFiles => this.Statuses.Sum(status => status.PermanentlySkippedFiles);
protected override async Task OnInitializedAsync()
{
//
@ -70,6 +72,21 @@ public partial class Embeddings : MSGComponentBase
_ => Color.Default,
};
/// <summary>
/// Names the list of failures for what it holds.
/// </summary>
/// <remarks>
/// A data source whose files were all skipped for good has nothing wrong with it, so calling
/// the list failures would contradict the green state right above it.
/// </remarks>
private string GetFailureListHeader(DataSourceEmbeddingStatus status) => status.FailedFiles > 0
? string.Format(T("Failure details ({0})"), status.Failures.Count)
: string.Format(T("Skipped files ({0})"), status.Failures.Count);
private static string GetFailureListIcon(DataSourceEmbeddingStatus status) => status.FailedFiles > 0
? Icons.Material.Filled.ReportProblem
: Icons.Material.Filled.SkipNext;
private bool CanRefresh(DataSourceEmbeddingStatus status)
{
return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&

4
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs
View File

@ -18,5 +18,7 @@ namespace AIStudio.Tools.Services;
/// <param name="FailureReason">What kind of failure it was. Everything that did not come from a provider stays at NONE.</param>
/// <param name="StatusCode">What the provider answered, where it answered at all.</param>
/// <param name="EmbeddingProviderName">The embedding provider that was asked.</param>
/// <param name="ExtractionCode">Why reading the file failed, where the failure was about reading it at all.</param>
/// <param name="IsPermanent">Whether the file stays out of the index until it changes.</param>
public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason, DateTimeOffset OccurredAtUtc, ProviderRequestFailureReason FailureReason = ProviderRequestFailureReason.NONE,
HttpStatusCode? StatusCode = null, string EmbeddingProviderName = "");
HttpStatusCode? StatusCode = null, string EmbeddingProviderName = "", FileExtractionErrorCode ExtractionCode = FileExtractionErrorCode.NONE, bool IsPermanent = false);

8
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -640,7 +640,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
permanentlySkippedFiles++;
// The stored reason keeps its place in the list, so the user still sees why:
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, permanentFailure.Message, permanentFailure.OccurredAtUtc));
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, permanentFailure.Message, permanentFailure.OccurredAtUtc, ExtractionCode: permanentFailure.Code, IsPermanent: true));
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError, failures: failureDetails, permanentlySkippedFiles: permanentlySkippedFiles));
continue;
}
@ -735,7 +735,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
permanentlySkippedFiles++;
var occurredAtUtc = DateTimeOffset.UtcNow;
var indexingMessage = exception.Code.ToIndexingUserMessage(file.Name);
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, indexingMessage, occurredAtUtc));
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, indexingMessage, occurredAtUtc, ExtractionCode: exception.Code, IsPermanent: true));
manifest.Files.Remove(file.FullName);
await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
@ -766,7 +766,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
//
failedFiles++;
lastError = exception.Message;
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name));
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name, ExtractionCode: exception is FileExtractionException extractionFailure ? extractionFailure.Code : FileExtractionErrorCode.NONE));
manifest.Files.Remove(file.FullName);
await this.ForgetPermanentFailureAsync(indexStore, dataSource, manifest, file.FullName, token);
await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
@ -1376,7 +1376,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
}
private static List<DataSourceEmbeddingFailure> CreatePermanentFailureDetails(DataSourceEmbeddingManifest manifest) => manifest.PermanentFailures
.Select(failure => new DataSourceEmbeddingFailure(failure.Key, failure.Value.Message, failure.Value.OccurredAtUtc))
.Select(failure => new DataSourceEmbeddingFailure(failure.Key, failure.Value.Message, failure.Value.OccurredAtUtc, ExtractionCode: failure.Value.Code, IsPermanent: true))
.ToList();
private static string GetFileEmbeddingReason(FileInfo file, string currentHash, EmbeddedFileRecord? existingRecord)