From 59e278086c59f0e93f81ad878690beacc5620bf0 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 9 Sep 2026 11:30:45 +0200 Subject: [PATCH] Show permanently skipped files as their own category --- .../Components/DataSourceManagement.razor | 2 +- .../Components/DataSourceManagement.razor.cs | 23 +++++++++++++++++++ app/MindWork AI Studio/Pages/Embeddings.razor | 18 ++++++++++++--- .../Pages/Embeddings.razor.cs | 19 ++++++++++++++- .../Services/DataSourceEmbeddingFailure.cs | 4 +++- .../Services/DataSourceEmbeddingService.cs | 8 +++---- 6 files changed, 64 insertions(+), 10 deletions(-) diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor b/app/MindWork AI Studio/Components/DataSourceManagement.razor index c8d28ad1..2bf28a55 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor @@ -40,7 +40,7 @@ @if (context is IInternalDataSource) { - + @(embeddingStatus is null ? T("Not available") : string.Format(T("{0} of {1}"), embeddingStatus.IndexedFiles, embeddingStatus.TotalFiles)) diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs index 46042fa5..0a121c80 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs @@ -68,6 +68,10 @@ public partial class DataSourceManagement : MSGComponentBase this.availableEmbeddingProviders.Add(new (provider.Name, provider.Id)); } + /// + /// Files which were skipped for good are none of the failed ones, so a data source made of + /// scanned documents stays green: there is nothing here for the user to fix. + /// private static Color GetIndexingStatusColor(DataSourceEmbeddingStatus? status) { if (status is null || status.State is DataSourceEmbeddingState.IDLE or DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.RUNNING) @@ -78,6 +82,25 @@ public partial class DataSourceManagement : MSGComponentBase : Color.Success; } + /// + /// Explains the indexing dot, including the files which stay out of the index. + /// + /// + /// The column shows the indexed files against the total, which reads as unfinished for a data + /// source whose remaining files were skipped for good. The tooltip is where that gap gets its + /// explanation. + /// + private string GetIndexingStatusTooltip(DataSourceEmbeddingStatus? status) + { + if (status is null) + return T("Waiting for indexing status"); + + if (status.PermanentlySkippedFiles == 0) + return status.StateLabel; + + return $"{status.StateLabel} — {string.Format(T("{0} files were skipped because they contain no readable text. AI Studio reads them again once they change."), status.PermanentlySkippedFiles)}"; + } + private string GetEmbeddingName(IDataSource dataSource) { if(dataSource is IInternalDataSource internalDataSource) diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor index 73cbed5e..65514e42 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor +++ b/app/MindWork AI Studio/Pages/Embeddings.razor @@ -6,11 +6,12 @@ @T("Background embeddings") - @T("AI Studio indexes local RAG data sources in the background. Finished files stay recorded so unchanged files can be skipped after a restart, while added or deleted files are detected during the next run.") + @T("AI Studio indexes local RAG data sources in the background. Finished files stay recorded so unchanged files can be skipped after a restart, while added or deleted files are detected during the next run. The same applies to documents without readable text, such as scanned pages: AI Studio remembers them and reads them again only once they change.") @string.Format(T("Indexed files: {0}"), this.TotalIndexedFiles) @string.Format(T("Pending files: {0}"), this.TotalPendingFiles) + @string.Format(T("Skipped files: {0}"), this.TotalPermanentlySkippedFiles) @string.Format(T("Failed files: {0}"), this.TotalFailedFiles) @@ -46,6 +47,13 @@ @string.Format(T("{0} of {1} files are indexed."), status.IndexedFiles, status.TotalFiles) + @if (status.PermanentlySkippedFiles > 0) + { + + @string.Format(T("Skipped files: {0}. AI Studio reads them again once they change."), status.PermanentlySkippedFiles) + + } + @if (status.FailedFiles > 0) { @@ -63,7 +71,7 @@ @if (status.Failures.Count > 0) { - + @foreach (var failure in status.Failures) { @@ -71,10 +79,14 @@ @failure.FilePath - + @failure.Reason + @if (failure.IsPermanent) + { + @T("Skipped until the file changes") + } @if (failure.FailureReason is not ProviderRequestFailureReason.NONE) { @failure.FailureReason.GetName() diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor.cs b/app/MindWork AI Studio/Pages/Embeddings.razor.cs index 356765e9..0416a2f3 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor.cs +++ b/app/MindWork AI Studio/Pages/Embeddings.razor.cs @@ -18,10 +18,12 @@ public partial class Embeddings : MSGComponentBase private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles); - private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles)); + private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles - status.PermanentlySkippedFiles)); private int TotalFailedFiles => this.Statuses.Sum(status => status.FailedFiles); + private int TotalPermanentlySkippedFiles => this.Statuses.Sum(status => status.PermanentlySkippedFiles); + protected override async Task OnInitializedAsync() { // @@ -70,6 +72,21 @@ public partial class Embeddings : MSGComponentBase _ => Color.Default, }; + /// + /// Names the list of failures for what it holds. + /// + /// + /// A data source whose files were all skipped for good has nothing wrong with it, so calling + /// the list failures would contradict the green state right above it. + /// + private string GetFailureListHeader(DataSourceEmbeddingStatus status) => status.FailedFiles > 0 + ? string.Format(T("Failure details ({0})"), status.Failures.Count) + : string.Format(T("Skipped files ({0})"), status.Failures.Count); + + private static string GetFailureListIcon(DataSourceEmbeddingStatus status) => status.FailedFiles > 0 + ? Icons.Material.Filled.ReportProblem + : Icons.Material.Filled.SkipNext; + private bool CanRefresh(DataSourceEmbeddingStatus status) { return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) && diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs index 8d77fad7..31f6cd7b 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs @@ -18,5 +18,7 @@ namespace AIStudio.Tools.Services; /// What kind of failure it was. Everything that did not come from a provider stays at NONE. /// What the provider answered, where it answered at all. /// The embedding provider that was asked. +/// Why reading the file failed, where the failure was about reading it at all. +/// Whether the file stays out of the index until it changes. public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason, DateTimeOffset OccurredAtUtc, ProviderRequestFailureReason FailureReason = ProviderRequestFailureReason.NONE, - HttpStatusCode? StatusCode = null, string EmbeddingProviderName = ""); \ No newline at end of file + HttpStatusCode? StatusCode = null, string EmbeddingProviderName = "", FileExtractionErrorCode ExtractionCode = FileExtractionErrorCode.NONE, bool IsPermanent = false); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index e846965a..6cac7238 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -640,7 +640,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM permanentlySkippedFiles++; // The stored reason keeps its place in the list, so the user still sees why: - failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, permanentFailure.Message, permanentFailure.OccurredAtUtc)); + failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, permanentFailure.Message, permanentFailure.OccurredAtUtc, ExtractionCode: permanentFailure.Code, IsPermanent: true)); this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError, failures: failureDetails, permanentlySkippedFiles: permanentlySkippedFiles)); continue; } @@ -735,7 +735,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM permanentlySkippedFiles++; var occurredAtUtc = DateTimeOffset.UtcNow; var indexingMessage = exception.Code.ToIndexingUserMessage(file.Name); - failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, indexingMessage, occurredAtUtc)); + failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, indexingMessage, occurredAtUtc, ExtractionCode: exception.Code, IsPermanent: true)); manifest.Files.Remove(file.FullName); await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token); @@ -766,7 +766,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM // failedFiles++; lastError = exception.Message; - failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name)); + failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name, ExtractionCode: exception is FileExtractionException extractionFailure ? extractionFailure.Code : FileExtractionErrorCode.NONE)); manifest.Files.Remove(file.FullName); await this.ForgetPermanentFailureAsync(indexStore, dataSource, manifest, file.FullName, token); await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token); @@ -1376,7 +1376,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } private static List CreatePermanentFailureDetails(DataSourceEmbeddingManifest manifest) => manifest.PermanentFailures - .Select(failure => new DataSourceEmbeddingFailure(failure.Key, failure.Value.Message, failure.Value.OccurredAtUtc)) + .Select(failure => new DataSourceEmbeddingFailure(failure.Key, failure.Value.Message, failure.Value.OccurredAtUtc, ExtractionCode: failure.Value.Code, IsPermanent: true)) .ToList(); private static string GetFileEmbeddingReason(FileInfo file, string currentHash, EmbeddedFileRecord? existingRecord)