diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor index 033a6867..f1c0907f 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor +++ b/app/MindWork AI Studio/Pages/Embeddings.razor @@ -68,7 +68,7 @@ - @string.Format(T("{0} of {1} files are indexed."), status.IndexedFiles, status.TotalFiles) + @this.GetFileProgressText(status) @if (status.PermanentlySkippedFiles > 0) diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor.cs b/app/MindWork AI Studio/Pages/Embeddings.razor.cs index 51f9d540..76cd2e5f 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor.cs +++ b/app/MindWork AI Studio/Pages/Embeddings.razor.cs @@ -1,3 +1,5 @@ +using System.Globalization; + using AIStudio.Components; using AIStudio.Dialogs.Settings; using AIStudio.Provider; @@ -35,6 +37,13 @@ public partial class Embeddings : MSGComponentBase private string? expandedDataSourceId; private bool userChoseExpansion; + /// + /// The language of AI Studio is chosen in its settings and does not move the thread's culture + /// along with it. Without this, a German reading a German page would find a file count written + /// with English separators. + /// + private CultureInfo currentCulture = CultureInfo.InvariantCulture; + private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles); private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles - status.PermanentlySkippedFiles)); @@ -69,20 +78,28 @@ public partial class Embeddings : MSGComponentBase return; } - this.ApplyFilters([], [ Event.RAG_EMBEDDING_STATUS_CHANGED, Event.CONFIGURATION_CHANGED ]); + this.ApplyFilters([], [ Event.RAG_EMBEDDING_STATUS_CHANGED, Event.CONFIGURATION_CHANGED, Event.PLUGINS_RELOADED ]); + await this.RefreshCulture(); await base.OnInitializedAsync(); this.ReloadStatuses(); } - protected override Task ProcessIncomingMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default + protected override async Task ProcessIncomingMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default { - if (triggeredEvent is Event.RAG_EMBEDDING_STATUS_CHANGED or Event.CONFIGURATION_CHANGED) + if (triggeredEvent is Event.CONFIGURATION_CHANGED or Event.PLUGINS_RELOADED) + await this.RefreshCulture(); + + if (triggeredEvent is Event.RAG_EMBEDDING_STATUS_CHANGED or Event.CONFIGURATION_CHANGED or Event.PLUGINS_RELOADED) { this.ReloadStatuses(); this.StateHasChanged(); } + } - return Task.CompletedTask; + private async Task RefreshCulture() + { + var activeLanguagePlugin = await this.SettingsManager.GetActiveLanguagePlugin(); + this.currentCulture = CommonTools.DeriveActiveCultureOrInvariant(activeLanguagePlugin.IETFTag); } private void ReloadStatuses() @@ -155,6 +172,34 @@ public partial class Embeddings : MSGComponentBase await dialogReference.Result; } + /// + /// What the panel of a data source says about its progress through the files. + /// + /// + /// While a file is being worked on, the sentence names that file and how far into it we are. + /// Counting finished files alone leaves the same sentence standing for hours on a document of + /// several thousand pages, and a progress which never moves cannot be told apart from one which + /// is stuck. The total number of blocks is not part of it: the blocks are produced while the + /// file is read, so nobody knows how many there will be until the file is done. + /// + private string GetFileProgressText(DataSourceEmbeddingStatus status) + { + if (status.State is not DataSourceEmbeddingState.RUNNING || status.CurrentFileBlock is not { } block) + return string.Format(T("{0} of {1} files are indexed."), this.FormatNumber(status.IndexedFiles), this.FormatNumber(status.TotalFiles)); + + // + // Everything already dealt with, plus the one in hand. Skipped and failed files are part of + // that: they are behind us in the folder, and leaving them out would let the number fall + // behind the file whose name is shown right next to it. + // + var currentFileNumber = Math.Min(status.TotalFiles, status.IndexedFiles + status.PermanentlySkippedFiles + status.FailedFiles + 1); + return status.CurrentFilePage is { } page + ? string.Format(T("File {0} of {1} is being indexed: block {2}, page {3}."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles), this.FormatNumber(block), this.FormatNumber(page)) + : string.Format(T("File {0} of {1} is being indexed: block {2}."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles), this.FormatNumber(block)); + } + + private string FormatNumber(int value) => value.ToString("N0", this.currentCulture); + private static Color GetStatusColor(DataSourceEmbeddingStatus status) => status.State switch { DataSourceEmbeddingState.RUNNING => Color.Warning, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index c8646a2d..553663cb 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -18,6 +18,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM { private const int VECTOR_STORE_OPTIMIZATION_CHUNK_THRESHOLD = 100_000; + /// + /// How often the block progress within one file is reported to the user interface at most. + /// + private static readonly TimeSpan BLOCK_PROGRESS_INTERVAL = TimeSpan.FromSeconds(3); + private readonly Channel queue = Channel.CreateUnbounded(); private readonly ConcurrentDictionary queuedIds = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary runningIds = new(StringComparer.OrdinalIgnoreCase); @@ -646,6 +651,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles)); + // + // What the page says while one file is being worked on. Without it, a document of + // several thousand pages leaves the same sentence standing for hours, and a progress + // which never moves cannot be told apart from one which is stuck. + // + var lastBlockReportUtc = DateTimeOffset.MinValue; + try { logger.LogInformation( @@ -658,7 +670,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM skippedFiles + completedFiles + 1, totalFiles); var startedAtUtc = DateTimeOffset.UtcNow; - var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token); + var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, ReportBlockProgress, token); token.ThrowIfCancellationRequested(); var fingerprintAfterEmbedding = BuildFileMetadataHash(file); if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal)) @@ -780,6 +792,24 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name); this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, failureMessage, failureDetails, permanentlySkippedFiles)); } + + continue; + + void ReportBlockProgress(int blockNumber, int? pageNumber) + { + // + // The first block goes out at once, so the line is there instead of blank. After + // that, at most one message every BLOCK_PROGRESS_INTERVAL: each one re-renders the + // embedding page, the navigation bar and the table in the settings, and the blocks + // of a large file arrive far faster than anybody can read them. + // + var nowUtc = DateTimeOffset.UtcNow; + if (blockNumber > 1 && nowUtc - lastBlockReportUtc < BLOCK_PROGRESS_INTERVAL) + return; + + lastBlockReportUtc = nowUtc; + this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles, blockNumber, pageNumber)); + } } manifest.SourceHash = metadataSnapshot.SourceHash; @@ -823,6 +853,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM IProvider provider, DataSourceEmbeddingManifest manifest, VectorStoreOptimizationTracker optimizationTracker, + Action reportBlockProgress, CancellationToken token) { var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id); @@ -845,6 +876,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM { batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber)); totalChunkCount++; + reportBlockProgress(totalChunkCount, chunk.PageNumber); if (batch.Count >= embeddingBatchSize) await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token); @@ -1443,7 +1475,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM string currentFile = "", string lastError = "", IReadOnlyList? failures = null, - int permanentlySkippedFiles = 0) + int permanentlySkippedFiles = 0, + int? currentFileBlock = null, + int? currentFilePage = null) { return new DataSourceEmbeddingStatus( dataSource.Id, @@ -1456,7 +1490,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM currentFile, lastError, failures?.ToList() ?? [], - permanentlySkippedFiles); + permanentlySkippedFiles, + currentFileBlock, + currentFilePage); } /// diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs index 2d878587..4627f716 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs @@ -3,6 +3,11 @@ using AIStudio.Tools.PluginSystem; namespace AIStudio.Tools.Services; +/// +/// CurrentFileBlock and CurrentFilePage are null rather than zero while nothing is known about +/// them: a file which is only about to start has no first block, and not every kind of document +/// has pages to count. Block numbers start at one, the way the page states them. +/// public sealed record DataSourceEmbeddingStatus( string DataSourceId, string DataSourceName, @@ -14,7 +19,9 @@ public sealed record DataSourceEmbeddingStatus( string CurrentFile, string LastError, IReadOnlyList Failures, - int PermanentlySkippedFiles = 0) + int PermanentlySkippedFiles = 0, + int? CurrentFileBlock = null, + int? CurrentFilePage = null) { private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));