Show block and page progress while indexing a file

app/MindWork AI Studio/Pages/Embeddings.razor

@@ -68,7 +68,7 @@
                             <MudProgressLinear Value="@status.ProgressPercent" Rounded="@true" Color="@GetStatusColor(status)" />
 
                             <MudText Typo="Typo.body2">
-                                @string.Format(T("{0} of {1} files are indexed."), status.IndexedFiles, status.TotalFiles)
+                                @this.GetFileProgressText(status)
                             </MudText>
 
                             @if (status.PermanentlySkippedFiles > 0)

app/MindWork AI Studio/Pages/Embeddings.razor.cs

@@ -1,3 +1,5 @@
+using System.Globalization;
+
 using AIStudio.Components;
 using AIStudio.Dialogs.Settings;
 using AIStudio.Provider;
@@ -35,6 +37,13 @@ public partial class Embeddings : MSGComponentBase
     private string? expandedDataSourceId;
     private bool userChoseExpansion;
 
+    /// <remarks>
+    /// The language of AI Studio is chosen in its settings and does not move the thread's culture
+    /// along with it. Without this, a German reading a German page would find a file count written
+    /// with English separators.
+    /// </remarks>
+    private CultureInfo currentCulture = CultureInfo.InvariantCulture;
+
     private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles);
 
     private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles - status.PermanentlySkippedFiles));
@@ -69,20 +78,28 @@ public partial class Embeddings : MSGComponentBase
             return;
         }
 
-        this.ApplyFilters([], [ Event.RAG_EMBEDDING_STATUS_CHANGED, Event.CONFIGURATION_CHANGED ]);
+        this.ApplyFilters([], [ Event.RAG_EMBEDDING_STATUS_CHANGED, Event.CONFIGURATION_CHANGED, Event.PLUGINS_RELOADED ]);
+        await this.RefreshCulture();
         await base.OnInitializedAsync();
         this.ReloadStatuses();
     }
 
-    protected override Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
+    protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
     {
-        if (triggeredEvent is Event.RAG_EMBEDDING_STATUS_CHANGED or Event.CONFIGURATION_CHANGED)
+        if (triggeredEvent is Event.CONFIGURATION_CHANGED or Event.PLUGINS_RELOADED)
+            await this.RefreshCulture();
+
+        if (triggeredEvent is Event.RAG_EMBEDDING_STATUS_CHANGED or Event.CONFIGURATION_CHANGED or Event.PLUGINS_RELOADED)
         {
             this.ReloadStatuses();
             this.StateHasChanged();
         }
+    }
 
-        return Task.CompletedTask;
+    private async Task RefreshCulture()
+    {
+        var activeLanguagePlugin = await this.SettingsManager.GetActiveLanguagePlugin();
+        this.currentCulture = CommonTools.DeriveActiveCultureOrInvariant(activeLanguagePlugin.IETFTag);
     }
 
     private void ReloadStatuses()
@@ -155,6 +172,34 @@ public partial class Embeddings : MSGComponentBase
         await dialogReference.Result;
     }
 
+    /// <summary>
+    /// What the panel of a data source says about its progress through the files.
+    /// </summary>
+    /// <remarks>
+    /// While a file is being worked on, the sentence names that file and how far into it we are.
+    /// Counting finished files alone leaves the same sentence standing for hours on a document of
+    /// several thousand pages, and a progress which never moves cannot be told apart from one which
+    /// is stuck. The total number of blocks is not part of it: the blocks are produced while the
+    /// file is read, so nobody knows how many there will be until the file is done.
+    /// </remarks>
+    private string GetFileProgressText(DataSourceEmbeddingStatus status)
+    {
+        if (status.State is not DataSourceEmbeddingState.RUNNING || status.CurrentFileBlock is not { } block)
+            return string.Format(T("{0} of {1} files are indexed."), this.FormatNumber(status.IndexedFiles), this.FormatNumber(status.TotalFiles));
+
+        //
+        // Everything already dealt with, plus the one in hand. Skipped and failed files are part of
+        // that: they are behind us in the folder, and leaving them out would let the number fall
+        // behind the file whose name is shown right next to it.
+        //
+        var currentFileNumber = Math.Min(status.TotalFiles, status.IndexedFiles + status.PermanentlySkippedFiles + status.FailedFiles + 1);
+        return status.CurrentFilePage is { } page
+            ? string.Format(T("File {0} of {1} is being indexed: block {2}, page {3}."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles), this.FormatNumber(block), this.FormatNumber(page))
+            : string.Format(T("File {0} of {1} is being indexed: block {2}."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles), this.FormatNumber(block));
+    }
+
+    private string FormatNumber(int value) => value.ToString("N0", this.currentCulture);
+
     private static Color GetStatusColor(DataSourceEmbeddingStatus status) => status.State switch
     {
         DataSourceEmbeddingState.RUNNING => Color.Warning,

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -18,6 +18,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 {
     private const int VECTOR_STORE_OPTIMIZATION_CHUNK_THRESHOLD = 100_000;
 
+    /// <summary>
+    /// How often the block progress within one file is reported to the user interface at most.
+    /// </summary>
+    private static readonly TimeSpan BLOCK_PROGRESS_INTERVAL = TimeSpan.FromSeconds(3);
+
     private readonly Channel<DataSourceEmbeddingQueueItem> queue = Channel.CreateUnbounded<DataSourceEmbeddingQueueItem>();
     private readonly ConcurrentDictionary<string, byte> queuedIds = new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<string, byte> runningIds = new(StringComparer.OrdinalIgnoreCase);
@@ -646,6 +651,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
             this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles));
 
+            //
+            // What the page says while one file is being worked on. Without it, a document of
+            // several thousand pages leaves the same sentence standing for hours, and a progress
+            // which never moves cannot be told apart from one which is stuck.
+            //
+            var lastBlockReportUtc = DateTimeOffset.MinValue;
+
             try
             {
                 logger.LogInformation(
@@ -658,7 +670,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                     skippedFiles + completedFiles + 1,
                     totalFiles);
                 var startedAtUtc = DateTimeOffset.UtcNow;
-                var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token);
+                var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, ReportBlockProgress, token);
                 token.ThrowIfCancellationRequested();
                 var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
                 if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal))
@@ -780,6 +792,24 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name);
                 this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, failureMessage, failureDetails, permanentlySkippedFiles));
             }
+
+            continue;
+
+            void ReportBlockProgress(int blockNumber, int? pageNumber)
+            {
+                //
+                // The first block goes out at once, so the line is there instead of blank. After
+                // that, at most one message every BLOCK_PROGRESS_INTERVAL: each one re-renders the
+                // embedding page, the navigation bar and the table in the settings, and the blocks
+                // of a large file arrive far faster than anybody can read them.
+                //
+                var nowUtc = DateTimeOffset.UtcNow;
+                if (blockNumber > 1 && nowUtc - lastBlockReportUtc < BLOCK_PROGRESS_INTERVAL)
+                    return;
+
+                lastBlockReportUtc = nowUtc;
+                this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles, blockNumber, pageNumber));
+            }
         }
 
         manifest.SourceHash = metadataSnapshot.SourceHash;
@@ -823,6 +853,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IProvider provider,
         DataSourceEmbeddingManifest manifest,
         VectorStoreOptimizationTracker optimizationTracker,
+        Action<int, int?> reportBlockProgress,
         CancellationToken token)
     {
         var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
@@ -845,6 +876,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         {
             batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
             totalChunkCount++;
+            reportBlockProgress(totalChunkCount, chunk.PageNumber);
 
             if (batch.Count >= embeddingBatchSize)
                 await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
@@ -1443,7 +1475,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         string currentFile = "",
         string lastError = "",
         IReadOnlyList<DataSourceEmbeddingFailure>? failures = null,
-        int permanentlySkippedFiles = 0)
+        int permanentlySkippedFiles = 0,
+        int? currentFileBlock = null,
+        int? currentFilePage = null)
     {
         return new DataSourceEmbeddingStatus(
             dataSource.Id,
@@ -1456,7 +1490,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             currentFile,
             lastError,
             failures?.ToList() ?? [],
-            permanentlySkippedFiles);
+            permanentlySkippedFiles,
+            currentFileBlock,
+            currentFilePage);
     }
 
     /// <remarks>

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs

@@ -3,6 +3,11 @@ using AIStudio.Tools.PluginSystem;
 
 namespace AIStudio.Tools.Services;
 
+/// <remarks>
+/// CurrentFileBlock and CurrentFilePage are null rather than zero while nothing is known about
+/// them: a file which is only about to start has no first block, and not every kind of document
+/// has pages to count. Block numbers start at one, the way the page states them.
+/// </remarks>
 public sealed record DataSourceEmbeddingStatus(
     string DataSourceId,
     string DataSourceName,
@@ -14,7 +19,9 @@ public sealed record DataSourceEmbeddingStatus(
     string CurrentFile,
     string LastError,
     IReadOnlyList<DataSourceEmbeddingFailure> Failures,
-    int PermanentlySkippedFiles = 0)
+    int PermanentlySkippedFiles = 0,
+    int? CurrentFileBlock = null,
+    int? CurrentFilePage = null)
 {
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));