Indexed large data sources in steps so the others get their turn (#1052)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs

@@ -32,7 +32,7 @@ public sealed partial class DataSourceEmbeddingService
             dataSource.Id);
 
         var activeRun = this.CancelActiveDataSourceRun(dataSource);
-        this.ClearQueuedDataSourceState(dataSourceId);
+        this.runQueue.Clear(dataSourceId);
         if (activeRun is not null)
             await activeRun.Completion.Task;
 
@@ -137,6 +137,8 @@ public sealed partial class DataSourceEmbeddingService
         IndexStoreClient? indexStore,
         CancellationToken token)
     {
+        // What failed while building the index which is thrown away now says nothing about the next one:
+        this.passes.TryRemove(dataSourceId, out _);
         await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token);
 
         indexStore ??= await databaseClientProvider.GetIndexStoreAsync(token);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -1,6 +1,5 @@
 using System.Collections.Concurrent;
 using System.Diagnostics.CodeAnalysis;
-using System.Threading.Channels;
 
 using AIStudio.Provider;
 using AIStudio.Settings;
@@ -33,13 +32,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     /// </summary>
     private readonly IReadOnlyList<IIndexedSourceIndexer> indexers = CreateIndexers(settingsManager, rustService, guardService, logger);
 
-    private readonly Channel<DataSourceEmbeddingQueueItem> queue = Channel.CreateUnbounded<DataSourceEmbeddingQueueItem>();
-    private readonly ConcurrentDictionary<string, byte> queuedIds = new(StringComparer.OrdinalIgnoreCase);
-    private readonly ConcurrentDictionary<string, byte> runningIds = new(StringComparer.OrdinalIgnoreCase);
-    private readonly ConcurrentDictionary<string, DataSourceEmbeddingRefreshMode> pendingRefreshModes = new(StringComparer.OrdinalIgnoreCase);
+    private readonly DataSourceRunQueue runQueue = new();
+
+    /// <summary>
+    /// Lets one run at a time embed, cf. ProcessDataSourceRunAsync.
+    /// </summary>
+    private readonly SemaphoreSlim runGate = new(1, 1);
+
     private readonly ConcurrentDictionary<string, DataSourceRunControl> activeRuns = new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<string, DataSourceEmbeddingStatus> statuses = new(StringComparer.OrdinalIgnoreCase);
-    private readonly object queueStateLock = new();
+
+    /// <summary>
+    /// The pass of every data source whose last step left more to do, and of none else, cf. IndexingPass.
+    /// </summary>
+    private readonly ConcurrentDictionary<string, IndexingPass> passes = new(StringComparer.OrdinalIgnoreCase);
 
     /// <summary>
     /// Guards the run signature of every active run, cf. DataSourceRunControl.RunSignature.
@@ -51,16 +57,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
 
-    private enum DataSourceQueueRequestResult
-    {
-        QUEUED,
-        ALREADY_QUEUED,
-        RUNNING,
-        RUNNING_MARKED_PENDING,
-    }
-
-    private sealed record DataSourceEmbeddingQueueItem(string DataSourceId, DataSourceEmbeddingRefreshMode RefreshMode);
-
     private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion)
     {
         /// <summary>
@@ -458,7 +454,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         this.RefreshWatchers();
         logger.LogDebug("Refreshed watcher state for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
 
-        var queueRequestResult = this.TryReserveDataSourceQueueSlot(dataSource.Id, queueAfterCurrentRun, refreshMode);
+        var queueRequestResult = this.runQueue.TryReserve(dataSource.Id, queueAfterCurrentRun, refreshMode);
 
         //
         // Every change of the embedding settings ends up here, whichever dialog made it. A run
@@ -471,11 +467,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         switch (queueRequestResult)
         {
             case DataSourceQueueRequestResult.ALREADY_QUEUED:
-                logger.LogDebug("Data source '{DataSourceName}' ({DataSourceId}) is already queued for background embeddings. Ignoring duplicate queue request.", dataSource.Name, dataSource.Id);
+                logger.LogDebug("Data source '{DataSourceName}' ({DataSourceId}) is already queued for background embeddings. Merged the queue request into the queued run. RefreshMode={RefreshMode}.", dataSource.Name, dataSource.Id, refreshMode);
                 return;
 
             case DataSourceQueueRequestResult.RUNNING:
-                logger.LogDebug("Data source '{DataSourceName}' ({DataSourceId}) is already being embedded. Ignoring duplicate queue request.", dataSource.Name, dataSource.Id);
+                logger.LogDebug("Data source '{DataSourceName}' ({DataSourceId}) is already being embedded. The queue request adds no further follow-up embedding run. RefreshMode={RefreshMode}.", dataSource.Name, dataSource.Id, refreshMode);
                 return;
 
             case DataSourceQueueRequestResult.RUNNING_MARKED_PENDING:
@@ -483,24 +479,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 return;
         }
 
+        var lane = this.GetQueueLane(dataSource.Id);
         logger.LogInformation(
-            "Queueing data source '{DataSourceName}' ({DataSourceId}) for background embedding hash check. RefreshMode={RefreshMode}.",
+            "Queueing data source '{DataSourceName}' ({DataSourceId}) for background embedding hash check. RefreshMode={RefreshMode}, Lane={Lane}.",
             dataSource.Name,
             dataSource.Id,
-            refreshMode);
+            refreshMode,
+            lane);
         if (!this.statuses.TryGetValue(dataSource.Id, out var currentStatus) || currentStatus.State is not DataSourceEmbeddingState.RUNNING)
-        {
-            this.UpsertStatus(this.CreateStatus(
-                dataSource,
-                DataSourceEmbeddingState.QUEUED,
-                currentStatus?.TotalDocuments ?? 0,
-                currentStatus?.IndexedDocuments ?? 0,
-                currentStatus?.FailedDocuments ?? 0,
-                failures: currentStatus?.Failures ?? [],
-                lastSyncUtc: currentStatus?.LastSyncUtc));
-        }
+            this.UpsertStatus(this.GetQueuedStatus(dataSource));
+
         logger.LogDebug("Upserting status for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
-        await this.queue.Writer.WriteAsync(new DataSourceEmbeddingQueueItem(dataSource.Id, refreshMode));
+        await this.runQueue.EnqueueAsync(dataSource.Id, lane);
         logger.LogDebug("Queued data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
     }
 
@@ -511,7 +501,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         this.RemoveWatcher(dataSource.Id);
         var activeRun = this.CancelActiveDataSourceRun(dataSource);
-        this.ClearQueuedDataSourceState(dataSource.Id);
+        this.runQueue.Clear(dataSource.Id);
         this.statuses.TryRemove(dataSource.Id, out _);
         if (activeRun is not null)
         {
@@ -538,54 +528,81 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         while (!stoppingToken.IsCancellationRequested)
         {
-            var queueItem = await this.queue.Reader.ReadAsync(stoppingToken);
-            var dataSourceId = queueItem.DataSourceId;
-            this.MarkDataSourceRunStarted(dataSourceId);
-
-            IIndexedDataSource? dataSource = null;
+            var runRequest = await this.runQueue.DequeueAsync(stoppingToken);
 
             try
             {
-                if (!this.TryGetConfiguredIndexedSource(dataSourceId, out dataSource))
-                    continue;
-
-                await this.ProcessDataSourceRunAsync(dataSource, queueItem.RefreshMode, stoppingToken);
+                await this.RunDataSourceAsync(runRequest.DataSourceId, runRequest.RefreshMode, stoppingToken);
             }
             catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
             {
                 break;
             }
-            catch (VectorStoreUnreadableException exception) when (dataSource is not null)
+        }
+    }
+
+    /// <summary>
+    /// Runs a data source which the queue marked as running, and queues its follow-up run afterward.
+    /// </summary>
+    /// <remarks>
+    /// The one bracket around every run, whether it came out of the queue or from the startup check.
+    /// Whoever calls this has marked the run as running, cf. DataSourceRunQueue.DequeueAsync and
+    /// TryBeginRun. That is what turns every request during the run into exactly one follow-up run,
+    /// the next step of a data source which did its share of the work among them. It also shows the
+    /// data source as queued again once the run ends, instead of leaving it on running.
+    /// </remarks>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    /// <param name="refreshMode">Why the run was asked for.</param>
+    /// <param name="token">The cancellation token.</param>
+    private async Task RunDataSourceAsync(string dataSourceId, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token)
+    {
+        IIndexedDataSource? dataSource = null;
+
+        try
+        {
+            if (!this.TryGetConfiguredIndexedSource(dataSourceId, out dataSource))
             {
-                //
-                // Nothing is deleted and nothing is rebuilt here. The data source says what is
-                // wrong with it, stays out of the selection while it says so, and waits for the
-                // user to ask for the repair.
-                //
-                logger.LogError(
-                    exception,
-                    "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.",
-                    dataSource.Name,
-                    dataSource.Id);
-                this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource));
+                // Deleted, or no longer indexed since a preview was switched off. Its pass would lead nowhere:
+                this.passes.TryRemove(dataSourceId, out _);
+                return;
             }
-            catch (Exception exception)
+
+            await this.ProcessDataSourceRunAsync(dataSource, refreshMode, token);
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (VectorStoreUnreadableException exception) when (dataSource is not null)
+        {
+            //
+            // Nothing is deleted and nothing is rebuilt here. The data source says what is
+            // wrong with it, stays out of the selection while it says so, and waits for the
+            // user to ask for the repair.
+            //
+            logger.LogError(
+                exception,
+                "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.",
+                dataSource.Name,
+                dataSource.Id);
+            this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource));
+        }
+        catch (Exception exception)
+        {
+            if (dataSource is null)
             {
-                if (dataSource is null)
-                {
-                    logger.LogError(exception, "Background embedding failed for data source '{DataSourceId}'.", dataSourceId);
-                }
-                else
-                {
-                    logger.LogError(exception, "Background embedding failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
-                    this.UpsertStatus(this.GetFallbackStatus(dataSource, string.Format(TB("The data source '{0}' could not be processed. The log file holds the details."), dataSource.Name)));
-                }
+                logger.LogError(exception, "Background embedding failed for data source '{DataSourceId}'. RefreshMode={RefreshMode}.", dataSourceId, refreshMode);
             }
-            finally
+            else
             {
-                await this.QueuePendingDataSourceRunAsync(dataSourceId, stoppingToken);
+                logger.LogError(exception, "Background embedding failed for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}.", dataSource.Name, dataSource.Id, refreshMode);
+                this.UpsertStatus(this.GetFallbackStatus(dataSource, string.Format(TB("The data source '{0}' could not be processed. The log file holds the details."), dataSource.Name)));
             }
         }
+        finally
+        {
+            await this.QueuePendingDataSourceRunAsync(dataSourceId, token);
+        }
     }
 
     public override async Task StopAsync(CancellationToken cancellationToken)
@@ -598,6 +615,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     {
         this.DisposeWatchers();
         base.Dispose();
+        this.runGate.Dispose();
     }
 
     private async Task ProcessDataSourceRunAsync(IDataSourceBase requestedDataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken parentToken)
@@ -630,7 +648,21 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         try
         {
-            await this.ProcessDataSourceAsync(dataSource, refreshMode, runControl, runTokenSource.Token);
+            //
+            // One run at a time, whichever way it started: the startup check goes on next to the
+            // queue once the automatic refresh is switched on while the app runs. The wait counts as
+            // part of the run, so deleting or repairing the data source ends the wait as well,
+            // instead of waiting for the run before it to finish first.
+            //
+            await this.runGate.WaitAsync(runTokenSource.Token);
+            try
+            {
+                await this.ProcessDataSourceAsync(dataSource, refreshMode, runControl, runTokenSource.Token);
+            }
+            finally
+            {
+                this.runGate.Release();
+            }
         }
         catch (OperationCanceledException) when (!parentToken.IsCancellationRequested && runTokenSource.IsCancellationRequested)
         {
@@ -641,23 +673,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 refreshMode);
 
             //
-            // A data source which is still there has its next run queued already. Its row says so
-            // instead of showing this run as running: a run of the startup check would otherwise
-            // keep that row until the check has worked through every other data source.
+            // A data source which is still there has its next run asked for already, which starts
+            // over with the current settings, in a pass of its own. Its row says so instead of
+            // showing this run as running.
             //
+            this.passes.TryRemove(dataSource.Id, out _);
             if (this.TryGetConfiguredIndexedSource(dataSource.Id, out var configuredDataSource))
-            {
-                this.statuses.TryGetValue(configuredDataSource.Id, out var currentStatus);
-                this.UpsertStatus(this.CreateStatus(
-                    configuredDataSource,
-                    DataSourceEmbeddingState.QUEUED,
-                    currentStatus?.TotalDocuments ?? 0,
-                    currentStatus?.IndexedDocuments ?? 0,
-                    currentStatus?.FailedDocuments ?? 0,
-                    lastError: currentStatus?.LastError ?? string.Empty,
-                    failures: currentStatus?.Failures ?? [],
-                    lastSyncUtc: currentStatus?.LastSyncUtc));
-            }
+                this.UpsertStatus(this.GetQueuedStatus(configuredDataSource));
         }
         finally
         {
@@ -678,10 +700,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     /// </remarks>
     /// <param name="dataSource">The data source to index.</param>
     /// <param name="refreshMode">Why the run was started, for the log.</param>
+    /// <param name="share">How much work the run does at most, as its kind of data source says.</param>
     /// <param name="runControl">The run, which records the embedding settings it works for.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>The context of the run, or null when there is no run.</returns>
-    private async Task<IndexedRunContext?> PrepareIndexedRunAsync(IIndexedDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, DataSourceRunControl runControl, CancellationToken token)
+    private async Task<IndexedRunContext?> PrepareIndexedRunAsync(IIndexedDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, RunShare share, DataSourceRunControl runControl, CancellationToken token)
     {
         logger.LogInformation(
             "Starting background embedding hash check for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}.",
@@ -793,7 +816,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, embeddingSignature, collectionName, vectorStore, indexStore, token);
         token.ThrowIfCancellationRequested();
 
-        return new IndexedRunContext(dataSource, embeddingProvider, embeddingProvider.CreateProvider(), vectorStore, indexStore, manifest, settingsManager, this.UpsertStatus, logger);
+        //
+        // Only now, with the run signature recorded and checked, and with whatever an outdated
+        // manifest threw away: a pass belongs to the embedding settings it began with.
+        //
+        var lastPass = this.passes.GetValueOrDefault(dataSource.Id);
+        var pass = IndexingPass.ContinueOrBegin(lastPass, refreshMode, runSignature);
+        this.passes[dataSource.Id] = pass;
+        if (pass == lastPass)
+            logger.LogInformation(
+                "Carrying on the indexing pass of data source '{DataSourceName}' ({DataSourceId}). FailedEarlierInPass={FailedEarlierInPass}, RefreshMode={RefreshMode}.",
+                dataSource.Name,
+                dataSource.Id,
+                pass.FailedDocumentCount,
+                refreshMode);
+
+        return new IndexedRunContext(dataSource, embeddingProvider, embeddingProvider.CreateProvider(), vectorStore, indexStore, manifest, share, pass, settingsManager, this.UpsertStatus, logger);
     }
 
     private async Task ProcessDataSourceAsync(IIndexedDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, DataSourceRunControl runControl, CancellationToken token)
@@ -807,18 +845,49 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             return;
         }
 
-        var context = await this.PrepareIndexedRunAsync(dataSource, refreshMode, runControl, token);
-        if (context is null)
-            return;
+        var outcome = IndexedRunOutcome.DONE;
+        try
+        {
+            var context = await this.PrepareIndexedRunAsync(dataSource, refreshMode, indexer.Share, runControl, token);
+            if (context is null)
+                return;
 
-        //
-        // Queued behind whatever else waits, so a data source which takes hours gives the others
-        // their turn. While this run is still active, that is the follow-up run every request
-        // during a run leaves behind. Should the user ask for a run of their own in the meantime,
-        // theirs comes first, and carries on just the same.
-        //
-        if (await indexer.ProcessAsync(context, refreshMode, token) is IndexedRunOutcome.MORE_TO_DO)
-            await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.CONTINUATION);
+            var runStartedUtc = DateTimeOffset.UtcNow;
+            outcome = await indexer.ProcessAsync(context, refreshMode, token);
+
+            //
+            // A kind of data source which never asks whether its run did its share holds up every
+            // other data source for as long as the run takes, and nothing fails to point that out:
+            //
+            var runDuration = DateTimeOffset.UtcNow - runStartedUtc;
+            if (context.ShareStartedUtc is null && runDuration > context.Share.MaxDuration * 2)
+                logger.LogWarning(
+                    "The run of data source '{DataSourceName}' ({DataSourceId}) took {RunMinutes:F0} minutes without asking once whether it did its share of {ShareMinutes:F0} minutes, so every other data source waited that long. RefreshMode={RefreshMode}.",
+                    dataSource.Name,
+                    dataSource.Id,
+                    runDuration.TotalMinutes,
+                    context.Share.MaxDuration.TotalMinutes,
+                    refreshMode);
+
+            //
+            // While this run is still active, that is the follow-up run every request during a run
+            // leaves behind. It waits in the back lane, behind every data source which begins a
+            // pass and behind the other ones which carry on, so a data source which takes hours
+            // gives the others their turn, cf. GetQueueLane. Should the user ask for a run of their
+            // own in the meantime, the two merge into one.
+            //
+            if (outcome is IndexedRunOutcome.MORE_TO_DO)
+                await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.CONTINUATION);
+        }
+        finally
+        {
+            //
+            // A pass lives only while the last step left more to do. Whatever else ended this run,
+            // the next one begins anew, and tries every document which failed so far once more:
+            //
+            if (outcome is not IndexedRunOutcome.MORE_TO_DO)
+                this.passes.TryRemove(dataSource.Id, out _);
+        }
     }
 
     private async Task DeleteCollectionAsync(string collectionName, VectorStoreClient? vectorStore, CancellationToken token)
@@ -889,15 +958,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             if (this.statuses.TryGetValue(dataSource.Id, out var knownStatus) && knownStatus.State is DataSourceEmbeddingState.RUNNING)
                 continue;
 
-            this.statuses[dataSource.Id] = this.CreateStatus(
-                dataSource,
-                DataSourceEmbeddingState.QUEUED,
-                knownStatus?.TotalDocuments ?? 0,
-                knownStatus?.IndexedDocuments ?? 0,
-                knownStatus?.FailedDocuments ?? 0,
-                failures: knownStatus?.Failures ?? [],
-                permanentlySkippedDocuments: knownStatus?.PermanentlySkippedDocuments ?? 0,
-                lastSyncUtc: knownStatus?.LastSyncUtc);
+            this.statuses[dataSource.Id] = this.GetQueuedStatus(dataSource);
         }
 
         // One message for the whole list, rather than one per data source:
@@ -906,28 +967,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         foreach (var dataSource in supportedDataSources)
         {
             token.ThrowIfCancellationRequested();
-            try
+
+            //
+            // A data source which runs, or waits in the queue, gets its run that way already.
+            // Running it here as well would run it twice in a row, or next to the run going on.
+            //
+            if (!this.runQueue.TryBeginRun(dataSource.Id))
             {
-                await this.ProcessDataSourceRunAsync(dataSource, DataSourceEmbeddingRefreshMode.STARTUP_HASH_CHECK, token);
-            }
-            catch (OperationCanceledException) when (token.IsCancellationRequested)
-            {
-                throw;
-            }
-            catch (VectorStoreUnreadableException exception)
-            {
-                logger.LogError(
-                    exception,
-                    "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.",
-                    dataSource.Name,
-                    dataSource.Id);
-                this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource));
-            }
-            catch (Exception exception)
-            {
-                logger.LogError(exception, "Initial embedding hash check failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
-                this.UpsertStatus(this.GetFallbackStatus(dataSource, string.Format(TB("The data source '{0}' could not be processed. The log file holds the details."), dataSource.Name)));
+                logger.LogDebug("Skipping the initial hash check of data source '{DataSourceName}' ({DataSourceId}) because it is running or queued already.", dataSource.Name, dataSource.Id);
+                continue;
             }
+
+            await this.RunDataSourceAsync(dataSource.Id, DataSourceEmbeddingRefreshMode.STARTUP_HASH_CHECK, token);
         }
 
         if (!settingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh)
@@ -1117,6 +1168,44 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             LastSyncUtc: lastSyncUtc);
     }
 
+    /// <summary>
+    /// Shows a data source as waiting for its next run.
+    /// </summary>
+    /// <param name="dataSource">The data source.</param>
+    /// <returns>Its status so far as a queued one, or an empty queued status when it has none yet.</returns>
+    private DataSourceEmbeddingStatus GetQueuedStatus(IDataSourceBase dataSource) => this.statuses.TryGetValue(dataSource.Id, out var status)
+        ? AsQueued(status, dataSource)
+        : this.CreateStatus(dataSource, DataSourceEmbeddingState.QUEUED, 0, 0, 0);
+
+    /// <summary>
+    /// Turns the status of a data source into one which waits for the next run.
+    /// </summary>
+    /// <remarks>
+    /// Keeps what the row showed of the runs so far: the counts, the documents skipped for good, the
+    /// failures with the sentence about the data source as a whole, and the last sync. A data source
+    /// which pauses after its share of the work shows its progress that way, instead of a bar which
+    /// jumps back with every step.
+    ///
+    /// Clears what belongs to a run going on: the current document, and whatever the user was asked
+    /// to decide. The next run asks that anew when it still applies. The name comes from the data
+    /// source, since it may have been renamed in the meantime.
+    /// </remarks>
+    /// <param name="status">The status so far.</param>
+    /// <param name="dataSource">The data source.</param>
+    /// <returns>The queued status.</returns>
+    internal static DataSourceEmbeddingStatus AsQueued(DataSourceEmbeddingStatus status, IDataSourceBase dataSource) => status with
+    {
+        DataSourceName = dataSource.Name,
+        DataSourceType = dataSource.Type,
+        State = DataSourceEmbeddingState.QUEUED,
+        CurrentDocument = string.Empty,
+        CurrentDocumentBlock = null,
+        CurrentDocumentPage = null,
+        VectorStoreUnreadable = false,
+        Attention = DataSourceAttention.NONE,
+        PendingRemovalCount = null,
+    };
+
     private DataSourceEmbeddingStatus GetFallbackStatus(IDataSourceBase dataSource, string errorMessage)
     {
         return this.CreateStatus(
@@ -1148,69 +1237,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             vectorStoreUnreadable: true);
     }
 
-    /// <remarks>
-    /// A request arriving while the data source is being embedded leaves a mark for one follow-up
-    /// run, and the mark keeps why it was asked for. The first request decides that: any later one
-    /// only confirms that a follow-up is needed, which it already is.
-    /// </remarks>
-    private DataSourceQueueRequestResult TryReserveDataSourceQueueSlot(string dataSourceId, bool queueAfterCurrentRun, DataSourceEmbeddingRefreshMode refreshMode)
-    {
-        lock (this.queueStateLock)
-        {
-            if (this.runningIds.ContainsKey(dataSourceId))
-            {
-                if (queueAfterCurrentRun && this.pendingRefreshModes.TryAdd(dataSourceId, refreshMode))
-                    return DataSourceQueueRequestResult.RUNNING_MARKED_PENDING;
-
-                return DataSourceQueueRequestResult.RUNNING;
-            }
-
-            if (!this.queuedIds.TryAdd(dataSourceId, 0))
-                return DataSourceQueueRequestResult.ALREADY_QUEUED;
-
-            return DataSourceQueueRequestResult.QUEUED;
-        }
-    }
-
-    private void MarkDataSourceRunStarted(string dataSourceId)
-    {
-        lock (this.queueStateLock)
-        {
-            this.queuedIds.TryRemove(dataSourceId, out _);
-            this.runningIds.TryAdd(dataSourceId, 0);
-        }
-    }
-
-    private bool TryCompleteDataSourceRun(string dataSourceId, bool allowPendingRequeue, out DataSourceEmbeddingRefreshMode pendingRefreshMode)
-    {
-        lock (this.queueStateLock)
-        {
-            this.runningIds.TryRemove(dataSourceId, out _);
-
-            if (!this.pendingRefreshModes.TryRemove(dataSourceId, out pendingRefreshMode))
-                return false;
-
-            return allowPendingRequeue && this.queuedIds.TryAdd(dataSourceId, 0);
-        }
-    }
-
-    private void ReleaseQueuedDataSourceRun(string dataSourceId)
-    {
-        lock (this.queueStateLock)
-        {
-            this.queuedIds.TryRemove(dataSourceId, out _);
-        }
-    }
-
-    private void ClearQueuedDataSourceState(string dataSourceId)
-    {
-        lock (this.queueStateLock)
-        {
-            this.queuedIds.TryRemove(dataSourceId, out _);
-            this.pendingRefreshModes.TryRemove(dataSourceId, out _);
-        }
-    }
-
     private DataSourceRunControl? CancelActiveDataSourceRun(IDataSourceBase dataSource)
     {
         if (!this.activeRuns.TryGetValue(dataSource.Id, out var activeRun))
@@ -1314,38 +1340,46 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     {
         IIndexedDataSource? dataSource = null;
         var isConfigured = !token.IsCancellationRequested && this.TryGetConfiguredIndexedSource(dataSourceId, out dataSource);
-        if (!this.TryCompleteDataSourceRun(dataSourceId, isConfigured, out var refreshMode))
+        if (!this.runQueue.TryComplete(dataSourceId, isConfigured, out var refreshMode))
             return;
 
         if (dataSource is null)
         {
-            this.ReleaseQueuedDataSourceRun(dataSourceId);
+            this.runQueue.Release(dataSourceId);
             return;
         }
 
-        logger.LogInformation("Queueing one follow-up embedding run for data source '{DataSourceName}' ({DataSourceId}) after changes arrived during the previous run. RefreshMode={RefreshMode}.", dataSource.Name, dataSource.Id, refreshMode);
+        var lane = this.GetQueueLane(dataSourceId);
+        logger.LogInformation("Queueing one follow-up embedding run for data source '{DataSourceName}' ({DataSourceId}) after changes arrived during the previous run, or after it did its share of the work. RefreshMode={RefreshMode}, Lane={Lane}.", dataSource.Name, dataSource.Id, refreshMode, lane);
 
-        this.statuses.TryGetValue(dataSource.Id, out var currentStatus);
-        this.UpsertStatus(this.CreateStatus(
-            dataSource,
-            DataSourceEmbeddingState.QUEUED,
-            currentStatus?.TotalDocuments ?? 0,
-            currentStatus?.IndexedDocuments ?? 0,
-            currentStatus?.FailedDocuments ?? 0,
-            lastError: currentStatus?.LastError ?? string.Empty,
-            failures: currentStatus?.Failures ?? [],
-            lastSyncUtc: currentStatus?.LastSyncUtc));
+        this.UpsertStatus(this.GetQueuedStatus(dataSource));
 
         try
         {
-            await this.queue.Writer.WriteAsync(new DataSourceEmbeddingQueueItem(dataSourceId, refreshMode), token);
+            await this.runQueue.EnqueueAsync(dataSourceId, lane, token);
         }
         catch (OperationCanceledException) when (token.IsCancellationRequested)
         {
-            this.ReleaseQueuedDataSourceRun(dataSourceId);
+            this.runQueue.Release(dataSourceId);
         }
     }
 
+    /// <summary>
+    /// Picks the lane of the queue in which the next run of a data source waits.
+    /// </summary>
+    /// <remarks>
+    /// A data source which carries on a pass waits behind every data source which begins one. A
+    /// mailbox at its interval therefore waits for one step at most, rather than for hours behind a
+    /// large folder. Since a pass begins without one, every pass gets one step in the front lane at
+    /// most: no data source keeps the front lane to itself, however long it takes.
+    ///
+    /// Asked when the run is enqueued, and only then. For a follow-up run, that is after the run
+    /// before it ended, which kept or dropped the pass by then.
+    /// </remarks>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    /// <returns>The back lane while the data source has a pass, the front lane otherwise.</returns>
+    private DataSourceQueueLane GetQueueLane(string dataSourceId) => this.passes.ContainsKey(dataSourceId) ? DataSourceQueueLane.BACK : DataSourceQueueLane.FRONT;
+
     private void UpsertStatus(DataSourceEmbeddingStatus status)
     {
         this.statuses[status.DataSourceId] = status;

app/MindWork AI Studio/Tools/Services/DataSourceQueueLane.cs

@@ -0,0 +1,23 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// The lanes of the queue of indexing runs, cf. DataSourceRunQueue.
+/// </summary>
+/// <remarks>
+/// Every run in the front lane comes out before any run in the back lane. DataSourceEmbeddingService
+/// picks the lane when it enqueues a run: a data source which carries on a pass of its indexing waits
+/// in the back lane, cf. IndexingPass. A mailbox therefore waits for one step at most, rather than for
+/// hours behind a large folder, and large data sources take turns with each other.
+/// </remarks>
+internal enum DataSourceQueueLane
+{
+    /// <summary>
+    /// For a data source which begins a pass, e.g. a mailbox at its interval.
+    /// </summary>
+    FRONT,
+
+    /// <summary>
+    /// For a data source which carries on a pass, after it did its share of the work.
+    /// </summary>
+    BACK,
+}

app/MindWork AI Studio/Tools/Services/DataSourceQueueRequestResult.cs

@@ -0,0 +1,27 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// What became of a request to queue an indexing run, cf. DataSourceRunQueue.TryReserve.
+/// </summary>
+internal enum DataSourceQueueRequestResult
+{
+    /// <summary>
+    /// The data source has its place in the queue reserved, and whoever asked enqueues it now.
+    /// </summary>
+    QUEUED,
+
+    /// <summary>
+    /// The data source waits in the queue already, and the request merged into that run.
+    /// </summary>
+    ALREADY_QUEUED,
+
+    /// <summary>
+    /// The data source runs, and either has its follow-up run already or was asked for none.
+    /// </summary>
+    RUNNING,
+
+    /// <summary>
+    /// The data source runs, and gets one follow-up run once the current one completes.
+    /// </summary>
+    RUNNING_MARKED_PENDING,
+}

app/MindWork AI Studio/Tools/Services/DataSourceRunQueue.cs

@@ -0,0 +1,268 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Threading.Channels;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// The queue of indexing runs: which data sources wait for a run, which run, and which get one more run afterward.
+/// </summary>
+/// <remarks>
+/// A data source holds at most one place in the queue, and at most one follow-up run while it runs.
+///
+/// The queue has two lanes, cf. DataSourceQueueLane. Every run in the front lane comes out before
+/// any run in the back lane, and each lane hands out its runs in the order they came in. The data
+/// sources in the back lane therefore take turns.
+///
+/// The channel carries copies of ids, each with its lane and a number. Why a run was asked for is
+/// kept next to it, together with the number of the copy which counts, and taking a run out of the
+/// queue takes both along. Any other copy of the same id which is still in the channel then finds
+/// nothing, or another number, and is skipped. That is how a data source which was deleted or
+/// repaired while it waited leaves nothing behind: no run in an outdated mode, no second run
+/// circling through the queue next to the one which was queued anew, and no copy in the front lane
+/// which lets a later run of the data source overtake the back lane.
+///
+/// Everything but the channel is guarded by one lock, so reserving, starting and completing a run
+/// never see each other halfway.
+/// </remarks>
+internal sealed class DataSourceRunQueue
+{
+    private readonly Channel<QueuedCopy> channel = Channel.CreateUnboundedPrioritized(new UnboundedPrioritizedChannelOptions<QueuedCopy> { Comparer = Comparer<QueuedCopy>.Create(CompareCopies) });
+
+    /// <summary>
+    /// The data sources which wait for a run, each with why it was asked for.
+    /// </summary>
+    private readonly Dictionary<string, DataSourceEmbeddingRefreshMode> queuedModes = new(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// The data sources which wait in the channel, each with the number of the copy which counts.
+    /// </summary>
+    /// <remarks>
+    /// A data source whose place is reserved gets its number once it is enqueued.
+    /// </remarks>
+    private readonly Dictionary<string, long> queuedCopyNumbers = new(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// The data sources whose run has started and not completed yet.
+    /// </summary>
+    private readonly HashSet<string> runningIds = new(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// The running data sources which get one more run once the current one completes, each with why it was asked for.
+    /// </summary>
+    private readonly Dictionary<string, DataSourceEmbeddingRefreshMode> pendingModes = new(StringComparer.OrdinalIgnoreCase);
+
+    private readonly Lock stateLock = new();
+
+    private long lastCopyNumber;
+
+    /// <summary>
+    /// One copy of the id of a data source in the channel.
+    /// </summary>
+    /// <param name="DataSourceId">The id of the data source.</param>
+    /// <param name="Lane">The lane the copy waits in.</param>
+    /// <param name="Number">Counts up with every copy. It keeps each lane in the order its copies came in, and tells the copy which counts from those left behind.</param>
+    private sealed record QueuedCopy(string DataSourceId, DataSourceQueueLane Lane, long Number);
+
+    /// <summary>
+    /// Reserves a place in the queue for a data source, or one follow-up run while it runs.
+    /// </summary>
+    /// <remarks>
+    /// A request for a data source which holds a place or a follow-up run already merges into it.
+    /// The first request decides why the run happens, and any later one only confirms that it is
+    /// needed. A manual retry is the exception and replaces any other mode: it is the only one which
+    /// signs in to a mailbox again after its server refused a sign-in, cf. MailboxIndexer. Behind a
+    /// sync at the interval which waits already, the retry the user asked for would be lost.
+    ///
+    /// The place is taken once its id is in the channel, cf. EnqueueAsync.
+    /// </remarks>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    /// <param name="queueAfterCurrentRun">Whether a request while the data source runs leaves a follow-up run.</param>
+    /// <param name="refreshMode">Why the run is asked for.</param>
+    /// <returns>QUEUED when the caller has to enqueue the data source now, or what became of the request instead.</returns>
+    public DataSourceQueueRequestResult TryReserve(string dataSourceId, bool queueAfterCurrentRun, DataSourceEmbeddingRefreshMode refreshMode)
+    {
+        lock (this.stateLock)
+        {
+            if (this.runningIds.Contains(dataSourceId))
+            {
+                if (!queueAfterCurrentRun)
+                    return DataSourceQueueRequestResult.RUNNING;
+
+                return TryAddOrMerge(this.pendingModes, dataSourceId, refreshMode)
+                    ? DataSourceQueueRequestResult.RUNNING_MARKED_PENDING
+                    : DataSourceQueueRequestResult.RUNNING;
+            }
+
+            return TryAddOrMerge(this.queuedModes, dataSourceId, refreshMode)
+                ? DataSourceQueueRequestResult.QUEUED
+                : DataSourceQueueRequestResult.ALREADY_QUEUED;
+        }
+    }
+
+    /// <summary>
+    /// Puts a data source into the queue, once its place is reserved.
+    /// </summary>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    /// <param name="lane">The lane the data source waits in.</param>
+    /// <param name="token">The cancellation token.</param>
+    public ValueTask EnqueueAsync(string dataSourceId, DataSourceQueueLane lane, CancellationToken token = default)
+    {
+        QueuedCopy copy;
+        lock (this.stateLock)
+        {
+            // Cleared since its place was reserved, e.g. because the data source was deleted:
+            if (!this.queuedModes.ContainsKey(dataSourceId))
+                return ValueTask.CompletedTask;
+
+            copy = new(dataSourceId, lane, ++this.lastCopyNumber);
+            this.queuedCopyNumbers[dataSourceId] = copy.Number;
+        }
+
+        return this.channel.Writer.WriteAsync(copy, token);
+    }
+
+    /// <summary>
+    /// Waits for the next run, takes it out of the queue, and marks it as running.
+    /// </summary>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The run.</returns>
+    public async ValueTask<DataSourceRunRequest> DequeueAsync(CancellationToken token)
+    {
+        while (true)
+        {
+            if (this.TryDequeue(out var runRequest))
+                return runRequest;
+
+            if (!await this.channel.Reader.WaitToReadAsync(token))
+                throw new ChannelClosedException();
+        }
+    }
+
+    /// <summary>
+    /// Takes the next run out of the queue, when there is one, and marks it as running.
+    /// </summary>
+    /// <param name="runRequest">The run, when there is one.</param>
+    /// <returns>True when a run was taken out of the queue.</returns>
+    public bool TryDequeue([NotNullWhen(true)] out DataSourceRunRequest? runRequest)
+    {
+        while (this.channel.Reader.TryRead(out var copy))
+        {
+            lock (this.stateLock)
+            {
+                // Released or cleared while it waited, or enqueued anew since, with another copy which counts instead:
+                if (!this.queuedCopyNumbers.TryGetValue(copy.DataSourceId, out var number) || number != copy.Number)
+                    continue;
+
+                this.queuedCopyNumbers.Remove(copy.DataSourceId);
+                this.queuedModes.Remove(copy.DataSourceId, out var refreshMode);
+                this.runningIds.Add(copy.DataSourceId);
+                runRequest = new(copy.DataSourceId, refreshMode);
+                return true;
+            }
+        }
+
+        runRequest = null;
+        return false;
+    }
+
+    /// <summary>
+    /// Marks a data source as running which did not come out of the queue, unless it runs or waits in the queue already.
+    /// </summary>
+    /// <remarks>
+    /// For the startup check, which works through every data source on its own. Its runs have to
+    /// count as running as well: a request during one of them then leaves one follow-up run, instead
+    /// of a second run next to it.
+    /// </remarks>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    /// <returns>True when the caller runs the data source now, and completes the run with TryComplete.</returns>
+    public bool TryBeginRun(string dataSourceId)
+    {
+        lock (this.stateLock)
+        {
+            if (this.queuedModes.ContainsKey(dataSourceId))
+                return false;
+
+            return this.runningIds.Add(dataSourceId);
+        }
+    }
+
+    /// <summary>
+    /// Marks the run of a data source as completed, and reserves the place of its follow-up run when it has one.
+    /// </summary>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    /// <param name="requeuePending">False drops the follow-up run, e.g. because the data source is gone.</param>
+    /// <param name="pendingRefreshMode">Why the follow-up run was asked for.</param>
+    /// <returns>True when the caller has to enqueue the follow-up run now.</returns>
+    public bool TryComplete(string dataSourceId, bool requeuePending, out DataSourceEmbeddingRefreshMode pendingRefreshMode)
+    {
+        lock (this.stateLock)
+        {
+            this.runningIds.Remove(dataSourceId);
+
+            if (!this.pendingModes.Remove(dataSourceId, out pendingRefreshMode))
+                return false;
+
+            return requeuePending && TryAddOrMerge(this.queuedModes, dataSourceId, pendingRefreshMode);
+        }
+    }
+
+    /// <summary>
+    /// Gives up the reserved place of a data source, because it could not be enqueued.
+    /// </summary>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    public void Release(string dataSourceId)
+    {
+        lock (this.stateLock)
+        {
+            this.queuedModes.Remove(dataSourceId);
+            this.queuedCopyNumbers.Remove(dataSourceId);
+        }
+    }
+
+    /// <summary>
+    /// Forgets the place and the follow-up run of a data source, e.g. because it was deleted.
+    /// </summary>
+    /// <remarks>
+    /// A run which is going on stays marked as running until it completes. Whoever calls this stops
+    /// that run on their own.
+    /// </remarks>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    public void Clear(string dataSourceId)
+    {
+        lock (this.stateLock)
+        {
+            this.queuedModes.Remove(dataSourceId);
+            this.queuedCopyNumbers.Remove(dataSourceId);
+            this.pendingModes.Remove(dataSourceId);
+        }
+    }
+
+    /// <summary>
+    /// Orders the copies in the channel: the front lane before the back lane, and each lane in the order its copies came in.
+    /// </summary>
+    private static int CompareCopies(QueuedCopy x, QueuedCopy y)
+    {
+        if (x.Lane != y.Lane)
+            return x.Lane is DataSourceQueueLane.FRONT ? -1 : 1;
+
+        return x.Number.CompareTo(y.Number);
+    }
+
+    /// <summary>
+    /// Records why a run of a data source was asked for, or merges the request into the one recorded already.
+    /// </summary>
+    /// <param name="modes">The recorded runs.</param>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    /// <param name="refreshMode">Why the run is asked for.</param>
+    /// <returns>True when there was no run recorded for the data source yet.</returns>
+    private static bool TryAddOrMerge(Dictionary<string, DataSourceEmbeddingRefreshMode> modes, string dataSourceId, DataSourceEmbeddingRefreshMode refreshMode)
+    {
+        if (modes.TryAdd(dataSourceId, refreshMode))
+            return true;
+
+        if (refreshMode is DataSourceEmbeddingRefreshMode.MANUAL_RETRY)
+            modes[dataSourceId] = refreshMode;
+
+        return false;
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceRunRequest.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// A run of a data source, as it comes out of the queue.
+/// </summary>
+/// <param name="DataSourceId">The id of the data source.</param>
+/// <param name="RefreshMode">Why the run was asked for.</param>
+internal sealed record DataSourceRunRequest(string DataSourceId, DataSourceEmbeddingRefreshMode RefreshMode);

app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md

@@ -21,7 +21,14 @@ The parts:
   attempt left behind.
 - **`EmbeddingDocument`** - one document: its key, its index row, its display name, and how to read its chunks.
 - **`DocumentRunProgress`** - counts the documents, records indexed and failed ones in the stores, publishes the
-  status and completes the run.
+  status, and completes or pauses the run.
+- **`RunShare`** - how much one run may do before the other data sources get their turn: five minutes, and for
+  a mailbox at most 500 mails. A large data source is indexed in steps, one run each. A run which returns
+  `MORE_TO_DO` gets its next step in the back lane of `DataSourceRunQueue`, behind every data source which begins
+  a pass, so a mailbox waits for one step at most.
+- **`IndexingPass`** - the steps of one data source, from its first step until one of them is done. It remembers
+  which documents failed during an earlier step, so the next steps do not read them again, and which failures
+  were reported to the user already.
 - **`TextChunker`** - cuts text into chunks the embedding provider accepts. Pick one of its strategies; do not
   write a chunker of your own.
 
@@ -35,6 +42,13 @@ To add a kind of data source:
 4. Keep whatever the kind has to remember beyond its documents in tables of its own in the index store, added
    by an EF Core migration (see `app/MindWork AI Studio/Tools/Databases/AGENTS.md`).
 5. Report every status through `DocumentRunProgress`, so all rows of the embedding page behave alike.
+6. Work in steps. Give the indexer its `Share`, and ask `DocumentRunProgress.HasDoneItsShare` before every
+   document. Leave out the documents which failed during an earlier step of the pass: either ask
+   `SkipIfFailedEarlierInPass` per document, or sort them out up front like `FileSourceIndexer.PartitionFiles`
+   and count them with `RecordFailedEarlierInPass`. Once the share is done, return what `PauseRunAsync`
+   returns: it decides whether the data source carries on in another run, or stops because its embedding
+   provider failed for everything. A run which takes more than twice its share without asking once ends up as
+   a warning in the log, while every other data source waits.
 
 Rules which are easy to break:
 - **A document key is not a path.** Only files use their full path as the key. Never pass a key through the
@@ -45,3 +59,11 @@ Rules which are easy to break:
   deliberately: that rebuilds every index.
 - **The service decides when, the indexer decides how.** Whether changes are tracked at all depends on the
   automatic refresh setting and the startup hash check, and only the embedding service decides that.
+- **A pause stores the source hash only when a partial index may be searched.** A stored source hash tells the
+  data source selection that a run got through the whole data source (`IsIndexAwaitingRebuild`). Files pass
+  `null` to `PauseRunAsync`, so a folder stays out of the selection until its first indexing or its rebuild is
+  complete. The mailbox passes its hash, since every mail it indexed can be searched right away.
+- **A pass lives only while the last step left more to do.** The service drops the pass of a data source after
+  every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a
+  step with other embedding settings, begin a new pass (`IndexingPass.ContinueOrBegin`). A pass kept beyond that
+  leaves out documents which would work by now, for example once the embedding provider is reachable again.

app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs

@@ -14,7 +14,8 @@ namespace AIStudio.Tools.Services.Indexing;
 /// Counts the documents, keeps the list of failures and turns both into the status the embedding
 /// page shows. What happens to a document after it was indexed, or after it failed, is the same for
 /// every kind of data source, so that is decided here as well: the stores learn about it, the
-/// manifest follows, and the user hears about it the way the failure deserves.
+/// manifest follows, and the user hears about it the way the failure deserves. So is when a run did
+/// its share of the work and leaves the rest to another one.
 /// </remarks>
 /// <param name="context">The run the documents are indexed in.</param>
 /// <param name="totalDocuments">How many documents the data source has, readable or not.</param>
@@ -31,12 +32,20 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
 
     private readonly List<DataSourceEmbeddingFailure> failures = inputFailures.ToList();
 
-    //
-    // Which kinds of provider failure the user was already told about in this run. A rejected
-    // API key is the same problem for every one of a few thousand documents, and one message
-    // is what it takes to send the user to the settings.
-    //
-    private readonly HashSet<ProviderRequestFailureReason> reportedFailureReasons = [];
+    /// <summary>
+    /// The documents this run read, whether indexing them worked or not, cf. HasDoneItsShare.
+    /// </summary>
+    private int readDocuments;
+
+    /// <summary>
+    /// The documents this run read which failed at the embedding provider, cf. PauseRunAsync.
+    /// </summary>
+    private int failedAtProvider;
+
+    /// <summary>
+    /// The documents this run read which failed for any other reason which may pass, cf. PauseRunAsync.
+    /// </summary>
+    private int failedOtherwise;
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DocumentRunProgress).Namespace, nameof(DocumentRunProgress));
 
@@ -96,6 +105,38 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
         this.failures.Add(new DataSourceEmbeddingFailure(documentKey, failure.Message, failure.OccurredAtUtc, ExtractionCode: failure.Code, IsPermanent: true));
     }
 
+    /// <summary>
+    /// Skips a document which failed during an earlier step of the same pass, unless it changed since.
+    /// </summary>
+    /// <remarks>
+    /// Such a document counts as failed, and its failure keeps its place in the list, the same as in
+    /// the step which read it. The counts stay right even when documents were deleted in between,
+    /// since only what the step comes across is counted. Should the document still be there once the
+    /// pass is through, the next pass tries it again.
+    /// </remarks>
+    /// <param name="documentKey">The key of the document.</param>
+    /// <param name="fingerprint">The fingerprint of the document as it is now.</param>
+    /// <returns>True when the document is skipped, and the caller moves on to the next one.</returns>
+    public bool SkipIfFailedEarlierInPass(string documentKey, string fingerprint)
+    {
+        if (!context.Pass.TryGetEarlierFailure(documentKey, fingerprint, out var failure))
+            return false;
+
+        this.RecordFailedEarlierInPass(failure);
+        return true;
+    }
+
+    /// <summary>
+    /// Counts a document which was found to have failed during an earlier step of the same pass, cf. SkipIfFailedEarlierInPass.
+    /// </summary>
+    /// <param name="failure">Why it failed back then, as IndexingPass.TryGetEarlierFailure tells.</param>
+    public void RecordFailedEarlierInPass(DataSourceEmbeddingFailure failure)
+    {
+        this.FailedDocuments++;
+        this.LastError = failure.Reason;
+        this.failures.Add(failure);
+    }
+
     /// <summary>
     /// Tells the user interface where the run stands.
     /// </summary>
@@ -155,7 +196,9 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
         await context.IndexStore.UpsertFileAsync(context.DataSource.Id, state, token);
         context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount);
         await context.ForgetPermanentFailureAsync(document.Key, token);
+        context.Pass.Forget(document.Key);
 
+        this.readDocuments++;
         this.IndexedDocuments++;
         if (isNew)
             this.NewDocuments++;
@@ -175,6 +218,9 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
     /// <param name="token">The cancellation token.</param>
     public async Task RecordDocumentFailureAsync(EmbeddingDocument document, Exception exception, CancellationToken token)
     {
+        // Whichever way it failed, trying cost its time. A mail may even fail before its text was read:
+        this.readDocuments++;
+
         var dataSource = context.DataSource;
         switch (exception)
         {
@@ -186,8 +232,11 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
                 // needs to offer the matching way out.
                 //
                 this.FailedDocuments++;
+                this.failedAtProvider++;
                 this.LastError = providerFailure.UserMessage;
-                this.failures.Add(new DataSourceEmbeddingFailure(document.Key, providerFailure.UserMessage, DateTimeOffset.UtcNow, providerFailure.FailureReason, providerFailure.StatusCode, context.EmbeddingProvider.Name, DisplayName: document.DisplayName));
+                var failure = new DataSourceEmbeddingFailure(document.Key, providerFailure.UserMessage, DateTimeOffset.UtcNow, providerFailure.FailureReason, providerFailure.StatusCode, context.EmbeddingProvider.Name, DisplayName: document.DisplayName);
+                this.failures.Add(failure);
+                context.Pass.RecordFailure(document.Key, document.State.Fingerprint, failure);
                 context.Manifest.Files.Remove(document.Key);
                 await context.ForgetPermanentFailureAsync(document.Key, token);
                 await context.CleanupFailedDocumentAsync(document.Key, token);
@@ -202,8 +251,8 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
                     providerFailure.StatusCode);
                 this.Publish(document.DisplayName);
 
-                // Once per kind of failure, not once per document:
-                if (this.reportedFailureReasons.Add(providerFailure.FailureReason))
+                // Once per kind of failure and pass, not once per document or step:
+                if (context.Pass.TryMarkReported(providerFailure.FailureReason))
                     await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, providerFailure.UserMessage));
 
                 break;
@@ -222,6 +271,7 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
                 var indexingMessage = this.GetFailureMessage(extractionFailure.Code, document);
                 this.failures.Add(new DataSourceEmbeddingFailure(document.Key, indexingMessage, occurredAtUtc, ExtractionCode: extractionFailure.Code, IsPermanent: true, DisplayName: document.DisplayName));
                 context.Manifest.Files.Remove(document.Key);
+                context.Pass.Forget(document.Key);
                 await context.CleanupFailedDocumentAsync(document.Key, token);
 
                 var state = document.State;
@@ -252,6 +302,7 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
                 // which would interrupt whatever the user is doing right now.
                 //
                 this.FailedDocuments++;
+                this.failedOtherwise++;
                 var extractionCode = exception is FileExtractionException extractionFailure ? extractionFailure.Code : FileExtractionErrorCode.NONE;
 
                 //
@@ -260,7 +311,9 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
                 //
                 var failureMessage = this.GetFailureMessage(extractionCode, document);
                 this.LastError = failureMessage;
-                this.failures.Add(new DataSourceEmbeddingFailure(document.Key, failureMessage, DateTimeOffset.UtcNow, EmbeddingProviderName: context.EmbeddingProvider.Name, ExtractionCode: extractionCode, DisplayName: document.DisplayName));
+                var failure = new DataSourceEmbeddingFailure(document.Key, failureMessage, DateTimeOffset.UtcNow, EmbeddingProviderName: context.EmbeddingProvider.Name, ExtractionCode: extractionCode, DisplayName: document.DisplayName);
+                this.failures.Add(failure);
+                context.Pass.RecordFailure(document.Key, document.State.Fingerprint, failure);
                 context.Manifest.Files.Remove(document.Key);
                 await context.ForgetPermanentFailureAsync(document.Key, token);
                 await context.CleanupFailedDocumentAsync(document.Key, token);
@@ -272,6 +325,30 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
         }
     }
 
+    /// <summary>
+    /// Whether the run did its share of the work, so other data sources get their turn.
+    /// </summary>
+    /// <remarks>
+    /// Asked before every document which is about to be read. The clock starts with the first
+    /// question, so every run reads at least one document. Counted are the documents which were
+    /// read, whether indexing them worked or not, cf. RunShare.
+    ///
+    /// The share only ends a run between two documents. A document which takes longer than the
+    /// whole share is read to its end, since its text arrives as a stream which cannot pick up where
+    /// it stopped.
+    /// </remarks>
+    /// <param name="nowUtc">The current time, passed in so the share can be tested.</param>
+    /// <returns>True when the run pauses now, cf. PauseRunAsync.</returns>
+    public bool HasDoneItsShare(DateTimeOffset nowUtc)
+    {
+        var shareStartedUtc = context.StartShareClock(nowUtc);
+        var share = context.Share;
+        if (share.MaxDocuments is { } maxDocuments && this.readDocuments >= maxDocuments)
+            return true;
+
+        return nowUtc - shareStartedUtc >= share.MaxDuration;
+    }
+
     /// <summary>
     /// Finishes the run: the collection is tidied up, the data source is marked as worked through,
     /// and the user interface learns how it went.
@@ -286,40 +363,58 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
     public async Task CompleteRunAsync(string sourceHash, string reason, CancellationToken token)
     {
         await this.StoreSourceHashAsync(sourceHash, reason, token);
-
-        //
-        // Documents which were skipped for good do not make a run unsuccessful: nothing is left to
-        // try, and a data source made of nothing but scanned images would otherwise ask for
-        // attention forever.
-        //
-        var hasFailures = this.FailedDocuments > 0;
-        var lastError = hasFailures
-            ? string.IsNullOrWhiteSpace(this.LastError)
-                ? context.DataSource is DataSourceMailbox
-                    ? TB("Some mails could not be indexed. The list below says which ones and why.")
-                    : TB("Some files could not be indexed. The list below says which ones and why.")
-                : this.LastError
-            : string.Empty;
-
-        context.PublishStatus(this.CreateStatus(hasFailures ? DataSourceEmbeddingState.FAILED : DataSourceEmbeddingState.COMPLETED, string.Empty, lastError, null, null));
+        this.PublishCompletion();
     }
 
     /// <summary>
-    /// Ends a run before it got through the whole data source, keeping what it did for the next one.
+    /// Ends a run which did its share of the work before it got through the whole data source, keeping what it did for the next one.
     /// </summary>
     /// <remarks>
-    /// Stores everything CompleteRunAsync stores, the hash included. For a data source which is
-    /// worked through in several runs, the hash therefore says that its index can be searched, not
-    /// that the data source was worked through as a whole: that is what LastSyncUtc tells. Without
-    /// it, a mailbox would stay out of every search for the hours its first sync takes.
+    /// A run hands on to another one unless it indexed nothing and every document it read failed at
+    /// the embedding provider. Then the provider is most likely down, or refuses the API key, and
+    /// every further run would fail the same way, one document after the other. Such a run is
+    /// completed with its failures instead, and waits for the next time the data source is queued.
+    /// Any other failure is about its document, which the pass skips from now on, so the next run
+    /// gets further all the same.
     ///
-    /// The user interface learns nothing here. Whatever comes next tells it: the run being queued
-    /// again, or the question the user has to answer first.
+    /// The hash is stored only when one is given. It then says that the index of the data source can
+    /// be searched, not that the data source was worked through as a whole: that is what LastSyncUtc
+    /// tells. Without it, a mailbox would stay out of every search for the hours its first sync
+    /// takes. Without a hash, the stored one stays as it is, so a folder which is indexed for the
+    /// first time stays out of every search until its index is complete.
+    ///
+    /// When the run carries on, the user interface learns nothing here. Whatever comes next tells
+    /// it: the run being queued again, or the question the user has to answer first.
     /// </remarks>
-    /// <param name="sourceHash">The hash of the data source as this run found it.</param>
-    /// <param name="reason">Why the collection is optimized now, for the log.</param>
+    /// <param name="searchableSourceHash">The hash of the data source as this run found it, when its index may be searched before it is complete. Null otherwise.</param>
+    /// <param name="reason">Why the run pauses, for the log.</param>
     /// <param name="token">The cancellation token.</param>
-    public Task PauseRunAsync(string sourceHash, string reason, CancellationToken token) => this.StoreSourceHashAsync(sourceHash, reason, token);
+    /// <returns>MORE_TO_DO when another run carries on, or DONE when this one was completed instead.</returns>
+    public async Task<IndexedRunOutcome> PauseRunAsync(string? searchableSourceHash, string reason, CancellationToken token)
+    {
+        if (searchableSourceHash is null)
+            await context.OptimizeCollectionIfNeededAsync(reason, token);
+        else
+            await this.StoreSourceHashAsync(searchableSourceHash, reason, token);
+
+        //
+        // Only what this run read counts. Documents which could not even be looked at fail in every
+        // run, and would stop a folder with a single locked subfolder after every step. Neither do
+        // documents which failed for a reason of their own: they are never read again anyway, and
+        // never got as far as the provider.
+        //
+        if (this.IndexedDocuments > 0 || this.failedAtProvider is 0 || this.failedOtherwise > 0)
+            return IndexedRunOutcome.MORE_TO_DO;
+
+        logger.LogWarning(
+            "Indexing data source '{DataSourceName}' ({DataSourceId}) stops without carrying on, since it indexed nothing and the embedding provider failed for all {FailedDocuments} document(s) it was sent.",
+            context.DataSource.Name,
+            context.DataSource.Id,
+            this.failedAtProvider);
+
+        this.PublishCompletion();
+        return IndexedRunOutcome.DONE;
+    }
 
     /// <summary>
     /// Tells the user interface how the data source stands as the index holds it, without a run.
@@ -354,7 +449,15 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
     /// <summary>
     /// Stores what a run found about the data source as a whole, and tidies up the collection.
     /// </summary>
-    private async Task StoreSourceHashAsync(string sourceHash, string reason, CancellationToken token)
+    /// <remarks>
+    /// For a run which stops for a reason of its own and leaves an index which may be searched,
+    /// e.g. the sync of a mailbox which waits for the user to agree to a removal. The user interface
+    /// learns nothing here.
+    /// </remarks>
+    /// <param name="sourceHash">The hash of the data source as this run found it.</param>
+    /// <param name="reason">Why the collection is optimized now, for the log.</param>
+    /// <param name="token">The cancellation token.</param>
+    public async Task StoreSourceHashAsync(string sourceHash, string reason, CancellationToken token)
     {
         context.Manifest.SourceHash = sourceHash;
         token.ThrowIfCancellationRequested();
@@ -365,6 +468,28 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
         token.ThrowIfCancellationRequested();
     }
 
+    /// <summary>
+    /// Tells the user interface that the run is over, and how it went.
+    /// </summary>
+    private void PublishCompletion()
+    {
+        //
+        // Documents which were skipped for good do not make a run unsuccessful: nothing is left to
+        // try, and a data source made of nothing but scanned images would otherwise ask for
+        // attention forever.
+        //
+        var hasFailures = this.FailedDocuments > 0;
+        var lastError = hasFailures
+            ? string.IsNullOrWhiteSpace(this.LastError)
+                ? context.DataSource is DataSourceMailbox
+                    ? TB("Some mails could not be indexed. The list below says which ones and why.")
+                    : TB("Some files could not be indexed. The list below says which ones and why.")
+                : this.LastError
+            : string.Empty;
+
+        context.PublishStatus(this.CreateStatus(hasFailures ? DataSourceEmbeddingState.FAILED : DataSourceEmbeddingState.COMPLETED, string.Empty, lastError, null, null));
+    }
+
     /// <summary>
     /// What the user reads about a document which could not be indexed.
     /// </summary>

app/MindWork AI Studio/Tools/Services/Indexing/FilePartition.cs

@@ -0,0 +1,14 @@
+namespace AIStudio.Tools.Services.Indexing;
+
+/// <summary>
+/// The files of a data source, sorted by what one run of the file indexer does with them.
+/// </summary>
+/// <remarks>
+/// Only the last group costs time. The others are counted before the first file is read, so the
+/// progress of a step starts where the step before left off, rather than at zero.
+/// </remarks>
+/// <param name="UnchangedCount">How many files were indexed and have not changed since.</param>
+/// <param name="StillUnreadable">The files which failed for a reason of their own and have not changed since, each with the stored reason.</param>
+/// <param name="FailedEarlierInPass">Why each file failed which failed during an earlier step of the pass, and is not tried again before the next pass.</param>
+/// <param name="ToRead">The files the run reads, in the order it reads them.</param>
+internal sealed record FilePartition(int UnchangedCount, IReadOnlyList<KeyValuePair<string, PermanentIndexingFailureRecord>> StillUnreadable, IReadOnlyList<DataSourceEmbeddingFailure> FailedEarlierInPass, IReadOnlyList<FileInfo> ToRead);

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs

@@ -23,6 +23,12 @@ namespace AIStudio.Tools.Services.Indexing;
 ///
 /// Changes are noticed by a file system watcher per data source. A burst of changes is waited out
 /// before a run is asked for, so saving a document twice in a row costs one run, not two.
+///
+/// A run reads files for its share of the time, cf. Share, and leaves the rest to the next one, so
+/// a folder which takes hours does not hold up every other data source. Every such step lists the
+/// folder anew, and counts what needs no reading before it reads the first file. A step stores no
+/// hash, so a folder which is indexed for the first time stays out of every search until its index
+/// is complete, cf. DocumentRunProgress.PauseRunAsync.
 /// </remarks>
 /// <param name="settingsManager">The settings, read for whether local data sources refresh on their own.</param>
 /// <param name="rustService">The runtime, which extracts the text of the files.</param>
@@ -49,6 +55,9 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     /// <inheritdoc />
     public bool Supports(IDataSourceBase dataSource) => dataSource is DataSourceLocalDirectory or DataSourceLocalFile;
 
+    /// <inheritdoc />
+    public RunShare Share => new(RunShare.DEFAULT_DURATION);
+
     /// <inheritdoc />
     public async Task<IndexedRunOutcome> ProcessAsync(IndexedRunContext context, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token)
     {
@@ -57,7 +66,9 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
 
         var manifest = context.Manifest;
 
+        var enumerationStartedUtc = DateTimeOffset.UtcNow;
         var inputFiles = this.GetInputFiles(dataSource);
+        var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc;
         var indexedFiles = inputFiles.Files;
         var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
 
@@ -71,12 +82,17 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
                 failure.Reason);
         }
 
+        //
+        // Listing the files is what every step of a large folder pays before it reads anything,
+        // hence the time it took:
+        //
         logger.LogInformation(
-            "Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, FailedFiles={FailedFiles}, Collection='{CollectionName}'.",
+            "Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, FailedFiles={FailedFiles}, EnumerationMs={EnumerationMs:F0}, Collection='{CollectionName}'.",
             dataSource.Name,
             dataSource.Id,
             indexedFiles.Count,
             inputFiles.FailedFiles,
+            enumerationDuration.TotalMilliseconds,
             context.CollectionName);
 
         var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles);
@@ -116,60 +132,61 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         }
 
         token.ThrowIfCancellationRequested();
+
+        //
+        // Everything which needs no reading is counted before the first file is read. Counted one
+        // by one, the progress of every step would start at zero, and climb through the files the
+        // steps before already did.
+        //
+        var partition = PartitionFiles(indexedFiles, metadataSnapshot.FileHashes, manifest, context.Pass);
+        progress.RecordUnchanged(partition.UnchangedCount);
+        foreach (var (filePath, permanentFailure) in partition.StillUnreadable)
+        {
+            logger.LogDebug(
+                "Skipping file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because reading it failed permanently before. FailureCode={FailureCode}, MetadataHashPrefix={MetadataHashPrefix}, OccurredAtUtc={OccurredAtUtc:O}.",
+                filePath,
+                dataSource.Name,
+                dataSource.Id,
+                permanentFailure.Code,
+                ShortHash(permanentFailure.Fingerprint),
+                permanentFailure.OccurredAtUtc);
+            progress.RecordStillUnreadable(filePath, permanentFailure);
+        }
+
+        foreach (var failure in partition.FailedEarlierInPass)
+            progress.RecordFailedEarlierInPass(failure);
+
+        logger.LogInformation(
+            "Sorted the files of data source '{DataSourceName}' ({DataSourceId}). Unchanged={UnchangedFiles}, PermanentlySkipped={PermanentlySkippedFiles}, FailedEarlierInPass={FailedEarlierInPass}, ToRead={FilesToRead}.",
+            dataSource.Name,
+            dataSource.Id,
+            partition.UnchangedCount,
+            partition.StillUnreadable.Count,
+            partition.FailedEarlierInPass.Count,
+            partition.ToRead.Count);
         progress.Publish();
 
         //
-        // Everything the runtime filters out of these files is reported once for the whole data
-        // source. A run over a few thousand documents which removes something in forty of them
-        // is one thing that happened to the user, not forty. The scope ends with this method, so
-        // the report arrives when the run is finished rather than in the middle of it.
+        // Everything the runtime filters out of these files is reported once for the whole step.
+        // A step over a few thousand documents which removes something in forty of them is one
+        // thing that happened to the user, not forty. The scope ends with this method, so the
+        // report arrives when the step is over rather than in the middle of it.
         //
         await using var promptInjectionReportingScope = guardService.BeginAction();
 
-        foreach (var file in indexedFiles)
+        for (var fileIndex = 0; fileIndex < partition.ToRead.Count; fileIndex++)
         {
             token.ThrowIfCancellationRequested();
+            if (progress.HasDoneItsShare(DateTimeOffset.UtcNow))
+                return await this.PauseAsync(dataSource, progress, refreshMode, partition.ToRead.Count - fileIndex, token);
 
+            var file = partition.ToRead[fileIndex];
             var fingerprint = metadataSnapshot.FileHashes[file.FullName];
-            if (manifest.Files.TryGetValue(file.FullName, out var existingRecord) &&
-                string.Equals(existingRecord.Fingerprint, fingerprint, StringComparison.Ordinal))
-            {
-                logger.LogDebug(
-                    "Skipping unchanged file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because the persisted metadata hash matches. MetadataHashPrefix={MetadataHashPrefix}, LastWriteUtc={LastWriteUtc:O}, FileSize={FileSize}.",
-                    file.FullName,
-                    dataSource.Name,
-                    dataSource.Id,
-                    ShortHash(fingerprint),
-                    file.LastWriteTimeUtc,
-                    file.Length);
-                progress.RecordUnchanged();
-                progress.Publish();
-                continue;
-            }
-
-            //
-            // A file which failed for a reason of its own is not read again until it changes.
-            // Without this, a folder holding hundreds of scanned documents without a text layer
-            // would spend half an hour on every start to arrive at the result we already have:
-            //
-            if (manifest.PermanentFailures.TryGetValue(file.FullName, out var permanentFailure) &&
-                string.Equals(permanentFailure.Fingerprint, fingerprint, StringComparison.Ordinal))
-            {
-                logger.LogDebug(
-                    "Skipping file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because reading it failed permanently before. FailureCode={FailureCode}, MetadataHashPrefix={MetadataHashPrefix}, OccurredAtUtc={OccurredAtUtc:O}.",
-                    file.FullName,
-                    dataSource.Name,
-                    dataSource.Id,
-                    permanentFailure.Code,
-                    ShortHash(fingerprint),
-                    permanentFailure.OccurredAtUtc);
-                progress.RecordStillUnreadable(file.FullName, permanentFailure);
-                progress.Publish();
-                continue;
-            }
+            var existingRecord = manifest.Files.GetValueOrDefault(file.FullName);
 
             var document = this.CreateFileDocument(context, dataSource, file, fingerprint);
             var reportBlockProgress = progress.BeginDocument(document);
+            var startedAtUtc = DateTimeOffset.UtcNow;
 
             try
             {
@@ -182,7 +199,6 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
                     ShortHash(fingerprint),
                     progress.DoneDocuments + 1,
                     totalFiles);
-                var startedAtUtc = DateTimeOffset.UtcNow;
                 var chunkCount = await context.IndexDocumentAsync(document, reportBlockProgress, token);
                 token.ThrowIfCancellationRequested();
                 var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
@@ -215,6 +231,21 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             {
                 await progress.RecordDocumentFailureAsync(document, exception, token);
             }
+
+            //
+            // A file is read to its end, however long that takes: its text arrives as a stream from
+            // the runtime, which cannot pick up where it stopped. Every other data source waits
+            // for it, so it should at least show up in the log:
+            //
+            var fileDuration = DateTimeOffset.UtcNow - startedAtUtc;
+            if (fileDuration > context.Share.MaxDuration)
+                logger.LogWarning(
+                    "Reading file '{FilePath}' of data source '{DataSourceName}' ({DataSourceId}) took {FileMinutes:F0} minutes, longer than the share of a whole run, which is {ShareMinutes:F0} minutes. Every other data source waited for it.",
+                    file.FullName,
+                    dataSource.Name,
+                    dataSource.Id,
+                    fileDuration.TotalMinutes,
+                    context.Share.MaxDuration.TotalMinutes);
         }
 
         await progress.CompleteRunAsync(metadataSnapshot.SourceHash, "data source embedding run finished", token);
@@ -236,6 +267,71 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return IndexedRunOutcome.DONE;
     }
 
+    /// <summary>
+    /// Sorts the files of a data source by what one run does with them.
+    /// </summary>
+    /// <remarks>
+    /// A file which failed for a reason of its own is not read again until it changes. Without
+    /// that, a folder holding hundreds of scanned documents without a text layer would spend half
+    /// an hour on every start to arrive at the result we already have. A file which failed during
+    /// an earlier step of the pass waits for the next pass, unless it changed since, cf. IndexingPass.
+    /// </remarks>
+    /// <param name="files">The files of the data source, in the order they are read.</param>
+    /// <param name="fingerprints">The fingerprint of every one of them, by its full path.</param>
+    /// <param name="manifest">What the index stores about the data source.</param>
+    /// <param name="pass">The pass the run is a step of.</param>
+    /// <returns>The files, sorted.</returns>
+    internal static FilePartition PartitionFiles(IReadOnlyList<FileInfo> files, IReadOnlyDictionary<string, string> fingerprints, DataSourceEmbeddingManifest manifest, IndexingPass pass)
+    {
+        var unchangedCount = 0;
+        var stillUnreadable = new List<KeyValuePair<string, PermanentIndexingFailureRecord>>();
+        var failedEarlierInPass = new List<DataSourceEmbeddingFailure>();
+        var toRead = new List<FileInfo>();
+
+        foreach (var file in files)
+        {
+            var fingerprint = fingerprints[file.FullName];
+            if (manifest.Files.TryGetValue(file.FullName, out var existingRecord) && string.Equals(existingRecord.Fingerprint, fingerprint, StringComparison.Ordinal))
+                unchangedCount++;
+            else if (manifest.PermanentFailures.TryGetValue(file.FullName, out var permanentFailure) && string.Equals(permanentFailure.Fingerprint, fingerprint, StringComparison.Ordinal))
+                stillUnreadable.Add(new(file.FullName, permanentFailure));
+            else if (pass.TryGetEarlierFailure(file.FullName, fingerprint, out var earlierFailure))
+                failedEarlierInPass.Add(earlierFailure);
+            else
+                toRead.Add(file);
+        }
+
+        return new(unchangedCount, stillUnreadable, failedEarlierInPass, toRead);
+    }
+
+    /// <summary>
+    /// Ends a run which did its share of the work before it read every file.
+    /// </summary>
+    /// <remarks>
+    /// Without a hash, cf. DocumentRunProgress.PauseRunAsync: a folder which is indexed for the
+    /// first time must not count as complete, and one which was complete before keeps the hash of
+    /// back then, which no longer matches, so the next run reads on.
+    /// </remarks>
+    private async Task<IndexedRunOutcome> PauseAsync(IDataSource dataSource, DocumentRunProgress progress, DataSourceEmbeddingRefreshMode refreshMode, int filesLeftToRead, CancellationToken token)
+    {
+        var outcome = await progress.PauseRunAsync(null, "data source embedding run paused", token);
+        if (outcome is not IndexedRunOutcome.MORE_TO_DO)
+            return outcome;
+
+        logger.LogInformation(
+            "Embedding data source '{DataSourceName}' ({DataSourceId}) carries on in another run. RefreshMode={RefreshMode}, Embedded={EmbeddedFiles}, Failed={FailedFiles}, Done={DoneFiles}/{TotalFiles}, LeftToRead={FilesLeftToRead}.",
+            dataSource.Name,
+            dataSource.Id,
+            refreshMode,
+            progress.IndexedDocuments,
+            progress.FailedDocuments,
+            progress.DoneDocuments,
+            progress.TotalDocuments,
+            filesLeftToRead);
+
+        return IndexedRunOutcome.MORE_TO_DO;
+    }
+
     /// <summary>
     /// Whether a file name marks a file which is never indexed, whatever its type.
     /// </summary>

app/MindWork AI Studio/Tools/Services/Indexing/IIndexedSourceIndexer.cs

@@ -24,13 +24,27 @@ internal interface IIndexedSourceIndexer : IDisposable
     /// <returns>True when this indexer is the one for it.</returns>
     public bool Supports(IDataSourceBase dataSource);
 
+    /// <summary>
+    /// How much work one run does at most, before it leaves the rest to another one.
+    /// </summary>
+    /// <remarks>
+    /// All data sources share one queue, and only one run works at a time. A run which goes on for
+    /// hours holds up every other data source for as long, a mailbox whose sync is due among them.
+    /// So a run asks DocumentRunProgress.HasDoneItsShare before every document it is about to read,
+    /// and pauses through DocumentRunProgress.PauseRunAsync once the answer is yes.
+    ///
+    /// Nothing fails when a run forgets to ask. The embedding service therefore writes a warning into
+    /// the log when a run took more than twice its share without asking once.
+    /// </remarks>
+    public RunShare Share { get; }
+
     /// <summary>
     /// Works through the documents of the data source of a run, from finding them to completing the run.
     /// </summary>
     /// <remarks>
-    /// A run may stop after its share of the work and leave the rest to another one, so a data source
-    /// which takes hours does not hold up all others for that long. Whether and when that run comes
-    /// is the embedding service's to decide.
+    /// A run stops after its share of the work, cf. Share, and leaves the rest to another one, so a
+    /// data source which takes hours does not hold up all others for that long. Whether and when that
+    /// run comes is the embedding service's to decide.
     /// </remarks>
     /// <param name="context">The prepared run, whose data source this indexer supports.</param>
     /// <param name="refreshMode">Why the run was started.</param>

app/MindWork AI Studio/Tools/Services/Indexing/IndexedRunContext.cs

@@ -24,10 +24,12 @@ namespace AIStudio.Tools.Services.Indexing;
 /// <param name="vectorStore">The vector store, known to be available.</param>
 /// <param name="indexStore">The index store, known to be available.</param>
 /// <param name="manifest">What the index stores about the data source, made to match the current embedding configuration.</param>
+/// <param name="share">How much work the run does at most, as its kind of data source says.</param>
+/// <param name="pass">The pass the run is a step of, which knows what failed during the steps before.</param>
 /// <param name="settingsManager">The settings, which the provider reads when it embeds.</param>
 /// <param name="publishStatus">Hands a new status of the data source to the embedding service.</param>
 /// <param name="logger">The logger of the embedding service, so the log reads the same whoever writes it.</param>
-internal sealed class IndexedRunContext(IIndexedDataSource dataSource, EmbeddingProvider embeddingProvider, IProvider provider, VectorStoreClient vectorStore, IndexStoreClient indexStore, DataSourceEmbeddingManifest manifest, SettingsManager settingsManager, Action<DataSourceEmbeddingStatus> publishStatus, ILogger logger)
+internal sealed class IndexedRunContext(IIndexedDataSource dataSource, EmbeddingProvider embeddingProvider, IProvider provider, VectorStoreClient vectorStore, IndexStoreClient indexStore, DataSourceEmbeddingManifest manifest, RunShare share, IndexingPass pass, SettingsManager settingsManager, Action<DataSourceEmbeddingStatus> publishStatus, ILogger logger)
 {
     /// <summary>
     /// After how many stored chunks the collection is optimized while a run is still going.
@@ -58,6 +60,26 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
 
     public string CollectionName { get; } = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
 
+    public RunShare Share => share;
+
+    public IndexingPass Pass => pass;
+
+    /// <summary>
+    /// When the run first asked whether it did its share of the work, or null while it never did.
+    /// </summary>
+    /// <remarks>
+    /// The clock of the share starts then, cf. DocumentRunProgress.HasDoneItsShare. The embedding
+    /// service reads it once the run is over, to notice a kind of data source which never asks.
+    /// </remarks>
+    public DateTimeOffset? ShareStartedUtc { get; private set; }
+
+    /// <summary>
+    /// Starts the clock of the share of the work, unless it runs already.
+    /// </summary>
+    /// <param name="nowUtc">The current time.</param>
+    /// <returns>When the clock started.</returns>
+    public DateTimeOffset StartShareClock(DateTimeOffset nowUtc) => this.ShareStartedUtc ??= nowUtc;
+
     /// <summary>
     /// Tells the user interface about a new status of the data source.
     /// </summary>

app/MindWork AI Studio/Tools/Services/Indexing/IndexingPass.cs

@@ -0,0 +1,132 @@
+using System.Diagnostics.CodeAnalysis;
+
+using AIStudio.Provider;
+
+namespace AIStudio.Tools.Services.Indexing;
+
+/// <summary>
+/// One way through a data source, from its first step to the step which gets through it.
+/// </summary>
+/// <remarks>
+/// A large data source is indexed in steps, each of them a run of its own, cf. RunShare. What one
+/// step learns about documents which failed has to reach the next one, or every step reads them
+/// again: that costs time, with a cloud provider money as well, and the list of failures would lose
+/// whatever the steps before found. A pass keeps it for as long as the steps go on.
+///
+/// The embedding service keeps a pass only while the last step of its data source left more to do.
+/// Any other end of a run ends the pass, and so does a reset of the stored index. A manual retry
+/// begins a new pass, as does a run for other embedding settings.
+///
+/// Documents which failed for a reason of their own are none of its business: the index remembers
+/// them for good, cf. PermanentIndexingFailure. Only the steps of one data source use a pass, one
+/// after the other, so it needs no lock.
+/// </remarks>
+/// <param name="runSignature">The embedding settings the pass works for.</param>
+internal sealed class IndexingPass(DataSourceRunSignature runSignature)
+{
+    /// <summary>
+    /// How often a document is tried during one pass at most, whether it changed in between or not.
+    /// </summary>
+    /// <remarks>
+    /// A file which changes faster than it can be read would otherwise be read in every step, e.g.
+    /// the log file of another program.
+    /// </remarks>
+    private const int MAX_ATTEMPTS = 2;
+
+    /// <summary>
+    /// A document which failed during this pass.
+    /// </summary>
+    /// <param name="Fingerprint">The fingerprint of the document when it failed the last time.</param>
+    /// <param name="Attempts">How often it failed during this pass.</param>
+    /// <param name="Failure">Why it failed the last time, as the list of failures shows it.</param>
+    private sealed record FailedDocument(string Fingerprint, int Attempts, DataSourceEmbeddingFailure Failure);
+
+    /// <summary>
+    /// The documents which failed during this pass for a reason which may pass, by their keys.
+    /// </summary>
+    private readonly Dictionary<string, FailedDocument> failedDocuments = new(StringComparer.OrdinalIgnoreCase);
+
+    //
+    // Which kinds of provider failure the user was already told about in this pass. A rejected API
+    // key is the same problem for every one of a few thousand documents, and one message is what it
+    // takes to send the user to the settings. Kept for the whole pass, so a recurring failure such
+    // as a rate limit does not show up anew with every step.
+    //
+    private readonly HashSet<ProviderRequestFailureReason> reportedFailureReasons = [];
+
+    public DataSourceRunSignature RunSignature => runSignature;
+
+    /// <summary>
+    /// How many documents failed during this pass and are not tried again for now.
+    /// </summary>
+    public int FailedDocumentCount => this.failedDocuments.Count;
+
+    /// <summary>
+    /// The pass a run works in: the one the last step of its data source left, or a new one.
+    /// </summary>
+    /// <param name="current">The pass the last step of the data source left, or null when there is none.</param>
+    /// <param name="refreshMode">Why the run was asked for.</param>
+    /// <param name="runSignature">The embedding settings the run works for.</param>
+    /// <returns>The pass of the run.</returns>
+    public static IndexingPass ContinueOrBegin(IndexingPass? current, DataSourceEmbeddingRefreshMode refreshMode, DataSourceRunSignature runSignature)
+    {
+        //
+        // A manual retry is the user asking for every failed document to be tried again, so it
+        // never carries on with what a pass knows. Neither does a run for other embedding settings:
+        // what failed with the old ones may well work with the new.
+        //
+        if (current is not null && refreshMode is not DataSourceEmbeddingRefreshMode.MANUAL_RETRY && current.RunSignature == runSignature)
+            return current;
+
+        return new(runSignature);
+    }
+
+    /// <summary>
+    /// Records that a document failed for a reason which may pass.
+    /// </summary>
+    /// <param name="documentKey">The key of the document.</param>
+    /// <param name="fingerprint">The fingerprint of the document as it was read.</param>
+    /// <param name="failure">Why it failed, as the list of failures shows it.</param>
+    public void RecordFailure(string documentKey, string fingerprint, DataSourceEmbeddingFailure failure)
+    {
+        var attempts = this.failedDocuments.TryGetValue(documentKey, out var earlierFailure) ? earlierFailure.Attempts + 1 : 1;
+        this.failedDocuments[documentKey] = new(fingerprint, attempts, failure);
+    }
+
+    /// <summary>
+    /// Forgets that a document failed, e.g. because it was indexed after all.
+    /// </summary>
+    /// <param name="documentKey">The key of the document.</param>
+    public void Forget(string documentKey) => this.failedDocuments.Remove(documentKey);
+
+    /// <summary>
+    /// Whether a document failed during this pass and is not tried again before the next one.
+    /// </summary>
+    /// <remarks>
+    /// A document which changed since it failed gets another try, as long as it did not fail
+    /// MAX_ATTEMPTS times during this pass already.
+    /// </remarks>
+    /// <param name="documentKey">The key of the document.</param>
+    /// <param name="fingerprint">The fingerprint of the document as it is now.</param>
+    /// <param name="failure">Why it failed the last time, when it is not tried again.</param>
+    /// <returns>True when the document is left alone.</returns>
+    public bool TryGetEarlierFailure(string documentKey, string fingerprint, [NotNullWhen(true)] out DataSourceEmbeddingFailure? failure)
+    {
+        failure = null;
+        if (!this.failedDocuments.TryGetValue(documentKey, out var failedDocument))
+            return false;
+
+        if (failedDocument.Attempts < MAX_ATTEMPTS && !string.Equals(failedDocument.Fingerprint, fingerprint, StringComparison.Ordinal))
+            return false;
+
+        failure = failedDocument.Failure;
+        return true;
+    }
+
+    /// <summary>
+    /// Marks a kind of provider failure as reported to the user.
+    /// </summary>
+    /// <param name="reason">The kind of provider failure.</param>
+    /// <returns>True when the user was not told about it during this pass yet.</returns>
+    public bool TryMarkReported(ProviderRequestFailureReason reason) => this.reportedFailureReasons.Add(reason);
+}

app/MindWork AI Studio/Tools/Services/Indexing/MailboxIndexer.Mails.cs

@@ -69,6 +69,17 @@ internal sealed partial class MailboxIndexer
             return;
         }
 
+        //
+        // A mail which failed during an earlier step of this sync waits for the next sync, unless
+        // it changed since. Asked only here: the mail counts as encountered all the same, and one
+        // which merely moved was linked above.
+        //
+        if (progress.SkipIfFailedEarlierInPass(key, mailHash))
+        {
+            progress.Publish();
+            return;
+        }
+
         //
         // Named after the subject as the server delivered it. The name is for the user alone, who
         // reads it on the embeddings page as in any mail program. A model only ever gets to read

app/MindWork AI Studio/Tools/Services/Indexing/MailboxIndexer.cs

@@ -28,10 +28,10 @@ namespace AIStudio.Tools.Services.Indexing;
 /// had its chance to link a mail which merely moved there. Even then, a mail is only orphaned, and
 /// deleted one run later, should it not turn up again by then.
 ///
-/// A run reads a limited number of mails, for a limited time, and leaves the rest to the next run.
-/// The first sync of a large mailbox takes hours, and the other data sources wait in the same
-/// queue. Nothing is removed before a run got through the whole mailbox, and a removal of a large
-/// part of it waits for the user to agree, cf. MailRemovalGuard.
+/// A run reads a limited number of mails, for a limited time, cf. Share, and leaves the rest to the
+/// next run. The first sync of a large mailbox takes hours, and the other data sources wait in the
+/// same queue. Nothing is removed before a run got through the whole mailbox, and a removal of a
+/// large part of it waits for the user to agree, cf. MailRemovalGuard.
 ///
 /// Signing in is guarded, because every refused attempt brings an account closer to being locked.
 /// A refused sign-in is stored, and no run signs in again until the user saves a new password,
@@ -58,16 +58,6 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
     /// </summary>
     private const int SUMMARY_BATCH_SIZE = 100;
 
-    /// <summary>
-    /// How many mails one run reads at most, whether embedding them works or not.
-    /// </summary>
-    private const int MAX_READ_MAILS_PER_RUN = 500;
-
-    /// <summary>
-    /// How long one run takes at most, before it leaves the rest to the next one.
-    /// </summary>
-    private static readonly TimeSpan MAX_RUN_DURATION = TimeSpan.FromMinutes(5);
-
     /// <summary>
     /// How often every mailbox is synced while AI Studio runs. It goes by the clock, so a computer
     /// which slept longer than this syncs right after waking up.
@@ -106,6 +96,13 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
     /// <inheritdoc />
     public bool Supports(IDataSourceBase dataSource) => dataSource is DataSourceMailbox;
 
+    /// <inheritdoc />
+    /// <remarks>
+    /// Besides the time, a run reads at most 500 mails, whether embedding them works or not.
+    /// Linking a mail the index holds already costs next to nothing, so only the time limits that.
+    /// </remarks>
+    public RunShare Share => new(RunShare.DEFAULT_DURATION, 500);
+
     /// <inheritdoc />
     public async Task<IndexedRunOutcome> ProcessAsync(IndexedRunContext context, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token)
     {
@@ -257,7 +254,7 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
             {
                 foreach (var folder in plannedFolders)
                 {
-                    if (await this.SyncNewMailsAsync(context, mailbox, connector, folder, progress, encounteredKeys, runStartedUtc, token))
+                    if (await this.SyncNewMailsAsync(context, mailbox, connector, folder, progress, encounteredKeys, token))
                         continue;
 
                     workedThrough = false;
@@ -278,7 +275,7 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
             {
                 logger.LogWarning("Syncing mailbox '{MailboxId}' would remove {RemovalCount} of {IndexedCount} location(s) from the index at once. The removal waits for the user to agree.", mailbox.Id, removals.Count, removals.IndexedCount);
                 await context.IndexStore.HoldBackMailRemovalAsync(mailbox.Id, removals.Count, token);
-                await progress.PauseRunAsync(sourceHash, "mailbox sync waits for a removal to be approved", token);
+                await progress.StoreSourceHashAsync(sourceHash, "mailbox sync waits for a removal to be approved", token);
                 progress.PublishRunFailure(GetRemovalMessage(removals.Count), DataSourceAttention.MASS_REMOVAL_PENDING, removals.Count);
                 return IndexedRunOutcome.DONE;
             }
@@ -344,21 +341,16 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
     /// Ends a run which did its share of the work before it got through the mailbox.
     /// </summary>
     /// <remarks>
-    /// Only a run which got something done hands on to another one. When every mail it read
-    /// failed, e.g. because the embedding provider is down, the next run would fail all the same,
-    /// and the one after it, without end. Such a run ends with its failures instead, and waits for
-    /// the next time the mailbox is queued.
+    /// The hash goes along, so the mails indexed so far can be searched right away. Whether another
+    /// run carries on is decided by DocumentRunProgress.PauseRunAsync, the same for every kind of
+    /// data source.
     /// </remarks>
     private async Task<IndexedRunOutcome> PauseAsync(DataSourceMailbox mailbox, DocumentRunProgress progress, string sourceHash, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token)
     {
-        if (progress.IndexedDocuments is 0 && progress.FailedDocuments > 0)
-        {
-            logger.LogWarning("Syncing mailbox '{MailboxId}' stops without carrying on, since none of the {FailedMails} mail(s) it read could be indexed.", mailbox.Id, progress.FailedDocuments);
-            await progress.CompleteRunAsync(sourceHash, "mailbox sync stopped without progress", token);
-            return IndexedRunOutcome.DONE;
-        }
+        var outcome = await progress.PauseRunAsync(sourceHash, "mailbox sync paused", token);
+        if (outcome is not IndexedRunOutcome.MORE_TO_DO)
+            return outcome;
 
-        await progress.PauseRunAsync(sourceHash, "mailbox sync paused", token);
         logger.LogInformation(
             "Syncing mailbox '{MailboxId}' carries on in another run. RefreshMode={RefreshMode}, Embedded={EmbeddedMails}, Failed={FailedMails}, Done={DoneMails}/{TotalMails}.",
             mailbox.Id,
@@ -433,7 +425,7 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
     /// The second pass through one folder: links or indexes the mails the index does not hold there yet.
     /// </summary>
     /// <returns>True when the folder is through, false when the run did its share of the work before.</returns>
-    private async Task<bool> SyncNewMailsAsync(IndexedRunContext context, DataSourceMailbox mailbox, ImapMailboxConnector connector, PlannedFolder folder, DocumentRunProgress progress, ISet<string> encounteredKeys, DateTimeOffset runStartedUtc, CancellationToken token)
+    private async Task<bool> SyncNewMailsAsync(IndexedRunContext context, DataSourceMailbox mailbox, ImapMailboxConnector connector, PlannedFolder folder, DocumentRunProgress progress, ISet<string> encounteredKeys, CancellationToken token)
     {
         if (folder.Plan.NewUids.Count > 0)
         {
@@ -451,13 +443,13 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
             foreach (var uidBatch in folder.Plan.NewUids.Chunk(SUMMARY_BATCH_SIZE))
             {
                 token.ThrowIfCancellationRequested();
-                if (HasDoneItsShare(progress, runStartedUtc))
+                if (progress.HasDoneItsShare(DateTimeOffset.UtcNow))
                     return false;
 
                 var summaries = await connector.FetchSummariesAsync(uidBatch, token);
                 foreach (var summary in summaries.OrderByDescending(summary => summary.UniqueId.Id))
                 {
-                    if (HasDoneItsShare(progress, runStartedUtc))
+                    if (progress.HasDoneItsShare(DateTimeOffset.UtcNow))
                         return false;
 
                     await this.SyncNewMailAsync(context, mailbox, connector, folder.Folder.FullName, summary, progress, encounteredKeys, token);
@@ -479,16 +471,6 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
         return true;
     }
 
-    /// <summary>
-    /// Whether a run did its share of the work, so other data sources get their turn.
-    /// </summary>
-    /// <remarks>
-    /// Counted are the mails which were read, whether embedding them worked or not. Linking a mail
-    /// the index holds already costs next to nothing, so only the time limits that.
-    /// </remarks>
-    private static bool HasDoneItsShare(DocumentRunProgress progress, DateTimeOffset runStartedUtc) =>
-        progress.IndexedDocuments + progress.FailedDocuments >= MAX_READ_MAILS_PER_RUN || DateTimeOffset.UtcNow - runStartedUtc >= MAX_RUN_DURATION;
-
     /// <summary>
     /// Works out what a run which got through the whole mailbox would remove.
     /// </summary>

app/MindWork AI Studio/Tools/Services/Indexing/RunShare.cs

@@ -0,0 +1,20 @@
+namespace AIStudio.Tools.Services.Indexing;
+
+/// <summary>
+/// How much work one indexing run does at most, before it leaves the rest to another run.
+/// </summary>
+/// <remarks>
+/// Only the work on documents counts, cf. DocumentRunProgress.HasDoneItsShare: the clock starts right
+/// before the first one. Whatever a kind of data source does before, e.g. listing the files of a
+/// folder, every run has to do anew, and counting it would leave a large folder little time for its
+/// documents.
+/// </remarks>
+/// <param name="MaxDuration">How long a run works on documents at most. More than zero.</param>
+/// <param name="MaxDocuments">How many documents a run reads at most, whether that works or not. Null when only the time counts.</param>
+internal sealed record RunShare(TimeSpan MaxDuration, int? MaxDocuments = null)
+{
+    /// <summary>
+    /// How long a run works on documents, unless its kind of data source has a reason for another time.
+    /// </summary>
+    public static readonly TimeSpan DEFAULT_DURATION = TimeSpan.FromMinutes(5);
+}