Show what happens to the data sources while they are re-indexed (#979)

app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs

@@ -3,11 +3,18 @@ using AIStudio.Settings;
 namespace AIStudio.Tools;
 
 /// <summary>
-/// Contains both the allowed and selected data sources.
+/// Contains the allowed and selected data sources, plus the ones waiting for their index.
 /// </summary>
 /// <remarks>
 /// The selected data sources are a subset of the allowed data sources.
+///
+/// The data sources waiting for a re-index are deliberately kept apart from the allowed ones rather
+/// than mixed in. Everything reading the allowed list -- the data source selection agent above all
+/// -- takes it to mean "may be used to answer with", and a source whose index is being rebuilt
+/// cannot answer anything. It is listed separately so the user interface can still show it and say
+/// why it is greyed out, instead of letting it vanish without a word.
 /// </remarks>
 /// <param name="AllowedDataSources">The allowed data sources.</param>
 /// <param name="SelectedDataSources">The selected data sources, which are a subset of the allowed data sources.</param>
-public readonly record struct AllowedSelectedDataSources(IReadOnlyList<IDataSource> AllowedDataSources, IReadOnlyList<IDataSource> SelectedDataSources);
+/// <param name="DataSourcesAwaitingReindex">The data sources which passed every check but cannot be searched until their index has been rebuilt.</param>
+public readonly record struct AllowedSelectedDataSources(IReadOnlyList<IDataSource> AllowedDataSources, IReadOnlyList<IDataSource> SelectedDataSources, IReadOnlyList<IDataSource> DataSourcesAwaitingReindex);

app/MindWork AI Studio/Tools/Databases/IndexStore/DataSourceIndexState.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Tools.Databases.IndexStore;
+
+/// <summary>
+/// What the index knows about a data source as a whole, without its files.
+/// </summary>
+/// <remarks>
+/// The manifest answers the same question, but reads every file and every stored failure of the
+/// data source to do so. That is the right thing before a run, and far too much for a question
+/// asked about several data sources every time somebody opens the data source selection.
+///
+/// SourceHash is the telling one: it is written once a run has worked through the whole data
+/// source, and resetting the index deletes the row it lives in. So an empty hash means no run has
+/// finished since the index was last discarded.
+/// </remarks>
+/// <param name="EmbeddingProviderId">The embedding provider the stored vectors were created with.</param>
+/// <param name="EmbeddingSignature">Identifies the embedding configuration the stored vectors belong to.</param>
+/// <param name="SourceHash">The hash of the data source as a whole, written when a run completes.</param>
+/// <param name="VectorSize">The dimension of the stored vectors.</param>
+public sealed record DataSourceIndexState(string EmbeddingProviderId, string EmbeddingSignature, string SourceHash, int VectorSize);

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs

@@ -6,6 +6,14 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
 {
     public abstract Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token);
 
+    /// <summary>
+    /// Reads what the index knows about a data source as a whole, without its files.
+    /// </summary>
+    /// <param name="dataSourceId">The data source to read.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The stored state, or null when the index holds nothing about this data source.</returns>
+    public abstract Task<DataSourceIndexState?> GetDataSourceStateAsync(string dataSourceId, CancellationToken token);
+
     public abstract Task UpsertDataSourceAsync(
         string dataSourceId,
         string dataSourceType,

app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs

@@ -23,8 +23,9 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
         await Task.CompletedTask;
     }
 
-    public override Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token) =>
-        Task.FromResult(new DataSourceEmbeddingManifest());
+    public override Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token) => Task.FromResult(new DataSourceEmbeddingManifest());
+
+    public override Task<DataSourceIndexState?> GetDataSourceStateAsync(string dataSourceId, CancellationToken token) => Task.FromResult<DataSourceIndexState?>(null);
 
     public override Task UpsertDataSourceAsync(
         string dataSourceId,

app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs

@@ -69,6 +69,16 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
         yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
     }
 
+    public override async Task<DataSourceIndexState?> GetDataSourceStateAsync(string dataSourceId, CancellationToken token)
+    {
+        await using var context = this.CreateContext();
+        return await context.DataSources
+            .AsNoTracking()
+            .Where(source => source.DataSourceId == dataSourceId)
+            .Select(source => new DataSourceIndexState(source.EmbeddingProviderId, source.EmbeddingSignature, source.SourceHash, source.VectorSize))
+            .FirstOrDefaultAsync(token);
+    }
+
     public override async Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token)
     {
         await using var context = this.CreateContext();

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -18,6 +18,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 {
     private const int VECTOR_STORE_OPTIMIZATION_CHUNK_THRESHOLD = 100_000;
 
+    /// <summary>
+    /// How often the block progress within one file is reported to the user interface at most.
+    /// </summary>
+    private static readonly TimeSpan BLOCK_PROGRESS_INTERVAL = TimeSpan.FromSeconds(3);
+
+    /// <summary>
+    /// How long the re-index check waits for the index database before it gives up.
+    /// </summary>
+    /// <remarks>
+    /// Asked while somebody waits for the data source selection to open, and possibly while a run
+    /// writes to the same database.
+    /// </remarks>
+    private static readonly TimeSpan REINDEX_CHECK_TIMEOUT = TimeSpan.FromSeconds(2);
+
     private readonly Channel<DataSourceEmbeddingQueueItem> queue = Channel.CreateUnbounded<DataSourceEmbeddingQueueItem>();
     private readonly ConcurrentDictionary<string, byte> queuedIds = new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<string, byte> runningIds = new(StringComparer.OrdinalIgnoreCase);
@@ -206,6 +220,103 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                || manifest.PermanentFailures.Count > 0;
     }
 
+    /// <summary>
+    /// Whether a data source cannot answer a search right now because its index has to be built anew.
+    /// </summary>
+    /// <remarks>
+    /// Says nothing about a data source which is only catching up with a handful of changed files:
+    /// everything indexed before is still there and still searchable. What this catches is the case
+    /// where the whole index was thrown away, or is about to be, because the embedding configuration
+    /// changed under it. Between discarding the old vectors and finishing the new ones, the data
+    /// source looks perfectly fine and finds nothing.
+    ///
+    /// Two things are asked, in this order. The stored signature tells whether the vectors still
+    /// belong to the current configuration; it is written back right after the reset, so on its own
+    /// it would call a rebuild in progress finished. The stored hash of the data source closes that
+    /// gap: it survives an ordinary run but not a reset, so an empty one means no run has completed
+    /// since the index was discarded.
+    ///
+    /// Anything unclear counts as not waiting. Whoever asks does so to grey out a row, and a data
+    /// source wrongly greyed out for good is worse than one which turns out to have nothing to say.
+    /// </remarks>
+    /// <param name="dataSource">The data source to ask about.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the data source is waiting for its index to be rebuilt.</returns>
+    public async Task<bool> IsAwaitingReindexAsync(IDataSource dataSource, CancellationToken token = default)
+    {
+        //
+        // This guard also keeps the index database out of the picture while local RAG is switched
+        // off: asking for the store creates the database and runs its migrations on the first call,
+        // which must not happen because somebody opened the data source selection.
+        //
+        if (!this.IsSupportedInternalDataSource(dataSource))
+            return false;
+
+        if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider))
+            return false;
+
+        try
+        {
+            //
+            // A timeout of its own: this runs while the user waits for a popover to open, and the
+            // embedding service may be writing to the same database at the time.
+            //
+            using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+            timeout.CancelAfter(REINDEX_CHECK_TIMEOUT);
+
+            var indexStore = await databaseClientProvider.GetIndexStoreAsync(timeout.Token);
+            if (!indexStore.IsAvailable)
+                return false;
+
+            var indexState = await indexStore.GetDataSourceStateAsync(dataSource.Id, timeout.Token);
+            var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
+            var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
+            var runState = this.statuses.TryGetValue(dataSource.Id, out var status) ? status.State : (DataSourceEmbeddingState?)null;
+
+            return IsIndexAwaitingRebuild(indexState, embeddingSignature, runState);
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "Could not tell whether data source '{DataSourceName}' ({DataSourceId}) is waiting for a re-index. Treating it as usable.", dataSource.Name, dataSource.Id);
+            return false;
+        }
+    }
+
+    /// <summary>
+    /// Decides from the stored index state alone whether a data source has to be indexed anew.
+    /// </summary>
+    /// <remarks>
+    /// Kept apart from reading the database so the decision itself can be pinned down in a test.
+    /// The order of the three questions is what makes it correct, see IsAwaitingReindexAsync.
+    /// </remarks>
+    /// <param name="indexState">What the index holds about the data source, or null when it holds nothing.</param>
+    /// <param name="currentEmbeddingSignature">The signature the current embedding configuration produces.</param>
+    /// <param name="runState">The state of this data source's last or current run, when one is known.</param>
+    /// <returns>True when the data source is waiting for its index to be rebuilt.</returns>
+    internal static bool IsIndexAwaitingRebuild(DataSourceIndexState? indexState, string currentEmbeddingSignature, DataSourceEmbeddingState? runState)
+    {
+        // Nothing stored at all: this data source has never been indexed, so there is nothing to
+        // search in it yet.
+        if (indexState is null)
+            return true;
+
+        // The stored vectors belong to another embedding configuration. They will be thrown away
+        // as soon as the next run starts, and they are of no use before that either.
+        if (!string.Equals(indexState.EmbeddingSignature, currentEmbeddingSignature, StringComparison.Ordinal))
+            return true;
+
+        // A run has worked through the whole data source since the index was last discarded.
+        if (!string.IsNullOrWhiteSpace(indexState.SourceHash))
+            return false;
+
+        //
+        // The index was discarded and nothing has finished since. A failed run is the exception:
+        // whatever it managed to index is searchable, and the embeddings page already names the
+        // problem, so there is nothing to be gained from locking the row as well.
+        //
+        return runState is not DataSourceEmbeddingState.FAILED;
+    }
+
     public Task QueueDataSourceAsync(IDataSource dataSource)
     {
         return this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK);
@@ -646,6 +757,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
             this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles));
 
+            //
+            // What the page says while one file is being worked on. Without it, a document of
+            // several thousand pages leaves the same sentence standing for hours, and a progress
+            // which never moves cannot be told apart from one which is stuck.
+            //
+            var lastBlockReportUtc = DateTimeOffset.MinValue;
+
             try
             {
                 logger.LogInformation(
@@ -658,7 +776,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                     skippedFiles + completedFiles + 1,
                     totalFiles);
                 var startedAtUtc = DateTimeOffset.UtcNow;
-                var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token);
+                var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, ReportBlockProgress, token);
                 token.ThrowIfCancellationRequested();
                 var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
                 if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal))
@@ -780,6 +898,24 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name);
                 this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, failureMessage, failureDetails, permanentlySkippedFiles));
             }
+
+            continue;
+
+            void ReportBlockProgress(int blockNumber, int? pageNumber)
+            {
+                //
+                // The first block goes out at once, so the line is there instead of blank. After
+                // that, at most one message every BLOCK_PROGRESS_INTERVAL: each one re-renders the
+                // embedding page, the navigation bar and the table in the settings, and the blocks
+                // of a large file arrive far faster than anybody can read them.
+                //
+                var nowUtc = DateTimeOffset.UtcNow;
+                if (blockNumber > 1 && nowUtc - lastBlockReportUtc < BLOCK_PROGRESS_INTERVAL)
+                    return;
+
+                lastBlockReportUtc = nowUtc;
+                this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles, blockNumber, pageNumber));
+            }
         }
 
         manifest.SourceHash = metadataSnapshot.SourceHash;
@@ -823,6 +959,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IProvider provider,
         DataSourceEmbeddingManifest manifest,
         VectorStoreOptimizationTracker optimizationTracker,
+        Action<int, int?> reportBlockProgress,
         CancellationToken token)
     {
         var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
@@ -845,6 +982,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         {
             batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
             totalChunkCount++;
+            reportBlockProgress(totalChunkCount, chunk.PageNumber);
 
             if (batch.Count >= embeddingBatchSize)
                 await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
@@ -1164,6 +1302,31 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG embedding state will be retried during this pass. File watchers will be activated after this check completes.",
             supportedDataSources.Count);
 
+        //
+        // Every data source gets its row before the first run starts. This pass works through them
+        // one after the other, and re-indexing a large source takes its time: without this, the
+        // embeddings page would show the one source being worked on and nothing else, which reads
+        // as if the others were gone rather than waiting their turn. The queueing path does the
+        // same thing when it reserves a slot, which is why it never had this problem.
+        //
+        foreach (var dataSource in supportedDataSources)
+        {
+            if (this.statuses.TryGetValue(dataSource.Id, out var knownStatus) && knownStatus.State is DataSourceEmbeddingState.RUNNING)
+                continue;
+
+            this.statuses[dataSource.Id] = this.CreateStatus(
+                dataSource,
+                DataSourceEmbeddingState.QUEUED,
+                knownStatus?.TotalFiles ?? 0,
+                knownStatus?.IndexedFiles ?? 0,
+                knownStatus?.FailedFiles ?? 0,
+                failures: knownStatus?.Failures ?? [],
+                permanentlySkippedFiles: knownStatus?.PermanentlySkippedFiles ?? 0);
+        }
+
+        // One message for the whole list, rather than one per data source:
+        this.PublishStatusChanged();
+
         foreach (var dataSource in supportedDataSources)
         {
             token.ThrowIfCancellationRequested();
@@ -1418,7 +1581,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         string currentFile = "",
         string lastError = "",
         IReadOnlyList<DataSourceEmbeddingFailure>? failures = null,
-        int permanentlySkippedFiles = 0)
+        int permanentlySkippedFiles = 0,
+        int? currentFileBlock = null,
+        int? currentFilePage = null)
     {
         return new DataSourceEmbeddingStatus(
             dataSource.Id,
@@ -1431,7 +1596,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             currentFile,
             lastError,
             failures?.ToList() ?? [],
-            permanentlySkippedFiles);
+            permanentlySkippedFiles,
+            currentFileBlock,
+            currentFilePage);
     }
 
     /// <remarks>

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs

@@ -3,6 +3,11 @@ using AIStudio.Tools.PluginSystem;
 
 namespace AIStudio.Tools.Services;
 
+/// <remarks>
+/// CurrentFileBlock and CurrentFilePage are null rather than zero while nothing is known about
+/// them: a file which is only about to start has no first block, and not every kind of document
+/// has pages to count. Block numbers start at one, the way the page states them.
+/// </remarks>
 public sealed record DataSourceEmbeddingStatus(
     string DataSourceId,
     string DataSourceName,
@@ -14,7 +19,9 @@ public sealed record DataSourceEmbeddingStatus(
     string CurrentFile,
     string LastError,
     IReadOnlyList<DataSourceEmbeddingFailure> Failures,
-    int PermanentlySkippedFiles = 0)
+    int PermanentlySkippedFiles = 0,
+    int? CurrentFileBlock = null,
+    int? CurrentFilePage = null)
 {
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));
 

app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs

@@ -13,7 +13,7 @@ namespace AIStudio.Tools.Services;
 
 public sealed class DataSourceLocalRetrievalService(
     SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider,
-    ILogger<DataSourceLocalRetrievalService> logger)
+    DataSourceEmbeddingService embeddingService, ILogger<DataSourceLocalRetrievalService> logger)
 {
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceLocalRetrievalService).Namespace, nameof(DataSourceLocalRetrievalService));
 
@@ -73,6 +73,23 @@ public sealed class DataSourceLocalRetrievalService(
         if (maxMatches == 0)
             return [];
 
+        //
+        // A data source waiting for its index is kept out of the selection before the RAG process
+        // starts. This catches whatever reaches retrieval another way, and turns an answer quietly
+        // put together without the data into a sentence saying so.
+        //
+        // Asked here rather than inside one of the two channels below, because both of them read
+        // what the rebuild is about to discard: with only the embedding signature changed, the old
+        // chunks are still in place and the keyword search would happily answer from them while
+        // the vector search finds nothing.
+        //
+        if (await embeddingService.IsAwaitingReindexAsync(dataSource, token))
+        {
+            logger.LogWarning("Skipping local retrieval for data source '{DataSourceName}' ({DataSourceId}) because its index has to be built anew.", dataSource.Name, dataSource.Id);
+            await this.ReportRetrievalGapAsync(dataSource, "index-rebuilding", string.Format(TB("The data source '{0}' was left out of the answer: it is being indexed again and cannot be searched until that is finished."), dataSource.Name));
+            return [];
+        }
+
         var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
         var vectorTask = this.SearchVectorAsync(dataSource, query, maxMatches, collectionName, token);
         var bm25Task = this.SearchBm25Async(dataSource, query, maxMatches, token);

app/MindWork AI Studio/Tools/Services/DataSourceService.cs

@@ -18,16 +18,18 @@ public sealed class DataSourceService
     // ReSharper disable once NotAccessedPositionalProperty.Local
     private readonly record struct ParticipatingProvider(string Role, bool IsTrusted, ConfidenceLevel ConfidenceLevel);
 
+    private readonly DataSourceEmbeddingService embeddingService;
     private readonly RustService rustService;
     private readonly SettingsManager settingsManager;
     private readonly ILogger<DataSourceService> logger;
 
-    public DataSourceService(SettingsManager settingsManager, ILogger<DataSourceService> logger, RustService rustService)
+    public DataSourceService(SettingsManager settingsManager, ILogger<DataSourceService> logger, RustService rustService, DataSourceEmbeddingService embeddingService)
     {
         this.logger = logger;
         this.rustService = rustService;
         this.settingsManager = settingsManager;
-        
+        this.embeddingService = embeddingService;
+
         this.logger.LogInformation("The data source service has been initialized.");
     }
     
@@ -49,7 +51,7 @@ public sealed class DataSourceService
         if (selectedLLMProvider == Settings.Provider.NONE)
         {
             this.logger.LogWarning("The selected LLM provider is not set. We cannot filter the data sources by any means.");
-            return new([], []);
+            return new([], [], []);
         }
         
         var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
@@ -78,7 +80,15 @@ public sealed class DataSourceService
         var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
         var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.Id, dataSourceOptions,
             new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager)));
-        return await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, requestedDataSources);
+        var allowedDataSources = await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, requestedDataSources);
+
+        //
+        // Whoever asks this way has no list to show, so a data source waiting for its index is
+        // dropped rather than marked. Handing it back would start a chat with a data source which
+        // finds nothing -- the very thing being greyed out elsewhere is meant to prevent.
+        //
+        var awaitingReindexIds = (await this.GetDataSourcesAwaitingReindex(allowedDataSources)).Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
+        return allowedDataSources.Where(source => !awaitingReindexIds.Contains(source.Id)).ToList();
     }
     
     /// <summary>
@@ -99,7 +109,7 @@ public sealed class DataSourceService
         if (selectedLLMProvider is NoProvider)
         {
             this.logger.LogWarning("The selected LLM provider is the default provider. We cannot filter the data sources by any means.");
-            return new([], []);
+            return new([], [], []);
         }
         
         var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
@@ -142,9 +152,47 @@ public sealed class DataSourceService
         var allDataSources = this.settingsManager.ConfigurationData.DataSources.ToList();
         var previousSelectedDataSourceIds = previousSelectedDataSources?.Select(source => source.Id).ToHashSet(StringComparer.Ordinal) ?? [];
         var filteredDataSources = await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, allDataSources);
-        var filteredSelectedDataSources = filteredDataSources.Where(source => previousSelectedDataSourceIds.Contains(source.Id)).ToList();
-        
-        return new(filteredDataSources, filteredSelectedDataSources);
+
+        //
+        // Which of the sources that passed every check cannot answer a search right now. They are
+        // held back from both lists below rather than removed altogether: a source whose index is
+        // being rebuilt is usable again in a while, and saying so on its own row beats letting it
+        // disappear from the selection without a word.
+        //
+        var awaitingReindex = await this.GetDataSourcesAwaitingReindex(filteredDataSources);
+        var awaitingReindexIds = awaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
+        var usableDataSources = filteredDataSources.Where(source => !awaitingReindexIds.Contains(source.Id)).ToList();
+        var filteredSelectedDataSources = usableDataSources.Where(source => previousSelectedDataSourceIds.Contains(source.Id)).ToList();
+
+        return new(usableDataSources, filteredSelectedDataSources, awaitingReindex);
+    }
+
+    /// <summary>
+    /// Picks out the data sources whose index has to be rebuilt before they can be searched.
+    /// </summary>
+    /// <remarks>
+    /// Asked for every data source at once, the same way the checks above run in parallel. Each
+    /// answer is a single row read from the index database, and anything unclear counts as usable.
+    /// </remarks>
+    /// <param name="dataSources">The data sources which passed every other check.</param>
+    /// <returns>Those of them which are waiting for their index, in the order they came in.</returns>
+    private async Task<IReadOnlyList<IDataSource>> GetDataSourcesAwaitingReindex(IReadOnlyList<IDataSource> dataSources)
+    {
+        var checks = new List<Task<bool>>(dataSources.Count);
+        foreach (var dataSource in dataSources)
+            checks.Add(this.embeddingService.IsAwaitingReindexAsync(dataSource));
+
+        var awaitingReindex = new List<IDataSource>();
+        for (var index = 0; index < dataSources.Count; index++)
+        {
+            if (await checks[index])
+            {
+                this.logger.LogInformation("The data source '{DataSourceName}' ({DataSourceId}) is waiting for its index to be rebuilt. It is shown, but cannot be selected.", dataSources[index].Name, dataSources[index].Id);
+                awaitingReindex.Add(dataSources[index]);
+            }
+        }
+
+        return awaitingReindex;
     }
 
     private async Task<IReadOnlyList<IDataSource>> GetAllowedDataSources(bool usingTrustedProvider, IReadOnlyList<ParticipatingProvider> participatingProviders, IReadOnlyCollection<IDataSource> requestedDataSources)