Ask before local data sources are indexed again (#983)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -199,16 +199,44 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             this.CanRefreshDataSource(dataSource);
     }
 
-    public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default)
+    /// <summary>
+    /// Whether the file or folder a data source reads must stay as it is.
+    /// </summary>
+    /// <remarks>
+    /// Locked as soon as the index holds anything, because where a data source reads from is what it
+    /// is: another folder is another data source, and the path reaches no signature, so swapping it
+    /// would leave the stored index describing documents nobody points at any more.
+    ///
+    /// The embedding provider used to be locked along with it and no longer is. It does reach the
+    /// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks
+    /// before it does. Locking it as well left a data source whose provider was deleted stuck on
+    /// keyword search for good, with no way back.
+    ///
+    /// Unclear counts as locked: an unavailable index database says nothing about what is stored.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source to ask about.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the source must not be changed.</returns>
+    public async Task<bool> ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default)
     {
         var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
         if (!indexStore.IsAvailable)
         {
-            logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
+            logger.LogWarning("Locking the source of data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
             return true;
         }
 
         var manifest = await indexStore.GetManifestAsync(dataSourceId, token);
+        return HasStoredIndexState(manifest);
+    }
+
+    /// <summary>
+    /// Whether the index holds anything at all about a data source.
+    /// </summary>
+    /// <param name="manifest">What the index store returned for it.</param>
+    /// <returns>True when there is stored index state.</returns>
+    private static bool HasStoredIndexState(DataSourceEmbeddingManifest manifest)
+    {
         return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId)
                || !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature)
                || !string.IsNullOrWhiteSpace(manifest.SourceHash)
@@ -220,6 +248,59 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                || manifest.PermanentFailures.Count > 0;
     }
 
+    /// <summary>
+    /// Picks the data sources which already hold something in the index.
+    /// </summary>
+    /// <remarks>
+    /// Asked before a setting is saved which would throw those indexes away, so the question can be
+    /// put to the user with the names in it. Anything unclear counts as holding something — the
+    /// opposite of IsAwaitingReindexAsync, and for the opposite reason: there, a wrongly greyed-out
+    /// row would stay wrong for good, while a question asked once too often costs a click, and one
+    /// skipped costs whatever a cloud provider charges for embedding everything again.
+    /// </remarks>
+    /// <param name="dataSources">The data sources to ask about.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>Those of them which have stored index state.</returns>
+    public async Task<IReadOnlyList<IDataSource>> GetDataSourcesWithStoredIndexAsync(IReadOnlyCollection<IDataSource> dataSources, CancellationToken token = default)
+    {
+        //
+        // Filtering first also keeps the index database from being created while local RAG is off:
+        // asking for the store runs its migrations on the first call, which must not happen because
+        // somebody opened a dialog.
+        //
+        var candidates = dataSources.Where(this.IsSupportedInternalDataSource).ToList();
+        if (candidates.Count == 0)
+            return [];
+
+        try
+        {
+            using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+            timeout.CancelAfter(REINDEX_CHECK_TIMEOUT);
+
+            var indexStore = await databaseClientProvider.GetIndexStoreAsync(timeout.Token);
+            if (!indexStore.IsAvailable)
+            {
+                logger.LogWarning("Could not tell which data sources hold a stored index because the local RAG index database '{DatabaseName}' is unavailable. Treating all {DataSourceCount} of them as affected.", indexStore.Name, candidates.Count);
+                return candidates;
+            }
+
+            var affected = new List<IDataSource>(candidates.Count);
+            foreach (var dataSource in candidates)
+            {
+                var manifest = await indexStore.GetManifestAsync(dataSource.Id, timeout.Token);
+                if (HasStoredIndexState(manifest))
+                    affected.Add(dataSource);
+            }
+
+            return affected;
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "Could not tell which of {DataSourceCount} data source(s) hold a stored index. Treating all of them as affected.", candidates.Count);
+            return candidates;
+        }
+    }
+
     /// <summary>
     /// Whether a data source cannot answer a search right now because its index has to be built anew.
     /// </summary>
@@ -269,7 +350,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 return false;
 
             var indexState = await indexStore.GetDataSourceStateAsync(dataSource.Id, timeout.Token);
-            var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
+            var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider);
             var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
             var runState = this.statuses.TryGetValue(dataSource.Id, out var status) ? status.State : (DataSourceEmbeddingState?)null;
 
@@ -1444,7 +1525,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IndexStoreClient indexStore,
         CancellationToken token)
     {
-        var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
+        var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider);
         var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
         var manifest = await indexStore.GetManifestAsync(dataSource.Id, token);