Upgrade Qdrant Edge to 0.8.0 and make unreadable vector stores repairable (#982)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -317,6 +317,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         return runState is not DataSourceEmbeddingState.FAILED;
     }
 
+    /// <summary>
+    /// Whether a data source cannot be searched because its vector store cannot be read anymore.
+    /// </summary>
+    /// <remarks>
+    /// Unlike the re-index check above, this reads no database at all: the state comes from the run
+    /// or the search which ran into the unreadable store, and is kept in memory only. That it does
+    /// not survive a restart is deliberate. The very same store may well open on the next start,
+    /// and a mark written to disk would then be wrong with nobody noticing. Until something touches
+    /// the store again, the data source counts as usable, and a failing search says so on its own.
+    /// </remarks>
+    /// <param name="dataSource">The data source to ask about.</param>
+    /// <returns>True when the data source waits for the user to have its index rebuilt.</returns>
+    public bool NeedsIndexRepair(IDataSource dataSource) =>
+        this.statuses.TryGetValue(dataSource.Id, out var status) &&
+        status is { State: DataSourceEmbeddingState.FAILED, VectorStoreUnreadable: true };
+
     public Task QueueDataSourceAsync(IDataSource dataSource)
     {
         return this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK);
@@ -430,6 +446,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             {
                 break;
             }
+            catch (VectorStoreUnreadableException exception) when (dataSource is not null)
+            {
+                //
+                // Nothing is deleted and nothing is rebuilt here. The data source says what is
+                // wrong with it, stays out of the selection while it says so, and waits for the
+                // user to ask for the repair.
+                //
+                logger.LogError(
+                    exception,
+                    "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.",
+                    dataSource.Name,
+                    dataSource.Id);
+                this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource));
+            }
             catch (Exception exception)
             {
                 if (dataSource is null)
@@ -873,6 +903,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                     ShortHash(fingerprint));
                 this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles));
             }
+            catch (VectorStoreUnreadableException)
+            {
+                //
+                // Not about this one file: the store of the whole data source cannot be opened, so
+                // every remaining file would fail the same way. Carrying on would fill the list
+                // with one entry per file and hide the single cause behind them.
+                //
+                throw;
+            }
             catch (Exception exception)
             {
                 //
@@ -1338,6 +1377,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             {
                 throw;
             }
+            catch (VectorStoreUnreadableException exception)
+            {
+                logger.LogError(
+                    exception,
+                    "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.",
+                    dataSource.Name,
+                    dataSource.Id);
+                this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource));
+            }
             catch (Exception exception)
             {
                 logger.LogError(exception, "Initial embedding hash check failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
@@ -1583,7 +1631,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IReadOnlyList<DataSourceEmbeddingFailure>? failures = null,
         int permanentlySkippedFiles = 0,
         int? currentFileBlock = null,
-        int? currentFilePage = null)
+        int? currentFilePage = null,
+        bool vectorStoreUnreadable = false)
     {
         return new DataSourceEmbeddingStatus(
             dataSource.Id,
@@ -1598,7 +1647,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             failures?.ToList() ?? [],
             permanentlySkippedFiles,
             currentFileBlock,
-            currentFilePage);
+            currentFilePage,
+            vectorStoreUnreadable);
     }
 
     /// <remarks>
@@ -1635,6 +1685,25 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage, DateTimeOffset.UtcNow)]);
     }
 
+    /// <remarks>
+    /// Deliberately not the message which came from the runtime: that one names a store name and a
+    /// path, is written in English for the log file, and says nothing about what happens next. What
+    /// the user needs to read is what this means for their chats and where the way out is.
+    /// </remarks>
+    private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSource dataSource)
+    {
+        var errorMessage = string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name);
+        return this.CreateStatus(
+            dataSource,
+            DataSourceEmbeddingState.FAILED,
+            0,
+            0,
+            1,
+            lastError: errorMessage,
+            failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage, DateTimeOffset.UtcNow)],
+            vectorStoreUnreadable: true);
+    }
+
     private DataSourceQueueRequestResult TryReserveDataSourceQueueSlot(string dataSourceId, bool queueAfterCurrentRun)
     {
         lock (this.queueStateLock)