diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreUnreadableException.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreUnreadableException.cs new file mode 100644 index 00000000..8829a9dd --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreUnreadableException.cs @@ -0,0 +1,13 @@ +namespace AIStudio.Tools.Databases.VectorStore; + +/// +/// Thrown when a vector store is there on disk, but cannot be opened. +/// +/// +/// Separate from every other database failure, because it is the one which no retry heals and which +/// the app must not heal on its own: building the index anew sends every document to the embedding +/// provider once more, which costs real money and, for a large data source, hours. So this failure +/// travels as its own type up to the places which can say so and offer the rebuild, and the decision +/// stays with the user. +/// +public sealed class VectorStoreUnreadableException(string message) : Exception(message); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 1f32eb14..b85ee951 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -430,6 +430,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM { break; } + catch (VectorStoreUnreadableException exception) when (dataSource is not null) + { + // + // Nothing is deleted and nothing is rebuilt here. The data source says what is + // wrong with it, stays out of the selection while it says so, and waits for the + // user to ask for the repair. + // + logger.LogError( + exception, + "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.", + dataSource.Name, + dataSource.Id); + this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource)); + } catch (Exception exception) { if (dataSource is null) @@ -873,6 +887,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM ShortHash(fingerprint)); this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles)); } + catch (VectorStoreUnreadableException) + { + // + // Not about this one file: the store of the whole data source cannot be opened, so + // every remaining file would fail the same way. Carrying on would fill the list + // with one entry per file and hide the single cause behind them. + // + throw; + } catch (Exception exception) { // @@ -1338,6 +1361,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM { throw; } + catch (VectorStoreUnreadableException exception) + { + logger.LogError( + exception, + "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.", + dataSource.Name, + dataSource.Id); + this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource)); + } catch (Exception exception) { logger.LogError(exception, "Initial embedding hash check failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); @@ -1583,7 +1615,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM IReadOnlyList? failures = null, int permanentlySkippedFiles = 0, int? currentFileBlock = null, - int? currentFilePage = null) + int? currentFilePage = null, + bool vectorStoreUnreadable = false) { return new DataSourceEmbeddingStatus( dataSource.Id, @@ -1598,7 +1631,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM failures?.ToList() ?? [], permanentlySkippedFiles, currentFileBlock, - currentFilePage); + currentFilePage, + vectorStoreUnreadable); } /// @@ -1635,6 +1669,25 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage, DateTimeOffset.UtcNow)]); } + /// + /// Deliberately not the message which came from the runtime: that one names a store name and a + /// path, is written in English for the log file, and says nothing about what happens next. What + /// the user needs to read is what this means for their chats and where the way out is. + /// + private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSource dataSource) + { + var errorMessage = string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name); + return this.CreateStatus( + dataSource, + DataSourceEmbeddingState.FAILED, + 0, + 0, + 1, + lastError: errorMessage, + failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage, DateTimeOffset.UtcNow)], + vectorStoreUnreadable: true); + } + private DataSourceQueueRequestResult TryReserveDataSourceQueueSlot(string dataSourceId, bool queueAfterCurrentRun) { lock (this.queueStateLock) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs index 4627f716..423fc7e9 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs @@ -7,6 +7,10 @@ namespace AIStudio.Tools.Services; /// CurrentFileBlock and CurrentFilePage are null rather than zero while nothing is known about /// them: a file which is only about to start has no first block, and not every kind of document /// has pages to count. Block numbers start at one, the way the page states them. +/// +/// VectorStoreUnreadable says why a data source failed, not only that it did. The UI needs that +/// difference to offer the repair for this one case, and it is carried as its own flag so nothing +/// has to read it back out of the message in LastError. /// public sealed record DataSourceEmbeddingStatus( string DataSourceId, @@ -21,7 +25,8 @@ public sealed record DataSourceEmbeddingStatus( IReadOnlyList Failures, int PermanentlySkippedFiles = 0, int? CurrentFileBlock = null, - int? CurrentFilePage = null) + int? CurrentFilePage = null, + bool VectorStoreUnreadable = false) { private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus)); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index efa318e7..56890389 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -180,6 +180,17 @@ public sealed class DataSourceLocalRetrievalService( await this.ReportRetrievalGapAsync(dataSource, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage)); return []; } + catch (VectorStoreUnreadableException exception) + { + // + // Its own gap key, because this is not a search which went wrong but an index which has + // to be built anew. Saying that once per session is what turns a silently shortened + // answer into one the user can do something about. + // + logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because its vector store cannot be read.", dataSource.Name, dataSource.Id); + await this.ReportRetrievalGapAsync(dataSource, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it on the background embeddings page."), dataSource.Name)); + return []; + } catch (Exception exception) { logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Databases.cs b/app/MindWork AI Studio/Tools/Services/RustService.Databases.cs index d4e8bf0f..406861b8 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Databases.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Databases.cs @@ -1,7 +1,18 @@ +using AIStudio.Tools.Databases.VectorStore; + namespace AIStudio.Tools.Services; public sealed partial class RustService { + /// + /// The issue code the Rust runtime sends when a vector store is there, but cannot be opened. + /// + /// + /// Mirrors ISSUE_CODE_STORE_UNREADABLE in runtime/src/qdrant_edge_database.rs. Reading the code + /// rather than the message is what keeps a reworded message on the Rust side harmless here. + /// + private const string ISSUE_CODE_STORE_UNREADABLE = "store-unreadable"; + public async Task GetDatabaseInfo( string databaseName, string infoPath, @@ -46,7 +57,7 @@ public sealed partial class RustService var operation = await response.Content.ReadFromJsonAsync(this.jsonRustSerializerOptions, cts.Token); if (operation is not { Success: true }) - throw new InvalidOperationException(operation?.Issue ?? $"The {databaseName} operation failed."); + throw CreateDatabaseException(operation?.Issue, operation?.IssueCode, $"The {databaseName} operation failed."); } public async Task ExecuteDatabaseQuery(string databaseName, string path, TRequest request, CancellationToken cancellationToken = default) @@ -59,12 +70,31 @@ public sealed partial class RustService var operation = await response.Content.ReadFromJsonAsync>(this.jsonRustSerializerOptions, cts.Token); if (operation is not { Success: true }) - throw new InvalidOperationException(operation?.Issue ?? $"The {databaseName} query failed."); + throw CreateDatabaseException(operation?.Issue, operation?.IssueCode, $"The {databaseName} query failed."); return operation.Data; } - private sealed record DatabaseOperationResponse(bool Success, string Issue); + /// + /// Turns a failed database response into the exception which fits its issue code. + /// + /// + /// Almost every failure says all it has to say in its message. A store which cannot be opened is + /// the exception: the only way out of it is a rebuild which costs the user money and time, so it + /// gets a type of its own and reaches the places which can offer that rebuild instead of + /// starting it unasked. + /// + private static Exception CreateDatabaseException(string? issue, string? issueCode, string fallbackMessage) + { + var message = string.IsNullOrWhiteSpace(issue) ? fallbackMessage : issue; + return issueCode switch + { + ISSUE_CODE_STORE_UNREADABLE => new VectorStoreUnreadableException(message), + _ => new InvalidOperationException(message), + }; + } - private sealed record DatabaseQueryResponse(bool Success, string Issue, TResult? Data); + private sealed record DatabaseOperationResponse(bool Success, string Issue, string IssueCode); + + private sealed record DatabaseQueryResponse(bool Success, string Issue, string IssueCode, TResult? Data); }