diff --git a/app/MindWork AI Studio/Components/DataSourceBlockReason.cs b/app/MindWork AI Studio/Components/DataSourceBlockReason.cs new file mode 100644 index 00000000..496e1c77 --- /dev/null +++ b/app/MindWork AI Studio/Components/DataSourceBlockReason.cs @@ -0,0 +1,29 @@ +namespace AIStudio.Components; + +/// +/// Why a data source is listed in the selection, but cannot be picked. +/// +/// +/// A reason rather than a yes or no, because the row has to say something different for each of +/// them: one asks the user to wait, the other one asks them to act. Asking somebody to wait for +/// something which will never happen on its own is the worse of the two mistakes. +/// +public enum DataSourceBlockReason +{ + /// + /// Nothing is in the way, the data source can be picked. + /// + NONE, + + /// + /// The index has to be built anew before this data source can answer a search. This passes by + /// itself, as soon as the background indexing has worked through the data source. + /// + AWAITING_REINDEX, + + /// + /// The index cannot be read anymore. This does not pass by itself: only the user can start the + /// rebuild, because it sends every document to the embedding provider once more. + /// + NEEDS_REPAIR, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor b/app/MindWork AI Studio/Components/DataSourceSelection.razor index 4a145120..11001174 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelection.razor +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor @@ -92,7 +92,7 @@ @foreach (var source in this.GetListedDataSources()) { - + } @@ -104,7 +104,7 @@ @foreach (var source in this.GetListedDataSources()) { - + } diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs b/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs index b2f7b8ce..f1e619c0 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs @@ -51,6 +51,8 @@ public partial class DataSourceSelection : MSGComponentBase private IReadOnlyList availableDataSources = []; private IReadOnlyList dataSourcesAwaitingReindex = []; private HashSet dataSourceIdsAwaitingReindex = new(StringComparer.Ordinal); + private IReadOnlyList dataSourcesNeedingRepair = []; + private HashSet dataSourceIdsNeedingRepair = new(StringComparer.Ordinal); private IReadOnlyCollection selectedDataSources = []; private bool aiBasedSourceSelection; private bool aiBasedValidation; @@ -230,15 +232,33 @@ public partial class DataSourceSelection : MSGComponentBase this.availableDataSources = sources.AllowedDataSources; this.dataSourcesAwaitingReindex = sources.DataSourcesAwaitingReindex; this.dataSourceIdsAwaitingReindex = sources.DataSourcesAwaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal); + this.dataSourcesNeedingRepair = sources.DataSourcesNeedingRepair; + this.dataSourceIdsNeedingRepair = sources.DataSourcesNeedingRepair.Select(source => source.Id).ToHashSet(StringComparer.Ordinal); this.selectedDataSources = sources.SelectedDataSources; this.waitingForDataSources = false; this.StateHasChanged(); } - private bool IsAwaitingReindex(IDataSource dataSource) => this.dataSourceIdsAwaitingReindex.Contains(dataSource.Id); + /// + /// Why a data source is listed but cannot be picked, if it cannot. + /// + /// + /// The repair is asked about first. The service hands a data source to one of the two lists + /// only, but should that ever change, the reason the user can act on is the one worth showing. + /// + private DataSourceBlockReason GetBlockReason(IDataSource dataSource) + { + if (this.dataSourceIdsNeedingRepair.Contains(dataSource.Id)) + return DataSourceBlockReason.NEEDS_REPAIR; + + if (this.dataSourceIdsAwaitingReindex.Contains(dataSource.Id)) + return DataSourceBlockReason.AWAITING_REINDEX; + + return DataSourceBlockReason.NONE; + } /// - /// The data sources the list shows: the usable ones, plus the ones waiting for their index. + /// The data sources the list shows: the usable ones, plus the ones which cannot be searched. /// /// /// Kept in the order the data sources were configured in, rather than usable ones first. A row @@ -247,11 +267,12 @@ public partial class DataSourceSelection : MSGComponentBase /// private IReadOnlyList GetListedDataSources() { - if (this.dataSourcesAwaitingReindex.Count == 0) + if (this.dataSourcesAwaitingReindex.Count == 0 && this.dataSourcesNeedingRepair.Count == 0) return this.availableDataSources; var listedIds = this.availableDataSources.Select(source => source.Id).ToHashSet(StringComparer.Ordinal); listedIds.UnionWith(this.dataSourceIdsAwaitingReindex); + listedIds.UnionWith(this.dataSourceIdsNeedingRepair); return this.GetConfiguredDataSourcesSnapshot().Where(source => listedIds.Contains(source.Id)).ToList(); } @@ -259,11 +280,11 @@ public partial class DataSourceSelection : MSGComponentBase /// The preselected but unusable data sources the warning box lists. /// /// - /// The ones waiting for their index are left out: they have a row of their own in the list + /// The ones which are only blocked are left out: they have a row of their own in the list /// above, which says the same thing in the place the user is already looking. /// private IReadOnlyList GetUnavailablePreselectedDataSourcesToList() => - this.GetUnavailablePreselectedDataSources().Where(source => !this.IsAwaitingReindex(source)).ToList(); + this.GetUnavailablePreselectedDataSources().Where(source => this.GetBlockReason(source) is DataSourceBlockReason.NONE).ToList(); private async Task EnabledChanged(bool state) { diff --git a/app/MindWork AI Studio/Components/DataSourceSelectionRow.razor b/app/MindWork AI Studio/Components/DataSourceSelectionRow.razor index 837a1681..d31184fc 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelectionRow.razor +++ b/app/MindWork AI Studio/Components/DataSourceSelectionRow.razor @@ -2,8 +2,8 @@ @using AIStudio.Provider @inherits MSGComponentBase - - + + @this.DataSource.Name @@ -11,9 +11,9 @@ @if (this.DataSource is IInternalDataSource internalSource) { - @if (this.IsAwaitingReindex) + @if (this.IsBlocked) { - + } diff --git a/app/MindWork AI Studio/Components/DataSourceSelectionRow.razor.cs b/app/MindWork AI Studio/Components/DataSourceSelectionRow.razor.cs index 51f602a4..8560c289 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelectionRow.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceSelectionRow.razor.cs @@ -10,9 +10,13 @@ namespace AIStudio.Components; /// can be used right now. /// /// -/// A data source waiting for its index stays in the list instead of disappearing from it, but -/// cannot be picked, and the tooltip says why. The tool selection next to it in the chat answers -/// the same question the same way. +/// A data source which cannot be used stays in the list instead of disappearing from it, but cannot +/// be picked, and the tooltip says why. The tool selection next to it in the chat answers the same +/// question the same way. +/// +/// Why it cannot be used decides what the row says and which icon it wears: an index being built +/// anew is a matter of waiting, an index which cannot be read is a matter of acting. Both are the +/// same row otherwise, which is why this is one component with a reason rather than two components. /// /// The tooltip sits around the list item rather than inside it: a disabled item has its pointer /// events switched off and would swallow the hover. @@ -26,10 +30,31 @@ public partial class DataSourceSelectionRow : MSGComponentBase public required IDataSource DataSource { get; set; } /// - /// Whether this data source has to be indexed anew before it can answer a search. + /// Why this data source cannot be picked right now, if it cannot. /// [Parameter] - public bool IsAwaitingReindex { get; set; } + public DataSourceBlockReason BlockReason { get; set; } = DataSourceBlockReason.NONE; + + private bool IsBlocked => this.BlockReason is not DataSourceBlockReason.NONE; + + private string GetBlockedTooltip() => this.BlockReason switch + { + DataSourceBlockReason.AWAITING_REINDEX => T("This data source is waiting to be indexed again. Until that is finished, it cannot be searched."), + DataSourceBlockReason.NEEDS_REPAIR => T("The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there."), + _ => string.Empty, + }; + + private string GetBlockedIcon() => this.BlockReason switch + { + DataSourceBlockReason.NEEDS_REPAIR => Icons.Material.Filled.ReportProblem, + _ => Icons.Material.Filled.HourglassTop, + }; + + private Color GetBlockedIconColor() => this.BlockReason switch + { + DataSourceBlockReason.NEEDS_REPAIR => Color.Error, + _ => Color.Warning, + }; private string GetConfidenceIconStyle(IInternalDataSource dataSource) => $"{dataSource.ConfidenceLevel.SetColorStyle(this.SettingsManager)} flex-shrink: 0;"; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs b/app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs index b2cf5d9c..ac932e37 100644 --- a/app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs +++ b/app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs @@ -3,7 +3,7 @@ using AIStudio.Settings; namespace AIStudio.Tools; /// -/// Contains the allowed and selected data sources, plus the ones waiting for their index. +/// Contains the allowed and selected data sources, plus the ones which cannot be searched right now. /// /// /// The selected data sources are a subset of the allowed data sources. @@ -13,8 +13,13 @@ namespace AIStudio.Tools; /// -- takes it to mean "may be used to answer with", and a source whose index is being rebuilt /// cannot answer anything. It is listed separately so the user interface can still show it and say /// why it is greyed out, instead of letting it vanish without a word. +/// +/// The same holds for the ones waiting for a repair, and they are a list of their own because the +/// two reasons call for different words: one passes by itself, the other one waits for the user. +/// A data source is in at most one of the two lists. /// /// The allowed data sources. /// The selected data sources, which are a subset of the allowed data sources. /// The data sources which passed every check but cannot be searched until their index has been rebuilt. -public readonly record struct AllowedSelectedDataSources(IReadOnlyList AllowedDataSources, IReadOnlyList SelectedDataSources, IReadOnlyList DataSourcesAwaitingReindex); \ No newline at end of file +/// The data sources which passed every check but whose index cannot be read anymore, so that only the user can get them back. +public readonly record struct AllowedSelectedDataSources(IReadOnlyList AllowedDataSources, IReadOnlyList SelectedDataSources, IReadOnlyList DataSourcesAwaitingReindex, IReadOnlyList DataSourcesNeedingRepair); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index b85ee951..01ea7b7c 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -317,6 +317,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return runState is not DataSourceEmbeddingState.FAILED; } + /// + /// Whether a data source cannot be searched because its vector store cannot be read anymore. + /// + /// + /// Unlike the re-index check above, this reads no database at all: the state comes from the run + /// or the search which ran into the unreadable store, and is kept in memory only. That it does + /// not survive a restart is deliberate. The very same store may well open on the next start, + /// and a mark written to disk would then be wrong with nobody noticing. Until something touches + /// the store again, the data source counts as usable, and a failing search says so on its own. + /// + /// The data source to ask about. + /// True when the data source waits for the user to have its index rebuilt. + public bool NeedsIndexRepair(IDataSource dataSource) => + this.statuses.TryGetValue(dataSource.Id, out var status) && + status is { State: DataSourceEmbeddingState.FAILED, VectorStoreUnreadable: true }; + public Task QueueDataSourceAsync(IDataSource dataSource) { return this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index 56890389..744c67a4 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -188,7 +188,7 @@ public sealed class DataSourceLocalRetrievalService( // answer into one the user can do something about. // logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because its vector store cannot be read.", dataSource.Name, dataSource.Id); - await this.ReportRetrievalGapAsync(dataSource, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it on the background embeddings page."), dataSource.Name)); + await this.ReportRetrievalGapAsync(dataSource, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."), dataSource.Name)); return []; } catch (Exception exception) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceService.cs index 8c70e82b..ce18b3b6 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceService.cs @@ -51,7 +51,7 @@ public sealed class DataSourceService if (selectedLLMProvider == Settings.Provider.NONE) { this.logger.LogWarning("The selected LLM provider is not set. We cannot filter the data sources by any means."); - return new([], [], []); + return new([], [], [], []); } var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager); @@ -83,12 +83,13 @@ public sealed class DataSourceService var allowedDataSources = await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, requestedDataSources); // - // Whoever asks this way has no list to show, so a data source waiting for its index is + // Whoever asks this way has no list to show, so a data source which cannot be searched is // dropped rather than marked. Handing it back would start a chat with a data source which // finds nothing -- the very thing being greyed out elsewhere is meant to prevent. // - var awaitingReindexIds = (await this.GetDataSourcesAwaitingReindex(allowedDataSources)).Select(source => source.Id).ToHashSet(StringComparer.Ordinal); - return allowedDataSources.Where(source => !awaitingReindexIds.Contains(source.Id)).ToList(); + var unsearchableIds = (await this.GetDataSourcesAwaitingReindex(allowedDataSources)).Select(source => source.Id).ToHashSet(StringComparer.Ordinal); + unsearchableIds.UnionWith(this.GetDataSourcesNeedingRepair(allowedDataSources).Select(source => source.Id)); + return allowedDataSources.Where(source => !unsearchableIds.Contains(source.Id)).ToList(); } /// @@ -109,7 +110,7 @@ public sealed class DataSourceService if (selectedLLMProvider is NoProvider) { this.logger.LogWarning("The selected LLM provider is the default provider. We cannot filter the data sources by any means."); - return new([], [], []); + return new([], [], [], []); } var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager); @@ -159,12 +160,21 @@ public sealed class DataSourceService // being rebuilt is usable again in a while, and saying so on its own row beats letting it // disappear from the selection without a word. // - var awaitingReindex = await this.GetDataSourcesAwaitingReindex(filteredDataSources); - var awaitingReindexIds = awaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal); - var usableDataSources = filteredDataSources.Where(source => !awaitingReindexIds.Contains(source.Id)).ToList(); + // A source whose index cannot be read is asked about first and then kept out of the other + // list: both reasons can be true at once, and of the two it is the only one the user can do + // anything about. Telling them to wait instead would be telling them to wait forever. + // + var needingRepair = this.GetDataSourcesNeedingRepair(filteredDataSources); + var needingRepairIds = needingRepair.Select(source => source.Id).ToHashSet(StringComparer.Ordinal); + var awaitingReindex = (await this.GetDataSourcesAwaitingReindex(filteredDataSources)).Where(source => !needingRepairIds.Contains(source.Id)).ToList(); + + var blockedIds = awaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal); + blockedIds.UnionWith(needingRepairIds); + + var usableDataSources = filteredDataSources.Where(source => !blockedIds.Contains(source.Id)).ToList(); var filteredSelectedDataSources = usableDataSources.Where(source => previousSelectedDataSourceIds.Contains(source.Id)).ToList(); - return new(usableDataSources, filteredSelectedDataSources, awaitingReindex); + return new(usableDataSources, filteredSelectedDataSources, awaitingReindex, needingRepair); } /// @@ -195,6 +205,30 @@ public sealed class DataSourceService return awaitingReindex; } + /// + /// Picks out the data sources whose index cannot be read anymore, so that they wait for a repair. + /// + /// + /// Reads nothing from a database, unlike the re-index check above: the state is held in memory + /// by the embedding service, which is why this one needs no parallelism and no timeout. + /// + /// The data sources which passed every other check. + /// Those of them which wait for a repair, in the order they came in. + private IReadOnlyList GetDataSourcesNeedingRepair(IReadOnlyList dataSources) + { + var needingRepair = new List(); + foreach (var dataSource in dataSources) + { + if (!this.embeddingService.NeedsIndexRepair(dataSource)) + continue; + + this.logger.LogInformation("The index of data source '{DataSourceName}' ({DataSourceId}) cannot be read. It is shown, but cannot be selected until it was repaired.", dataSource.Name, dataSource.Id); + needingRepair.Add(dataSource); + } + + return needingRepair; + } + private async Task> GetAllowedDataSources(bool usingTrustedProvider, IReadOnlyList participatingProviders, IReadOnlyCollection requestedDataSources) { var filteredDataSources = new List(requestedDataSources.Count);