From e5cfa94335a7c95dc3d753e378f4c593eacfc796 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Fri, 18 Sep 2026 14:12:50 +0200 Subject: [PATCH] Offer a repair for data sources with an unreadable index --- .../Assistants/I18N/allTexts.lua | 21 ++++++++++ .../Components/DataSourceManagement.razor | 12 ++++++ .../Components/DataSourceManagement.razor.cs | 19 +++++++++ app/MindWork AI Studio/Pages/Embeddings.razor | 6 +++ .../Pages/Embeddings.razor.cs | 28 ++++++++++++- .../Tools/DataSourceRepair.cs | 42 +++++++++++++++++++ .../DataSourceEmbeddingService.State.cs | 37 ++++++++++++++++ 7 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/DataSourceRepair.cs diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index c2d7e65a..59749c43 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3808,6 +3808,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2675917723"] = "No -- No valid embedding UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2698203405"] = "No valid embedding" +-- Repair this data source by indexing it anew +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2771708618"] = "Repair this data source by indexing it anew" + -- Embedding UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2838542994"] = "Embedding" @@ -3955,6 +3958,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Avail -- This data source is waiting to be indexed again. Until that is finished, it cannot be searched. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "This data source is waiting to be indexed again. Until that is finished, it cannot be searched." +-- The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T4047623216"] = "The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there." + -- Tools (Optional) UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Tools (Optional)" @@ -9139,6 +9145,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} of {1} files -- Background embeddings UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Background embeddings" +-- Repair this data source by indexing it anew +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2771708618"] = "Repair this data source by indexing it anew" + -- Refresh this data source UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Refresh this data source" @@ -10813,6 +10822,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM -- Qdrant Edge is not available. UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available." +-- Repair Data Source +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source" + +-- The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now? +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T857336889"] = "The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?" + -- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment. UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment." @@ -12007,6 +12022,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T4515612 -- The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T667058890"] = "The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider." +-- The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T831900720"] = "The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that." + -- The folder '{0}' does not exist. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T871336081"] = "The folder '{0}' does not exist." @@ -12055,6 +12073,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T40 -- Page {0} UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}" +-- The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T59210871"] = "The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings." + -- The data source '{0}' was left out of the answer because searching it failed. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T934856625"] = "The data source '{0}' was left out of the answer because searching it failed." diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor b/app/MindWork AI Studio/Components/DataSourceManagement.razor index 76f8e314..7d2010a0 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor @@ -59,6 +59,18 @@ + @* + Outside the two branches below on purpose: an index which cannot be read is a + matter of this machine, not of the configuration. Hiding the repair for a data + source the organization manages would leave it locked out of every chat with no + way back, and the selection points here for it. + *@ + @if (this.CanRepairDataSource(context)) + { + + + + } @if (context.IsEnterpriseConfiguration) { diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs index 0a121c80..47461d8f 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs @@ -128,6 +128,25 @@ public partial class DataSourceManagement : MSGComponentBase return this.SettingsManager.ConfigurationData.DataSources.Any(this.CanRefreshDataSource); } + /// + /// Shown only while the index of this data source cannot be read. The refresh button next to it + /// stays as it is: it would open the same store and fail the same way, but it is offered for + /// every internal data source regardless of state, and singling this one out would say more + /// about the state than that button ever has. + /// + private bool CanRepairDataSource(IDataSource dataSource) + { + return this.DataSourceEmbeddingService.NeedsIndexRepair(dataSource); + } + + private async Task RepairDataSource(IDataSource dataSource) + { + if (!this.CanRepairDataSource(dataSource)) + return; + + await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, dataSource.Id, dataSource.Name); + } + private async Task AutomaticRefreshChanged(bool enabled) { this.SettingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = enabled; diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor index f1c0907f..32b7b6ae 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor +++ b/app/MindWork AI Studio/Pages/Embeddings.razor @@ -61,6 +61,12 @@ } + @if (this.CanRepair(status)) + { + + + + } diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor.cs b/app/MindWork AI Studio/Pages/Embeddings.razor.cs index cc8839b3..fa576c74 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor.cs +++ b/app/MindWork AI Studio/Pages/Embeddings.razor.cs @@ -318,11 +318,26 @@ public partial class Embeddings : MSGComponentBase await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the file location: {0}"), issue))); } + /// + /// An unreadable index is left to the repair button below: another attempt would open the same + /// store and fail the same way, so offering both would be offering one that does nothing. + /// private bool CanRefresh(DataSourceEmbeddingStatus status) { return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) && - status.State is not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED && - (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0); + status is { VectorStoreUnreadable: false, State: not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED } && + (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0); + } + + /// + /// Offered for the one failure which no further attempt gets past. It is a button of its own + /// and not the refresh one, because what it does is not what the user expects of a refresh: + /// everything indexed so far is thrown away and paid for again. + /// + private bool CanRepair(DataSourceEmbeddingStatus status) + { + return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) && + status is { State: DataSourceEmbeddingState.FAILED, VectorStoreUnreadable: true }; } /// @@ -340,4 +355,13 @@ public partial class Embeddings : MSGComponentBase this.ReloadStatuses(); await this.InvokeAsync(this.StateHasChanged); } + + private async Task RepairDataSource(DataSourceEmbeddingStatus status) + { + if (!await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, status.DataSourceId, status.DataSourceName)) + return; + + this.ReloadStatuses(); + await this.InvokeAsync(this.StateHasChanged); + } } diff --git a/app/MindWork AI Studio/Tools/DataSourceRepair.cs b/app/MindWork AI Studio/Tools/DataSourceRepair.cs new file mode 100644 index 00000000..14fb3611 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataSourceRepair.cs @@ -0,0 +1,42 @@ +using AIStudio.Dialogs; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Services; + +namespace AIStudio.Tools; + +/// +/// Asks whether a data source should be indexed anew, and starts the rebuild when the user agrees. +/// +/// +/// Kept here rather than in the two places which offer the repair -- the background embeddings page +/// and the data source table -- so the sentence naming what a rebuild costs cannot drift apart +/// between them. Naming both costs is the whole reason for asking at all. +/// +public static class DataSourceRepair +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceRepair).Namespace, nameof(DataSourceRepair)); + + /// + /// Asks the user, and rebuilds the index of the data source when they agree. + /// + /// The dialog service to ask with. + /// The service which does the rebuild. + /// The data source to repair. + /// The name of that data source, as the question names it. + /// True when the rebuild was started. + public static async Task ConfirmAndRepairAsync(IDialogService dialogService, DataSourceEmbeddingService embeddingService, string dataSourceId, string dataSourceName) + { + var dialogParameters = new DialogParameters + { + { x => x.Message, string.Format(TB("The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"), dataSourceName) }, + }; + + var dialogReference = await dialogService.ShowAsync(TB("Repair Data Source"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN); + var dialogResult = await dialogReference.Result; + if (dialogResult is null || dialogResult.Canceled) + return false; + + await embeddingService.RepairDataSourceAsync(dataSourceId); + return true; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs index 06ce878f..22f0711c 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs @@ -5,6 +5,43 @@ namespace AIStudio.Tools.Services; public sealed partial class DataSourceEmbeddingService { + /// + /// Throws away everything stored for one data source and starts a fresh indexing run. + /// + /// + /// The one way out of an index which cannot be read, and nothing in the app takes it by itself: + /// a rebuild sends every document of the data source to the embedding provider once more, which + /// costs money with a cloud provider and hours with a large data source. It happens because the + /// user asked for it, after being told both. + /// + /// An active run is stopped first, the same way deleting a data source does it. The repair is + /// offered for a failed data source only, so there should be none -- but a file watcher may + /// well have queued one between the click and this call, and discarding the index next to a + /// live run would leave it half thrown away. + /// + /// The data source to build anew. + public async Task RepairDataSourceAsync(string dataSourceId) + { + if (!this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) || !this.IsSupportedInternalDataSource(dataSource)) + return; + + logger.LogWarning( + "Repairing data source '{DataSourceName}' ({DataSourceId}) on the user's request: the stored index is discarded and built anew.", + dataSource.Name, + dataSource.Id); + + var activeRun = this.CancelActiveDataSourceRun(dataSource); + this.ClearQueuedDataSourceState(dataSourceId); + if (activeRun is not null) + await activeRun.Completion.Task; + + await this.ResetPersistedStateAsync(dataSourceId, null, null, CancellationToken.None); + this.statuses.TryRemove(dataSourceId, out _); + this.PublishStatusChanged(); + + await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.MANUAL_RETRY); + } + private async Task ResetPersistedStateAsync( string dataSourceId, VectorStoreClient? vectorStore,