diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua
index c2d7e65a..59749c43 100644
--- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua
+++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua
@@ -3808,6 +3808,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2675917723"] = "No
-- No valid embedding
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2698203405"] = "No valid embedding"
+-- Repair this data source by indexing it anew
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2771708618"] = "Repair this data source by indexing it anew"
+
-- Embedding
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2838542994"] = "Embedding"
@@ -3955,6 +3958,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Avail
-- This data source is waiting to be indexed again. Until that is finished, it cannot be searched.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "This data source is waiting to be indexed again. Until that is finished, it cannot be searched."
+-- The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T4047623216"] = "The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there."
+
-- Tools (Optional)
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Tools (Optional)"
@@ -9139,6 +9145,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} of {1} files
-- Background embeddings
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Background embeddings"
+-- Repair this data source by indexing it anew
+UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2771708618"] = "Repair this data source by indexing it anew"
+
-- Refresh this data source
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Refresh this data source"
@@ -10813,6 +10822,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
-- Qdrant Edge is not available.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
+-- Repair Data Source
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source"
+
+-- The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T857336889"] = "The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"
+
-- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment."
@@ -12007,6 +12022,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T4515612
-- The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T667058890"] = "The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider."
+-- The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T831900720"] = "The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."
+
-- The folder '{0}' does not exist.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T871336081"] = "The folder '{0}' does not exist."
@@ -12055,6 +12073,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T40
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}"
+-- The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T59210871"] = "The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."
+
-- The data source '{0}' was left out of the answer because searching it failed.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T934856625"] = "The data source '{0}' was left out of the answer because searching it failed."
diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor b/app/MindWork AI Studio/Components/DataSourceManagement.razor
index 76f8e314..7d2010a0 100644
--- a/app/MindWork AI Studio/Components/DataSourceManagement.razor
+++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor
@@ -59,6 +59,18 @@
+ @*
+ Outside the two branches below on purpose: an index which cannot be read is a
+ matter of this machine, not of the configuration. Hiding the repair for a data
+ source the organization manages would leave it locked out of every chat with no
+ way back, and the selection points here for it.
+ *@
+ @if (this.CanRepairDataSource(context))
+ {
+
+
+
+ }
@if (context.IsEnterpriseConfiguration)
{
diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs
index 0a121c80..47461d8f 100644
--- a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs
+++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs
@@ -128,6 +128,25 @@ public partial class DataSourceManagement : MSGComponentBase
return this.SettingsManager.ConfigurationData.DataSources.Any(this.CanRefreshDataSource);
}
+ ///
+ /// Shown only while the index of this data source cannot be read. The refresh button next to it
+ /// stays as it is: it would open the same store and fail the same way, but it is offered for
+ /// every internal data source regardless of state, and singling this one out would say more
+ /// about the state than that button ever has.
+ ///
+ private bool CanRepairDataSource(IDataSource dataSource)
+ {
+ return this.DataSourceEmbeddingService.NeedsIndexRepair(dataSource);
+ }
+
+ private async Task RepairDataSource(IDataSource dataSource)
+ {
+ if (!this.CanRepairDataSource(dataSource))
+ return;
+
+ await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, dataSource.Id, dataSource.Name);
+ }
+
private async Task AutomaticRefreshChanged(bool enabled)
{
this.SettingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = enabled;
diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor
index f1c0907f..32b7b6ae 100644
--- a/app/MindWork AI Studio/Pages/Embeddings.razor
+++ b/app/MindWork AI Studio/Pages/Embeddings.razor
@@ -61,6 +61,12 @@
}
+ @if (this.CanRepair(status))
+ {
+
+
+
+ }
diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor.cs b/app/MindWork AI Studio/Pages/Embeddings.razor.cs
index cc8839b3..fa576c74 100644
--- a/app/MindWork AI Studio/Pages/Embeddings.razor.cs
+++ b/app/MindWork AI Studio/Pages/Embeddings.razor.cs
@@ -318,11 +318,26 @@ public partial class Embeddings : MSGComponentBase
await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the file location: {0}"), issue)));
}
+ ///
+ /// An unreadable index is left to the repair button below: another attempt would open the same
+ /// store and fail the same way, so offering both would be offering one that does nothing.
+ ///
private bool CanRefresh(DataSourceEmbeddingStatus status)
{
return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
- status.State is not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED &&
- (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
+ status is { VectorStoreUnreadable: false, State: not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED } &&
+ (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
+ }
+
+ ///
+ /// Offered for the one failure which no further attempt gets past. It is a button of its own
+ /// and not the refresh one, because what it does is not what the user expects of a refresh:
+ /// everything indexed so far is thrown away and paid for again.
+ ///
+ private bool CanRepair(DataSourceEmbeddingStatus status)
+ {
+ return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
+ status is { State: DataSourceEmbeddingState.FAILED, VectorStoreUnreadable: true };
}
///
@@ -340,4 +355,13 @@ public partial class Embeddings : MSGComponentBase
this.ReloadStatuses();
await this.InvokeAsync(this.StateHasChanged);
}
+
+ private async Task RepairDataSource(DataSourceEmbeddingStatus status)
+ {
+ if (!await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, status.DataSourceId, status.DataSourceName))
+ return;
+
+ this.ReloadStatuses();
+ await this.InvokeAsync(this.StateHasChanged);
+ }
}
diff --git a/app/MindWork AI Studio/Tools/DataSourceRepair.cs b/app/MindWork AI Studio/Tools/DataSourceRepair.cs
new file mode 100644
index 00000000..14fb3611
--- /dev/null
+++ b/app/MindWork AI Studio/Tools/DataSourceRepair.cs
@@ -0,0 +1,42 @@
+using AIStudio.Dialogs;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tools;
+
+///
+/// Asks whether a data source should be indexed anew, and starts the rebuild when the user agrees.
+///
+///
+/// Kept here rather than in the two places which offer the repair -- the background embeddings page
+/// and the data source table -- so the sentence naming what a rebuild costs cannot drift apart
+/// between them. Naming both costs is the whole reason for asking at all.
+///
+public static class DataSourceRepair
+{
+ private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceRepair).Namespace, nameof(DataSourceRepair));
+
+ ///
+ /// Asks the user, and rebuilds the index of the data source when they agree.
+ ///
+ /// The dialog service to ask with.
+ /// The service which does the rebuild.
+ /// The data source to repair.
+ /// The name of that data source, as the question names it.
+ /// True when the rebuild was started.
+ public static async Task ConfirmAndRepairAsync(IDialogService dialogService, DataSourceEmbeddingService embeddingService, string dataSourceId, string dataSourceName)
+ {
+ var dialogParameters = new DialogParameters
+ {
+ { x => x.Message, string.Format(TB("The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"), dataSourceName) },
+ };
+
+ var dialogReference = await dialogService.ShowAsync(TB("Repair Data Source"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN);
+ var dialogResult = await dialogReference.Result;
+ if (dialogResult is null || dialogResult.Canceled)
+ return false;
+
+ await embeddingService.RepairDataSourceAsync(dataSourceId);
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs
index 06ce878f..22f0711c 100644
--- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs
+++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs
@@ -5,6 +5,43 @@ namespace AIStudio.Tools.Services;
public sealed partial class DataSourceEmbeddingService
{
+ ///
+ /// Throws away everything stored for one data source and starts a fresh indexing run.
+ ///
+ ///
+ /// The one way out of an index which cannot be read, and nothing in the app takes it by itself:
+ /// a rebuild sends every document of the data source to the embedding provider once more, which
+ /// costs money with a cloud provider and hours with a large data source. It happens because the
+ /// user asked for it, after being told both.
+ ///
+ /// An active run is stopped first, the same way deleting a data source does it. The repair is
+ /// offered for a failed data source only, so there should be none -- but a file watcher may
+ /// well have queued one between the click and this call, and discarding the index next to a
+ /// live run would leave it half thrown away.
+ ///
+ /// The data source to build anew.
+ public async Task RepairDataSourceAsync(string dataSourceId)
+ {
+ if (!this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) || !this.IsSupportedInternalDataSource(dataSource))
+ return;
+
+ logger.LogWarning(
+ "Repairing data source '{DataSourceName}' ({DataSourceId}) on the user's request: the stored index is discarded and built anew.",
+ dataSource.Name,
+ dataSource.Id);
+
+ var activeRun = this.CancelActiveDataSourceRun(dataSource);
+ this.ClearQueuedDataSourceState(dataSourceId);
+ if (activeRun is not null)
+ await activeRun.Completion.Task;
+
+ await this.ResetPersistedStateAsync(dataSourceId, null, null, CancellationToken.None);
+ this.statuses.TryRemove(dataSourceId, out _);
+ this.PublishStatusChanged();
+
+ await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.MANUAL_RETRY);
+ }
+
private async Task ResetPersistedStateAsync(
string dataSourceId,
VectorStoreClient? vectorStore,