MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 01:53:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Offer a repair for data sources with an unreadable index

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-18 14:12:50 +02:00
parent 4aed43d057
commit e5cfa94335
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
7 changed files with 163 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -3808,6 +3808,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2675917723"] = "No
-- No valid embedding
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2698203405"] = "No valid embedding"
-- Repair this data source by indexing it anew
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2771708618"] = "Repair this data source by indexing it anew"
-- Embedding
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2838542994"] = "Embedding"
@ -3955,6 +3958,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Avail
-- This data source is waiting to be indexed again. Until that is finished, it cannot be searched.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "This data source is waiting to be indexed again. Until that is finished, it cannot be searched."
-- The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T4047623216"] = "The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there."
-- Tools (Optional)
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Tools (Optional)"
@ -9139,6 +9145,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} of {1} files
-- Background embeddings
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Background embeddings"
-- Repair this data source by indexing it anew
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2771708618"] = "Repair this data source by indexing it anew"
-- Refresh this data source
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Refresh this data source"
@ -10813,6 +10822,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
-- Qdrant Edge is not available.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
-- Repair Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source"
-- The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T857336889"] = "The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"
-- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment."
@ -12007,6 +12022,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T4515612
-- The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T667058890"] = "The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider."
-- The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T831900720"] = "The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."
-- The folder '{0}' does not exist.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T871336081"] = "The folder '{0}' does not exist."
@ -12055,6 +12073,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T40
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}"
-- The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T59210871"] = "The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."
-- The data source '{0}' was left out of the answer because searching it failed.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T934856625"] = "The data source '{0}' was left out of the answer because searching it failed."

12
app/MindWork AI Studio/Components/DataSourceManagement.razor
View File

@ -59,6 +59,18 @@
<MudTooltip Text="@T("Information")">
<MudIconButton Color="Color.Info" Icon="@Icons.Material.Outlined.Info" OnClick="@(() => this.ShowInformation(context))"/>
</MudTooltip>
@*
Outside the two branches below on purpose: an index which cannot be read is a
matter of this machine, not of the configuration. Hiding the repair for a data
source the organization manages would leave it locked out of every chat with no
way back, and the selection points here for it.
*@
@if (this.CanRepairDataSource(context))
{
<MudTooltip Text="@T("Repair this data source by indexing it anew")">
<MudIconButton Color="Color.Warning" Icon="@Icons.Material.Filled.Build" OnClick="@(() => this.RepairDataSource(context))"/>
</MudTooltip>
}
@if (context.IsEnterpriseConfiguration)
{
<MudTooltip Text="@T("This data source is managed by your organization.")">

19
app/MindWork AI Studio/Components/DataSourceManagement.razor.cs
View File

@ -128,6 +128,25 @@ public partial class DataSourceManagement : MSGComponentBase
return this.SettingsManager.ConfigurationData.DataSources.Any(this.CanRefreshDataSource);
}
/// <remarks>
/// Shown only while the index of this data source cannot be read. The refresh button next to it
/// stays as it is: it would open the same store and fail the same way, but it is offered for
/// every internal data source regardless of state, and singling this one out would say more
/// about the state than that button ever has.
/// </remarks>
private bool CanRepairDataSource(IDataSource dataSource)
{
return this.DataSourceEmbeddingService.NeedsIndexRepair(dataSource);
}
private async Task RepairDataSource(IDataSource dataSource)
{
if (!this.CanRepairDataSource(dataSource))
return;
await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, dataSource.Id, dataSource.Name);
}
private async Task AutomaticRefreshChanged(bool enabled)
{
this.SettingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = enabled;

6
app/MindWork AI Studio/Pages/Embeddings.razor
View File

@ -61,6 +61,12 @@
<MudIconButton Icon="@Icons.Material.Filled.Sync" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.RefreshDataSource(status))" />
</MudTooltip>
}
@if (this.CanRepair(status))
{
<MudTooltip Text="@T("Repair this data source by indexing it anew")">
<MudIconButton Icon="@Icons.Material.Filled.Build" Color="Color.Warning" Size="Size.Small" OnClick="@(async () => await this.RepairDataSource(status))" />
</MudTooltip>
}
</div>
</TitleContent>
<ChildContent>

28
app/MindWork AI Studio/Pages/Embeddings.razor.cs
View File

@ -318,11 +318,26 @@ public partial class Embeddings : MSGComponentBase
await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the file location: {0}"), issue)));
}
/// <remarks>
/// An unreadable index is left to the repair button below: another attempt would open the same
/// store and fail the same way, so offering both would be offering one that does nothing.
/// </remarks>
private bool CanRefresh(DataSourceEmbeddingStatus status)
{
return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
status.State is not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED &&
(status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
status is { VectorStoreUnreadable: false, State: not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED } &&
(status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
}
/// <remarks>
/// Offered for the one failure which no further attempt gets past. It is a button of its own
/// and not the refresh one, because what it does is not what the user expects of a refresh:
/// everything indexed so far is thrown away and paid for again.
/// </remarks>
private bool CanRepair(DataSourceEmbeddingStatus status)
{
return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
status is { State: DataSourceEmbeddingState.FAILED, VectorStoreUnreadable: true };
}
/// <summary>
@ -340,4 +355,13 @@ public partial class Embeddings : MSGComponentBase
this.ReloadStatuses();
await this.InvokeAsync(this.StateHasChanged);
}
private async Task RepairDataSource(DataSourceEmbeddingStatus status)
{
if (!await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, status.DataSourceId, status.DataSourceName))
return;
this.ReloadStatuses();
await this.InvokeAsync(this.StateHasChanged);
}
}

42
app/MindWork AI Studio/Tools/DataSourceRepair.cs Normal file
View File

@ -0,0 +1,42 @@
using AIStudio.Dialogs;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
namespace AIStudio.Tools;
/// <summary>
/// Asks whether a data source should be indexed anew, and starts the rebuild when the user agrees.
/// </summary>
/// <remarks>
/// Kept here rather than in the two places which offer the repair -- the background embeddings page
/// and the data source table -- so the sentence naming what a rebuild costs cannot drift apart
/// between them. Naming both costs is the whole reason for asking at all.
/// </remarks>
public static class DataSourceRepair
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceRepair).Namespace, nameof(DataSourceRepair));
/// <summary>
/// Asks the user, and rebuilds the index of the data source when they agree.
/// </summary>
/// <param name="dialogService">The dialog service to ask with.</param>
/// <param name="embeddingService">The service which does the rebuild.</param>
/// <param name="dataSourceId">The data source to repair.</param>
/// <param name="dataSourceName">The name of that data source, as the question names it.</param>
/// <returns>True when the rebuild was started.</returns>
public static async Task<bool> ConfirmAndRepairAsync(IDialogService dialogService, DataSourceEmbeddingService embeddingService, string dataSourceId, string dataSourceName)
{
var dialogParameters = new DialogParameters<ConfirmDialog>
{
{ x => x.Message, string.Format(TB("The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"), dataSourceName) },
};
var dialogReference = await dialogService.ShowAsync<ConfirmDialog>(TB("Repair Data Source"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN);
var dialogResult = await dialogReference.Result;
if (dialogResult is null || dialogResult.Canceled)
return false;
await embeddingService.RepairDataSourceAsync(dataSourceId);
return true;
}
}

37
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs
View File

@ -5,6 +5,43 @@ namespace AIStudio.Tools.Services;
public sealed partial class DataSourceEmbeddingService
{
/// <summary>
/// Throws away everything stored for one data source and starts a fresh indexing run.
/// </summary>
/// <remarks>
/// The one way out of an index which cannot be read, and nothing in the app takes it by itself:
/// a rebuild sends every document of the data source to the embedding provider once more, which
/// costs money with a cloud provider and hours with a large data source. It happens because the
/// user asked for it, after being told both.
///
/// An active run is stopped first, the same way deleting a data source does it. The repair is
/// offered for a failed data source only, so there should be none -- but a file watcher may
/// well have queued one between the click and this call, and discarding the index next to a
/// live run would leave it half thrown away.
/// </remarks>
/// <param name="dataSourceId">The data source to build anew.</param>
public async Task RepairDataSourceAsync(string dataSourceId)
{
if (!this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) || !this.IsSupportedInternalDataSource(dataSource))
return;
logger.LogWarning(
"Repairing data source '{DataSourceName}' ({DataSourceId}) on the user's request: the stored index is discarded and built anew.",
dataSource.Name,
dataSource.Id);
var activeRun = this.CancelActiveDataSourceRun(dataSource);
this.ClearQueuedDataSourceState(dataSourceId);
if (activeRun is not null)
await activeRun.Completion.Task;
await this.ResetPersistedStateAsync(dataSourceId, null, null, CancellationToken.None);
this.statuses.TryRemove(dataSourceId, out _);
this.PublishStatusChanged();
await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.MANUAL_RETRY);
}
private async Task ResetPersistedStateAsync(
string dataSourceId,
VectorStoreClient? vectorStore,