MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 00:53:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Lock data sources with an unreadable index in the selection

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-18 13:58:18 +02:00
parent d69304606c
commit 4aed43d057
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
9 changed files with 158 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
app/MindWork AI Studio/Components/DataSourceBlockReason.cs Normal file
View File

@ -0,0 +1,29 @@
namespace AIStudio.Components;
/// <summary>
/// Why a data source is listed in the selection, but cannot be picked.
/// </summary>
/// <remarks>
/// A reason rather than a yes or no, because the row has to say something different for each of
/// them: one asks the user to wait, the other one asks them to act. Asking somebody to wait for
/// something which will never happen on its own is the worse of the two mistakes.
/// </remarks>
public enum DataSourceBlockReason
{
/// <summary>
/// Nothing is in the way, the data source can be picked.
/// </summary>
NONE,
/// <summary>
/// The index has to be built anew before this data source can answer a search. This passes by
/// itself, as soon as the background indexing has worked through the data source.
/// </summary>
AWAITING_REINDEX,
/// <summary>
/// The index cannot be read anymore. This does not pass by itself: only the user can start the
/// rebuild, because it sends every document to the embedding provider once more.
/// </summary>
NEEDS_REPAIR,
}

4
app/MindWork AI Studio/Components/DataSourceSelection.razor
View File

@ -92,7 +92,7 @@
<MudList T="IDataSource" Dense="@true" Class="data-source-rows" SelectionMode="@this.GetListSelectionMode()" @bind-SelectedValues:get="@this.selectedDataSources" @bind-SelectedValues:set="@this.SelectionChanged" Style="max-height: 14em;">
@foreach (var source in this.GetListedDataSources())
{
<DataSourceSelectionRow DataSource="@source" IsAwaitingReindex="@this.IsAwaitingReindex(source)"/>
<DataSourceSelectionRow DataSource="@source" BlockReason="@this.GetBlockReason(source)"/>
}
</MudList>
</MudField>
@ -104,7 +104,7 @@
<MudList T="IDataSource" Dense="@true" Class="data-source-rows" SelectionMode="MudBlazor.SelectionMode.SingleSelection" SelectedValues="@this.selectedDataSources" Style="max-height: 14em;">
@foreach (var source in this.GetListedDataSources())
{
<DataSourceSelectionRow DataSource="@source" IsAwaitingReindex="@this.IsAwaitingReindex(source)"/>
<DataSourceSelectionRow DataSource="@source" BlockReason="@this.GetBlockReason(source)"/>
}
</MudList>
</ExpansionPanel>

31
app/MindWork AI Studio/Components/DataSourceSelection.razor.cs
View File

@ -51,6 +51,8 @@ public partial class DataSourceSelection : MSGComponentBase
private IReadOnlyList<IDataSource> availableDataSources = [];
private IReadOnlyList<IDataSource> dataSourcesAwaitingReindex = [];
private HashSet<string> dataSourceIdsAwaitingReindex = new(StringComparer.Ordinal);
private IReadOnlyList<IDataSource> dataSourcesNeedingRepair = [];
private HashSet<string> dataSourceIdsNeedingRepair = new(StringComparer.Ordinal);
private IReadOnlyCollection<IDataSource> selectedDataSources = [];
private bool aiBasedSourceSelection;
private bool aiBasedValidation;
@ -230,15 +232,33 @@ public partial class DataSourceSelection : MSGComponentBase
this.availableDataSources = sources.AllowedDataSources;
this.dataSourcesAwaitingReindex = sources.DataSourcesAwaitingReindex;
this.dataSourceIdsAwaitingReindex = sources.DataSourcesAwaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
this.dataSourcesNeedingRepair = sources.DataSourcesNeedingRepair;
this.dataSourceIdsNeedingRepair = sources.DataSourcesNeedingRepair.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
this.selectedDataSources = sources.SelectedDataSources;
this.waitingForDataSources = false;
this.StateHasChanged();
}
private bool IsAwaitingReindex(IDataSource dataSource) => this.dataSourceIdsAwaitingReindex.Contains(dataSource.Id);
/// <summary>
/// Why a data source is listed but cannot be picked, if it cannot.
/// </summary>
/// <remarks>
/// The repair is asked about first. The service hands a data source to one of the two lists
/// only, but should that ever change, the reason the user can act on is the one worth showing.
/// </remarks>
private DataSourceBlockReason GetBlockReason(IDataSource dataSource)
{
if (this.dataSourceIdsNeedingRepair.Contains(dataSource.Id))
return DataSourceBlockReason.NEEDS_REPAIR;
if (this.dataSourceIdsAwaitingReindex.Contains(dataSource.Id))
return DataSourceBlockReason.AWAITING_REINDEX;
return DataSourceBlockReason.NONE;
}
/// <summary>
/// The data sources the list shows: the usable ones, plus the ones waiting for their index.
/// The data sources the list shows: the usable ones, plus the ones which cannot be searched.
/// </summary>
/// <remarks>
/// Kept in the order the data sources were configured in, rather than usable ones first. A row
@ -247,11 +267,12 @@ public partial class DataSourceSelection : MSGComponentBase
/// </remarks>
private IReadOnlyList<IDataSource> GetListedDataSources()
{
if (this.dataSourcesAwaitingReindex.Count == 0)
if (this.dataSourcesAwaitingReindex.Count == 0 && this.dataSourcesNeedingRepair.Count == 0)
return this.availableDataSources;
var listedIds = this.availableDataSources.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
listedIds.UnionWith(this.dataSourceIdsAwaitingReindex);
listedIds.UnionWith(this.dataSourceIdsNeedingRepair);
return this.GetConfiguredDataSourcesSnapshot().Where(source => listedIds.Contains(source.Id)).ToList();
}
@ -259,11 +280,11 @@ public partial class DataSourceSelection : MSGComponentBase
/// The preselected but unusable data sources the warning box lists.
/// </summary>
/// <remarks>
/// The ones waiting for their index are left out: they have a row of their own in the list
/// The ones which are only blocked are left out: they have a row of their own in the list
/// above, which says the same thing in the place the user is already looking.
/// </remarks>
private IReadOnlyList<IDataSource> GetUnavailablePreselectedDataSourcesToList() =>
this.GetUnavailablePreselectedDataSources().Where(source => !this.IsAwaitingReindex(source)).ToList();
this.GetUnavailablePreselectedDataSources().Where(source => this.GetBlockReason(source) is DataSourceBlockReason.NONE).ToList();
private async Task EnabledChanged(bool state)
{

8
app/MindWork AI Studio/Components/DataSourceSelectionRow.razor
View File

@ -2,8 +2,8 @@
@using AIStudio.Provider
@inherits MSGComponentBase
<MudTooltip Text="@T("This data source is waiting to be indexed again. Until that is finished, it cannot be searched.")" Disabled="@(!this.IsAwaitingReindex)" RootStyle="display: block;" Placement="Placement.Top">
<MudListItem Value="@this.DataSource" Disabled="@this.IsAwaitingReindex">
<MudTooltip Text="@this.GetBlockedTooltip()" Disabled="@(!this.IsBlocked)" RootStyle="display: block;" Placement="Placement.Top">
<MudListItem Value="@this.DataSource" Disabled="@this.IsBlocked">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
<MudText Typo="Typo.body2" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
@this.DataSource.Name
@ -11,9 +11,9 @@
@if (this.DataSource is IInternalDataSource internalSource)
{
<MudSpacer/>
@if (this.IsAwaitingReindex)
@if (this.IsBlocked)
{
<MudIcon Icon="@Icons.Material.Filled.HourglassTop" Size="Size.Small" Color="Color.Warning" Style="flex-shrink: 0;"/>
<MudIcon Icon="@this.GetBlockedIcon()" Size="Size.Small" Color="@this.GetBlockedIconColor()" Style="flex-shrink: 0;"/>
}
<MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
<MudIcon Icon="@Icons.Material.Filled.Security" Size="Size.Small" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>

35
app/MindWork AI Studio/Components/DataSourceSelectionRow.razor.cs
View File

@ -10,9 +10,13 @@ namespace AIStudio.Components;
/// can be used right now.
/// </summary>
/// <remarks>
/// A data source waiting for its index stays in the list instead of disappearing from it, but
/// cannot be picked, and the tooltip says why. The tool selection next to it in the chat answers
/// the same question the same way.
/// A data source which cannot be used stays in the list instead of disappearing from it, but cannot
/// be picked, and the tooltip says why. The tool selection next to it in the chat answers the same
/// question the same way.
///
/// Why it cannot be used decides what the row says and which icon it wears: an index being built
/// anew is a matter of waiting, an index which cannot be read is a matter of acting. Both are the
/// same row otherwise, which is why this is one component with a reason rather than two components.
///
/// The tooltip sits around the list item rather than inside it: a disabled item has its pointer
/// events switched off and would swallow the hover.
@ -26,10 +30,31 @@ public partial class DataSourceSelectionRow : MSGComponentBase
public required IDataSource DataSource { get; set; }
/// <summary>
/// Whether this data source has to be indexed anew before it can answer a search.
/// Why this data source cannot be picked right now, if it cannot.
/// </summary>
[Parameter]
public bool IsAwaitingReindex { get; set; }
public DataSourceBlockReason BlockReason { get; set; } = DataSourceBlockReason.NONE;
private bool IsBlocked => this.BlockReason is not DataSourceBlockReason.NONE;
private string GetBlockedTooltip() => this.BlockReason switch
{
DataSourceBlockReason.AWAITING_REINDEX => T("This data source is waiting to be indexed again. Until that is finished, it cannot be searched."),
DataSourceBlockReason.NEEDS_REPAIR => T("The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there."),
_ => string.Empty,
};
private string GetBlockedIcon() => this.BlockReason switch
{
DataSourceBlockReason.NEEDS_REPAIR => Icons.Material.Filled.ReportProblem,
_ => Icons.Material.Filled.HourglassTop,
};
private Color GetBlockedIconColor() => this.BlockReason switch
{
DataSourceBlockReason.NEEDS_REPAIR => Color.Error,
_ => Color.Warning,
};
private string GetConfidenceIconStyle(IInternalDataSource dataSource) => $"{dataSource.ConfidenceLevel.SetColorStyle(this.SettingsManager)} flex-shrink: 0;";
}

9
app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs
View File

@ -3,7 +3,7 @@ using AIStudio.Settings;
namespace AIStudio.Tools;
/// <summary>
/// Contains the allowed and selected data sources, plus the ones waiting for their index.
/// Contains the allowed and selected data sources, plus the ones which cannot be searched right now.
/// </summary>
/// <remarks>
/// The selected data sources are a subset of the allowed data sources.
@ -13,8 +13,13 @@ namespace AIStudio.Tools;
/// -- takes it to mean "may be used to answer with", and a source whose index is being rebuilt
/// cannot answer anything. It is listed separately so the user interface can still show it and say
/// why it is greyed out, instead of letting it vanish without a word.
///
/// The same holds for the ones waiting for a repair, and they are a list of their own because the
/// two reasons call for different words: one passes by itself, the other one waits for the user.
/// A data source is in at most one of the two lists.
/// </remarks>
/// <param name="AllowedDataSources">The allowed data sources.</param>
/// <param name="SelectedDataSources">The selected data sources, which are a subset of the allowed data sources.</param>
/// <param name="DataSourcesAwaitingReindex">The data sources which passed every check but cannot be searched until their index has been rebuilt.</param>
public readonly record struct AllowedSelectedDataSources(IReadOnlyList<IDataSource> AllowedDataSources, IReadOnlyList<IDataSource> SelectedDataSources, IReadOnlyList<IDataSource> DataSourcesAwaitingReindex);
/// <param name="DataSourcesNeedingRepair">The data sources which passed every check but whose index cannot be read anymore, so that only the user can get them back.</param>
public readonly record struct AllowedSelectedDataSources(IReadOnlyList<IDataSource> AllowedDataSources, IReadOnlyList<IDataSource> SelectedDataSources, IReadOnlyList<IDataSource> DataSourcesAwaitingReindex, IReadOnlyList<IDataSource> DataSourcesNeedingRepair);

16
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -317,6 +317,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
return runState is not DataSourceEmbeddingState.FAILED;
}
/// <summary>
/// Whether a data source cannot be searched because its vector store cannot be read anymore.
/// </summary>
/// <remarks>
/// Unlike the re-index check above, this reads no database at all: the state comes from the run
/// or the search which ran into the unreadable store, and is kept in memory only. That it does
/// not survive a restart is deliberate. The very same store may well open on the next start,
/// and a mark written to disk would then be wrong with nobody noticing. Until something touches
/// the store again, the data source counts as usable, and a failing search says so on its own.
/// </remarks>
/// <param name="dataSource">The data source to ask about.</param>
/// <returns>True when the data source waits for the user to have its index rebuilt.</returns>
public bool NeedsIndexRepair(IDataSource dataSource) =>
this.statuses.TryGetValue(dataSource.Id, out var status) &&
status is { State: DataSourceEmbeddingState.FAILED, VectorStoreUnreadable: true };
public Task QueueDataSourceAsync(IDataSource dataSource)
{
return this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK);

2
app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs
View File

@ -188,7 +188,7 @@ public sealed class DataSourceLocalRetrievalService(
// answer into one the user can do something about.
//
logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because its vector store cannot be read.", dataSource.Name, dataSource.Id);
await this.ReportRetrievalGapAsync(dataSource, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it on the background embeddings page."), dataSource.Name));
await this.ReportRetrievalGapAsync(dataSource, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."), dataSource.Name));
return [];
}
catch (Exception exception)

52
app/MindWork AI Studio/Tools/Services/DataSourceService.cs
View File

@ -51,7 +51,7 @@ public sealed class DataSourceService
if (selectedLLMProvider == Settings.Provider.NONE)
{
this.logger.LogWarning("The selected LLM provider is not set. We cannot filter the data sources by any means.");
return new([], [], []);
return new([], [], [], []);
}
var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
@ -83,12 +83,13 @@ public sealed class DataSourceService
var allowedDataSources = await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, requestedDataSources);
//
// Whoever asks this way has no list to show, so a data source waiting for its index is
// Whoever asks this way has no list to show, so a data source which cannot be searched is
// dropped rather than marked. Handing it back would start a chat with a data source which
// finds nothing -- the very thing being greyed out elsewhere is meant to prevent.
//
var awaitingReindexIds = (await this.GetDataSourcesAwaitingReindex(allowedDataSources)).Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
return allowedDataSources.Where(source => !awaitingReindexIds.Contains(source.Id)).ToList();
var unsearchableIds = (await this.GetDataSourcesAwaitingReindex(allowedDataSources)).Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
unsearchableIds.UnionWith(this.GetDataSourcesNeedingRepair(allowedDataSources).Select(source => source.Id));
return allowedDataSources.Where(source => !unsearchableIds.Contains(source.Id)).ToList();
}
/// <summary>
@ -109,7 +110,7 @@ public sealed class DataSourceService
if (selectedLLMProvider is NoProvider)
{
this.logger.LogWarning("The selected LLM provider is the default provider. We cannot filter the data sources by any means.");
return new([], [], []);
return new([], [], [], []);
}
var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
@ -159,12 +160,21 @@ public sealed class DataSourceService
// being rebuilt is usable again in a while, and saying so on its own row beats letting it
// disappear from the selection without a word.
//
var awaitingReindex = await this.GetDataSourcesAwaitingReindex(filteredDataSources);
var awaitingReindexIds = awaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
var usableDataSources = filteredDataSources.Where(source => !awaitingReindexIds.Contains(source.Id)).ToList();
// A source whose index cannot be read is asked about first and then kept out of the other
// list: both reasons can be true at once, and of the two it is the only one the user can do
// anything about. Telling them to wait instead would be telling them to wait forever.
//
var needingRepair = this.GetDataSourcesNeedingRepair(filteredDataSources);
var needingRepairIds = needingRepair.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
var awaitingReindex = (await this.GetDataSourcesAwaitingReindex(filteredDataSources)).Where(source => !needingRepairIds.Contains(source.Id)).ToList();
var blockedIds = awaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
blockedIds.UnionWith(needingRepairIds);
var usableDataSources = filteredDataSources.Where(source => !blockedIds.Contains(source.Id)).ToList();
var filteredSelectedDataSources = usableDataSources.Where(source => previousSelectedDataSourceIds.Contains(source.Id)).ToList();
return new(usableDataSources, filteredSelectedDataSources, awaitingReindex);
return new(usableDataSources, filteredSelectedDataSources, awaitingReindex, needingRepair);
}
/// <summary>
@ -195,6 +205,30 @@ public sealed class DataSourceService
return awaitingReindex;
}
/// <summary>
/// Picks out the data sources whose index cannot be read anymore, so that they wait for a repair.
/// </summary>
/// <remarks>
/// Reads nothing from a database, unlike the re-index check above: the state is held in memory
/// by the embedding service, which is why this one needs no parallelism and no timeout.
/// </remarks>
/// <param name="dataSources">The data sources which passed every other check.</param>
/// <returns>Those of them which wait for a repair, in the order they came in.</returns>
private IReadOnlyList<IDataSource> GetDataSourcesNeedingRepair(IReadOnlyList<IDataSource> dataSources)
{
var needingRepair = new List<IDataSource>();
foreach (var dataSource in dataSources)
{
if (!this.embeddingService.NeedsIndexRepair(dataSource))
continue;
this.logger.LogInformation("The index of data source '{DataSourceName}' ({DataSourceId}) cannot be read. It is shown, but cannot be selected until it was repaired.", dataSource.Name, dataSource.Id);
needingRepair.Add(dataSource);
}
return needingRepair;
}
private async Task<IReadOnlyList<IDataSource>> GetAllowedDataSources(bool usingTrustedProvider, IReadOnlyList<ParticipatingProvider> participatingProviders, IReadOnlyCollection<IDataSource> requestedDataSources)
{
var filteredDataSources = new List<IDataSource>(requestedDataSources.Count);