MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 03:33:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Show why an embedding or a retrieval failed

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-06 16:13:28 +02:00
parent 21a01499da
commit 871bbfe79a
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
5 changed files with 134 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
app/MindWork AI Studio/Pages/Embeddings.razor
View File

@ -1,5 +1,6 @@
@attribute [Route(Routes.EMBEDDINGS)]
@inherits MSGComponentBase
@using AIStudio.Provider
<MudStack Spacing="3" Class="pr-2 pb-4" Style="height: 100%; min-height: 0; overflow-y: auto;">
<MudPaper Class="pa-4 border-dashed border rounded-lg" Elevation="0">
@ -73,6 +74,26 @@
<MudText Typo="Typo.caption" Color="Color.Warning" Style="word-break: break-word;">
@failure.Reason
</MudText>
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="1" AlignItems="AlignItems.Center">
@if (failure.FailureReason is not ProviderRequestFailureReason.NONE)
{
<MudChip T="string" Size="Size.Small" Color="Color.Warning" Variant="Variant.Outlined">@failure.FailureReason.GetName()</MudChip>
}
@if (!string.IsNullOrWhiteSpace(failure.EmbeddingProviderName))
{
<MudText Typo="Typo.caption">@string.Format(T("Embedding provider: {0}"), failure.EmbeddingProviderName)</MudText>
}
@if (failure.OccurredAtUtc > DateTimeOffset.MinValue)
{
<MudText Typo="Typo.caption">@failure.OccurredAtUtc.ToLocalTime().ToString("g")</MudText>
}
@if (failure.FailureReason.IsFixedInProviderSettings())
{
<MudButton Variant="Variant.Text" Size="Size.Small" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Settings" OnClick="@this.OpenEmbeddingProviderSettings">
@T("Open the settings")
</MudButton>
}
</MudStack>
</MudStack>
}
</MudStack>
@ -80,7 +101,12 @@
</MudExpansionPanels>
}
@if (!string.IsNullOrWhiteSpace(status.LastError) && status.Failures.Count == 0)
@*
Shown next to the list, not instead of it: the list says which files failed,
while this is the one sentence about the data source as a whole. Hiding it
as soon as a single file failed is what made it invisible in practice.
*@
@if (!string.IsNullOrWhiteSpace(status.LastError))
{
<MudAlert Severity="Severity.Warning" Variant="Variant.Text">
@status.LastError

9
app/MindWork AI Studio/Pages/Embeddings.razor.cs
View File

@ -77,6 +77,15 @@ public partial class Embeddings : MSGComponentBase
(status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
}
/// <summary>
/// Takes the user to the settings, where the embedding providers are configured.
/// </summary>
/// <remarks>
/// Offered only for the failures a setting fixes, such as a rejected API key. Reading what
/// went wrong and then having to find the right page is where people give up.
/// </remarks>
private void OpenEmbeddingProviderSettings() => this.NavigationManager.NavigateTo(Routes.SETTINGS);
private async Task RefreshDataSource(DataSourceEmbeddingStatus status)
{
await this.DataSourceEmbeddingService.RetryDataSourceAsync(status.DataSourceId);

47
app/MindWork AI Studio/Provider/ProviderRequestFailureReasonExtensions.cs Normal file
View File

@ -0,0 +1,47 @@
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Provider;
public static class ProviderRequestFailureReasonExtensions
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ProviderRequestFailureReasonExtensions).Namespace, nameof(ProviderRequestFailureReasonExtensions));
/// <summary>
/// Names the kind of failure in a few words.
/// </summary>
/// <remarks>
/// Meant as a label beside the full message, so a list of failures can be scanned instead of
/// read: twenty entries which all say API key are one problem, not twenty.
/// </remarks>
public static string GetName(this ProviderRequestFailureReason failureReason) => failureReason switch
{
ProviderRequestFailureReason.INSUFFICIENT_QUOTA => TB("No credits left"),
ProviderRequestFailureReason.TOO_MANY_REQUESTS => TB("Too many requests"),
ProviderRequestFailureReason.MODEL_NOT_SUPPORTED_BY_PROVIDER => TB("Model not offered"),
ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY => TB("API key problem"),
ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR => TB("Not permitted"),
ProviderRequestFailureReason.PROVIDER_UNAVAILABLE => TB("Provider unreachable"),
ProviderRequestFailureReason.MODEL_NOT_FOUND => TB("Model unknown"),
ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED => TB("Text too long"),
ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED => TB("No embeddings"),
ProviderRequestFailureReason.INVALID_RESPONSE => TB("Unreadable answer"),
ProviderRequestFailureReason.UNKNOWN => TB("Unknown cause"),
_ => string.Empty,
};
/// <summary>
/// Gets a value indicating whether the way out of this failure is in the provider settings.
/// </summary>
/// <remarks>
/// Only for the failures a setting actually fixes. Pointing at the settings for a provider
/// which is merely overloaded would send the user looking for a mistake they never made.
/// </remarks>
public static bool IsFixedInProviderSettings(this ProviderRequestFailureReason failureReason) => failureReason is
ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY or
ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR or
ProviderRequestFailureReason.MODEL_NOT_FOUND or
ProviderRequestFailureReason.MODEL_NOT_SUPPORTED_BY_PROVIDER or
ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED or
ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED;
}

2
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs
View File

@ -15,7 +15,7 @@ public sealed record DataSourceEmbeddingStatus(
string LastError,
IReadOnlyList<DataSourceEmbeddingFailure> Failures)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));
public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round(this.IndexedFiles * 100d / this.TotalFiles), 0, 100);

50
app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs
View File

@ -17,6 +17,13 @@ public sealed class DataSourceLocalRetrievalService(
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceLocalRetrievalService).Namespace, nameof(DataSourceLocalRetrievalService));
//
// Which gaps the user was already told about in this session. Retrieval runs for every single
// message, so without this one broken embedding provider would put a warning on every prompt.
//
private readonly HashSet<string> reportedRetrievalGaps = new(StringComparer.Ordinal);
private readonly Lock retrievalGapLock = new();
private enum RetrievalChannel
{
VECTOR,
@ -109,12 +116,14 @@ public sealed class DataSourceLocalRetrievalService(
dataSource.Name,
dataSource.Id,
vectorStore.Name);
await this.ReportRetrievalGapAsync(dataSource, "no-vector-store", string.Format(TB("The data source '{0}' was left out of the answer: its local index is not available."), dataSource.Name));
return [];
}
if (!DataSourceEmbeddingProviders.TryResolve(settingsManager, dataSource, out var embeddingProvider))
{
logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the selected embedding provider is not available.", dataSource.Name, dataSource.Id);
await this.ReportRetrievalGapAsync(dataSource, "no-embedding-provider", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider is not available. Please check it in the settings."), dataSource.Name));
return [];
}
@ -128,6 +137,7 @@ public sealed class DataSourceLocalRetrievalService(
if (vector is null || vector.Count == 0)
{
logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because query embedding returned no vector.", dataSource.Name, dataSource.Id);
await this.ReportRetrievalGapAsync(dataSource, "no-query-vector", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector for your message."), dataSource.Name, embeddingProvider.Name));
return [];
}
@ -143,13 +153,50 @@ public sealed class DataSourceLocalRetrievalService(
{
throw;
}
catch (ProviderRequestException exception)
{
//
// The embedding provider named the cause and what to do about it. That sentence is
// worth far more to the user than the fact that a search came back empty:
//
logger.LogWarning(
exception,
"Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because the embedding provider failed. FailureReason={FailureReason}, StatusCode={StatusCode}.",
dataSource.Name, dataSource.Id, exception.FailureReason, exception.StatusCode);
await this.ReportRetrievalGapAsync(dataSource, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage));
return [];
}
catch (Exception exception)
{
logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
await this.ReportRetrievalGapAsync(dataSource, "vector-search-failed", string.Format(TB("The data source '{0}' was left out of the answer because searching it failed."), dataSource.Name));
return [];
}
}
/// <summary>
/// Tells the user once that a data source cannot take part in answering.
/// </summary>
/// <remarks>
/// A failed search is not an error of the chat: the model still answers, only without what
/// this data source knows. Saying so once is what keeps somebody from trusting an answer
/// which was put together without half of its sources. Saying it with every prompt would be
/// worse than saying nothing, which is why every gap is reported once per session.
/// </remarks>
/// <param name="dataSource">The data source which could not be searched.</param>
/// <param name="gapKey">What kind of gap this is, so a different problem is reported again.</param>
/// <param name="userMessage">What to tell the user.</param>
private async Task ReportRetrievalGapAsync(IInternalDataSource dataSource, string gapKey, string userMessage)
{
lock (this.retrievalGapLock)
{
if (!this.reportedRetrievalGaps.Add($"{dataSource.Id}::{gapKey}"))
return;
}
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.SearchOff, userMessage));
}
private async Task<bool> QueryFitsEmbeddingProviderAsync(
IInternalDataSource dataSource,
EmbeddingProvider embeddingProvider,
@ -166,6 +213,7 @@ public sealed class DataSourceLocalRetrievalService(
query.Length,
RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH,
providerTokenLimit);
await this.ReportRetrievalGapAsync(dataSource, "query-too-long", string.Format(TB("The data source '{0}' was left out of the answer because your message is too long to search with."), dataSource.Name));
return false;
}
@ -178,6 +226,7 @@ public sealed class DataSourceLocalRetrievalService(
dataSource.Id,
embeddingProvider.Name,
tokenCountResponse?.Message ?? "No response was returned by the tokenizer service.");
await this.ReportRetrievalGapAsync(dataSource, "no-token-count", string.Format(TB("The data source '{0}' was left out of the answer: the tokenizer of its embedding provider '{1}' is not available."), dataSource.Name, embeddingProvider.Name));
return false;
}
@ -191,6 +240,7 @@ public sealed class DataSourceLocalRetrievalService(
queryTokenCount,
embeddingProvider.Name,
providerTokenLimit);
await this.ReportRetrievalGapAsync(dataSource, "query-over-token-limit", string.Format(TB("The data source '{0}' was left out of the answer because your message is longer than its embedding provider '{1}' accepts."), dataSource.Name, embeddingProvider.Name));
return false;
}