diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor index 10dcd396..73cbed5e 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor +++ b/app/MindWork AI Studio/Pages/Embeddings.razor @@ -1,5 +1,6 @@ @attribute [Route(Routes.EMBEDDINGS)] @inherits MSGComponentBase +@using AIStudio.Provider @@ -73,6 +74,26 @@ @failure.Reason + + @if (failure.FailureReason is not ProviderRequestFailureReason.NONE) + { + @failure.FailureReason.GetName() + } + @if (!string.IsNullOrWhiteSpace(failure.EmbeddingProviderName)) + { + @string.Format(T("Embedding provider: {0}"), failure.EmbeddingProviderName) + } + @if (failure.OccurredAtUtc > DateTimeOffset.MinValue) + { + @failure.OccurredAtUtc.ToLocalTime().ToString("g") + } + @if (failure.FailureReason.IsFixedInProviderSettings()) + { + + @T("Open the settings") + + } + } @@ -80,7 +101,12 @@ } - @if (!string.IsNullOrWhiteSpace(status.LastError) && status.Failures.Count == 0) + @* + Shown next to the list, not instead of it: the list says which files failed, + while this is the one sentence about the data source as a whole. Hiding it + as soon as a single file failed is what made it invisible in practice. + *@ + @if (!string.IsNullOrWhiteSpace(status.LastError)) { @status.LastError diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor.cs b/app/MindWork AI Studio/Pages/Embeddings.razor.cs index a094de7d..356765e9 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor.cs +++ b/app/MindWork AI Studio/Pages/Embeddings.razor.cs @@ -77,6 +77,15 @@ public partial class Embeddings : MSGComponentBase (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0); } + /// + /// Takes the user to the settings, where the embedding providers are configured. + /// + /// + /// Offered only for the failures a setting fixes, such as a rejected API key. Reading what + /// went wrong and then having to find the right page is where people give up. + /// + private void OpenEmbeddingProviderSettings() => this.NavigationManager.NavigateTo(Routes.SETTINGS); + private async Task RefreshDataSource(DataSourceEmbeddingStatus status) { await this.DataSourceEmbeddingService.RetryDataSourceAsync(status.DataSourceId); diff --git a/app/MindWork AI Studio/Provider/ProviderRequestFailureReasonExtensions.cs b/app/MindWork AI Studio/Provider/ProviderRequestFailureReasonExtensions.cs new file mode 100644 index 00000000..7c0cc4a8 --- /dev/null +++ b/app/MindWork AI Studio/Provider/ProviderRequestFailureReasonExtensions.cs @@ -0,0 +1,47 @@ +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Provider; + +public static class ProviderRequestFailureReasonExtensions +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ProviderRequestFailureReasonExtensions).Namespace, nameof(ProviderRequestFailureReasonExtensions)); + + /// + /// Names the kind of failure in a few words. + /// + /// + /// Meant as a label beside the full message, so a list of failures can be scanned instead of + /// read: twenty entries which all say API key are one problem, not twenty. + /// + public static string GetName(this ProviderRequestFailureReason failureReason) => failureReason switch + { + ProviderRequestFailureReason.INSUFFICIENT_QUOTA => TB("No credits left"), + ProviderRequestFailureReason.TOO_MANY_REQUESTS => TB("Too many requests"), + ProviderRequestFailureReason.MODEL_NOT_SUPPORTED_BY_PROVIDER => TB("Model not offered"), + ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY => TB("API key problem"), + ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR => TB("Not permitted"), + ProviderRequestFailureReason.PROVIDER_UNAVAILABLE => TB("Provider unreachable"), + ProviderRequestFailureReason.MODEL_NOT_FOUND => TB("Model unknown"), + ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED => TB("Text too long"), + ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED => TB("No embeddings"), + ProviderRequestFailureReason.INVALID_RESPONSE => TB("Unreadable answer"), + ProviderRequestFailureReason.UNKNOWN => TB("Unknown cause"), + + _ => string.Empty, + }; + + /// + /// Gets a value indicating whether the way out of this failure is in the provider settings. + /// + /// + /// Only for the failures a setting actually fixes. Pointing at the settings for a provider + /// which is merely overloaded would send the user looking for a mistake they never made. + /// + public static bool IsFixedInProviderSettings(this ProviderRequestFailureReason failureReason) => failureReason is + ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY or + ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR or + ProviderRequestFailureReason.MODEL_NOT_FOUND or + ProviderRequestFailureReason.MODEL_NOT_SUPPORTED_BY_PROVIDER or + ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED or + ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs index 2047e81b..14017390 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs @@ -15,7 +15,7 @@ public sealed record DataSourceEmbeddingStatus( string LastError, IReadOnlyList Failures) { - private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService)); + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus)); public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round(this.IndexedFiles * 100d / this.TotalFiles), 0, 100); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index aa1b8ca0..8b48e162 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -17,6 +17,13 @@ public sealed class DataSourceLocalRetrievalService( { private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceLocalRetrievalService).Namespace, nameof(DataSourceLocalRetrievalService)); + // + // Which gaps the user was already told about in this session. Retrieval runs for every single + // message, so without this one broken embedding provider would put a warning on every prompt. + // + private readonly HashSet reportedRetrievalGaps = new(StringComparer.Ordinal); + private readonly Lock retrievalGapLock = new(); + private enum RetrievalChannel { VECTOR, @@ -109,12 +116,14 @@ public sealed class DataSourceLocalRetrievalService( dataSource.Name, dataSource.Id, vectorStore.Name); + await this.ReportRetrievalGapAsync(dataSource, "no-vector-store", string.Format(TB("The data source '{0}' was left out of the answer: its local index is not available."), dataSource.Name)); return []; } if (!DataSourceEmbeddingProviders.TryResolve(settingsManager, dataSource, out var embeddingProvider)) { logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the selected embedding provider is not available.", dataSource.Name, dataSource.Id); + await this.ReportRetrievalGapAsync(dataSource, "no-embedding-provider", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider is not available. Please check it in the settings."), dataSource.Name)); return []; } @@ -128,6 +137,7 @@ public sealed class DataSourceLocalRetrievalService( if (vector is null || vector.Count == 0) { logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because query embedding returned no vector.", dataSource.Name, dataSource.Id); + await this.ReportRetrievalGapAsync(dataSource, "no-query-vector", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector for your message."), dataSource.Name, embeddingProvider.Name)); return []; } @@ -143,13 +153,50 @@ public sealed class DataSourceLocalRetrievalService( { throw; } + catch (ProviderRequestException exception) + { + // + // The embedding provider named the cause and what to do about it. That sentence is + // worth far more to the user than the fact that a search came back empty: + // + logger.LogWarning( + exception, + "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because the embedding provider failed. FailureReason={FailureReason}, StatusCode={StatusCode}.", + dataSource.Name, dataSource.Id, exception.FailureReason, exception.StatusCode); + await this.ReportRetrievalGapAsync(dataSource, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage)); + return []; + } catch (Exception exception) { logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); + await this.ReportRetrievalGapAsync(dataSource, "vector-search-failed", string.Format(TB("The data source '{0}' was left out of the answer because searching it failed."), dataSource.Name)); return []; } } + /// + /// Tells the user once that a data source cannot take part in answering. + /// + /// + /// A failed search is not an error of the chat: the model still answers, only without what + /// this data source knows. Saying so once is what keeps somebody from trusting an answer + /// which was put together without half of its sources. Saying it with every prompt would be + /// worse than saying nothing, which is why every gap is reported once per session. + /// + /// The data source which could not be searched. + /// What kind of gap this is, so a different problem is reported again. + /// What to tell the user. + private async Task ReportRetrievalGapAsync(IInternalDataSource dataSource, string gapKey, string userMessage) + { + lock (this.retrievalGapLock) + { + if (!this.reportedRetrievalGaps.Add($"{dataSource.Id}::{gapKey}")) + return; + } + + await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.SearchOff, userMessage)); + } + private async Task QueryFitsEmbeddingProviderAsync( IInternalDataSource dataSource, EmbeddingProvider embeddingProvider, @@ -166,6 +213,7 @@ public sealed class DataSourceLocalRetrievalService( query.Length, RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH, providerTokenLimit); + await this.ReportRetrievalGapAsync(dataSource, "query-too-long", string.Format(TB("The data source '{0}' was left out of the answer because your message is too long to search with."), dataSource.Name)); return false; } @@ -178,6 +226,7 @@ public sealed class DataSourceLocalRetrievalService( dataSource.Id, embeddingProvider.Name, tokenCountResponse?.Message ?? "No response was returned by the tokenizer service."); + await this.ReportRetrievalGapAsync(dataSource, "no-token-count", string.Format(TB("The data source '{0}' was left out of the answer: the tokenizer of its embedding provider '{1}' is not available."), dataSource.Name, embeddingProvider.Name)); return false; } @@ -191,6 +240,7 @@ public sealed class DataSourceLocalRetrievalService( queryTokenCount, embeddingProvider.Name, providerTokenLimit); + await this.ReportRetrievalGapAsync(dataSource, "query-over-token-limit", string.Format(TB("The data source '{0}' was left out of the answer because your message is longer than its embedding provider '{1}' accepts."), dataSource.Name, embeddingProvider.Name)); return false; }