diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs index 72f35397..a7363adf 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs @@ -85,7 +85,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource _ => string.Empty }; - return await this.RetrieveDataAsync(latestUserPrompt, lastUserPrompt.ToERIContentType, thread, this.MaxMatches, token); + return await this.RetrieveDataAsync(latestUserPrompt, lastUserPrompt.ToERIContentType, thread, this.MaxMatches, token) ?? []; } /// @@ -103,11 +103,15 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource // returns fewer matches than asked for ends the paging early, which errs on the safe side. // var contexts = await this.RetrieveDataAsync(query, ContentType.TEXT, thread, window, token); + if (contexts is null) + return RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] }; + var (pageContexts, hasMore) = RetrievalPaging.Cut(contexts, page, this.MaxMatches); return new RetrievalPage(pageContexts, hasMore); } - private async Task> RetrieveDataAsync(string latestUserPrompt, ContentType latestUserPromptType, ChatThread thread, int maxMatches, CancellationToken token) + /// What the ERI server found, or null when it could not be searched. + private async Task?> RetrieveDataAsync(string latestUserPrompt, ContentType latestUserPromptType, ChatThread thread, int maxMatches, CancellationToken token) { // Important: Do not dispose the RustService here, as it is a singleton. var rustService = Program.SERVICE_PROVIDER.GetRequiredService(); @@ -175,11 +179,11 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource } logger.LogWarning($"Was not able to retrieve data from the ERI data source '{this.Name}'. Message: {retrievalResponse.Message}"); - return []; + return null; } logger.LogWarning($"Was not able to authenticate with the ERI data source '{this.Name}'. Message: {authResponse.Message}"); - return []; + return null; } public static bool TryParseConfiguration(int idx, LuaTable table, Guid configPluginId, out DataSourceERI_V1 dataSource) diff --git a/app/MindWork AI Studio/Settings/IDataSource.cs b/app/MindWork AI Studio/Settings/IDataSource.cs index 89d3c3e6..05a134fe 100644 --- a/app/MindWork AI Studio/Settings/IDataSource.cs +++ b/app/MindWork AI Studio/Settings/IDataSource.cs @@ -43,12 +43,16 @@ public interface IDataSource : IConfigurationObject /// lets the model work it out from the conversation, and search as often as it takes. The first /// page holds what the retrieval above finds for the same text. How the pages are cut is /// described in RetrievalPaging. + /// + /// Since the user did not write the query, the user is not told about problems with it. They + /// arrive in RetrievalPage.Gaps instead, together with everything else which kept the search + /// from covering the whole data source. /// /// What to search for. /// The page to retrieve, from 1 up to RetrievalPaging.GetLastPage for MaxMatches. /// The chat thread. /// The cancellation token. - /// The retrieved data contexts of this page, and whether the next page is worth asking for. + /// The retrieved data contexts of this page, whether the next page is worth asking for, and what the search could not cover. /// The page is below 1 or beyond the last page. public Task RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/RAG/RetrievalGap.cs b/app/MindWork AI Studio/Tools/RAG/RetrievalGap.cs new file mode 100644 index 00000000..d0d25a83 --- /dev/null +++ b/app/MindWork AI Studio/Tools/RAG/RetrievalGap.cs @@ -0,0 +1,25 @@ +namespace AIStudio.Tools.RAG; + +/// +/// What kept a search from covering the whole data source. +/// +public enum RetrievalGap +{ + /// + /// The data source could not be searched at all, e.g., while it is being indexed again, or + /// when its ERI server could not be reached. + /// + NOT_SEARCHED, + + /// + /// Part of the search failed, e.g., the vector search while the embedding provider is not + /// available. The matches came from the rest of it and may be incomplete. + /// + PARTLY_SEARCHED, + + /// + /// The query could not be used for part of the search, e.g., because it is longer than the + /// embedding model accepts. A shorter query would be searched in full. + /// + QUERY_NOT_SEARCHABLE, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs b/app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs index f34ddd05..cfc7e14c 100644 --- a/app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs +++ b/app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs @@ -19,4 +19,16 @@ public sealed record RetrievalPage(IReadOnlyList Contexts, bo /// A page without any matches and nothing after it. /// public static readonly RetrievalPage EMPTY = new([], false); + + /// + /// What kept the search from covering the whole data source. Empty when nothing did. + /// + /// + /// Without this, a data source which could not be searched would look like one which found + /// nothing, and the model would tell the user their documents do not mention what they might + /// well mention. A local data source tells the user about its own problems as well, since only + /// the user can fix those. Not so about problems of the query: it was written by whoever asked + /// for this page, and so is a better one. + /// + public IReadOnlyList Gaps { get; init; } = []; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index e2e25f63..e7b250a1 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -54,6 +54,31 @@ public sealed class DataSourceLocalRetrievalService( int Rank); // ReSharper restore NotAccessedPositionalProperty.Local + /// + /// What kept one retrieval from covering the whole data source. + /// + /// Whether the query is the user's own message, which decides who hears about its problems. + private sealed class RetrievalRun(bool queryWrittenByUser) + { + // Both channels search at the same time: + private readonly Lock gapLock = new(); + private readonly HashSet gaps = []; + + public bool QueryWrittenByUser => queryWrittenByUser; + + public void Add(RetrievalGap gap) + { + lock (this.gapLock) + this.gaps.Add(gap); + } + + public IReadOnlyList GetGaps() + { + lock (this.gapLock) + return this.gaps.Order().ToList(); + } + } + public Task> RetrieveDataAsync(DataSourceLocalFile dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) => this.RetrieveDataAsync(dataSource, lastUserPrompt, token); @@ -61,19 +86,19 @@ public sealed class DataSourceLocalRetrievalService( this.RetrieveDataAsync(dataSource, lastUserPrompt, token); public Task RetrieveDataAsync(DataSourceLocalFile dataSource, string query, int page, ChatThread thread, CancellationToken token = default) => - this.RetrievePageAsync(dataSource, query, page, token); + this.RetrievePageAsync(dataSource, query, page, new RetrievalRun(queryWrittenByUser: false), token); public Task RetrieveDataAsync(DataSourceLocalDirectory dataSource, string query, int page, ChatThread thread, CancellationToken token = default) => - this.RetrievePageAsync(dataSource, query, page, token); + this.RetrievePageAsync(dataSource, query, page, new RetrievalRun(queryWrittenByUser: false), token); private async Task> RetrieveDataAsync(IInternalDataSource dataSource, IContent lastUserPrompt, CancellationToken token) { // The first page is what this retrieval has always returned: - var firstPage = await this.RetrievePageAsync(dataSource, GetQueryText(lastUserPrompt), 1, token); + var firstPage = await this.RetrievePageAsync(dataSource, GetQueryText(lastUserPrompt), 1, new RetrievalRun(queryWrittenByUser: true), token); return firstPage.Contexts; } - private async Task RetrievePageAsync(IInternalDataSource dataSource, string query, int page, CancellationToken token) + private async Task RetrievePageAsync(IInternalDataSource dataSource, string query, int page, RetrievalRun run, CancellationToken token) { var pageSize = (int)dataSource.MaxMatches; var window = RetrievalPaging.GetWindowSize(page, pageSize); @@ -99,13 +124,13 @@ public sealed class DataSourceLocalRetrievalService( if (await embeddingService.IsAwaitingReindexAsync(dataSource, token)) { logger.LogWarning("Skipping local retrieval for data source '{DataSourceName}' ({DataSourceId}) because its index has to be built anew.", dataSource.Name, dataSource.Id); - await this.ReportRetrievalGapAsync(dataSource, "index-rebuilding", string.Format(TB("The data source '{0}' was left out of the answer: it is being indexed again and cannot be searched until that is finished."), dataSource.Name)); - return RetrievalPage.EMPTY; + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.NOT_SEARCHED, "index-rebuilding", string.Format(TB("The data source '{0}' was left out of the answer: it is being indexed again and cannot be searched until that is finished."), dataSource.Name)); + return RetrievalPage.EMPTY with { Gaps = run.GetGaps() }; } var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id); - var vectorTask = this.SearchVectorAsync(dataSource, query, window, collectionName, token); - var bm25Task = this.SearchBm25Async(dataSource, query, window, token); + var vectorTask = this.SearchVectorAsync(dataSource, query, window, collectionName, run, token); + var bm25Task = this.SearchBm25Async(dataSource, query, window, run, token); await Task.WhenAll(vectorTask, bm25Task); token.ThrowIfCancellationRequested(); @@ -117,8 +142,9 @@ public sealed class DataSourceLocalRetrievalService( page, pageSize); + var gaps = run.GetGaps(); logger.LogInformation( - "Retrieved {MergedHits} local RAG hits on page {Page} for data source '{DataSourceName}' ({DataSourceId}). VectorCandidates={VectorHits}, BM25Candidates={BM25Hits}, RequestedPerChannel={RequestedPerChannel}, HasMore={HasMore}.", + "Retrieved {MergedHits} local RAG hits on page {Page} for data source '{DataSourceName}' ({DataSourceId}). VectorCandidates={VectorHits}, BM25Candidates={BM25Hits}, RequestedPerChannel={RequestedPerChannel}, HasMore={HasMore}, Gaps=[{Gaps}].", hits.Count, page, dataSource.Name, @@ -126,14 +152,15 @@ public sealed class DataSourceLocalRetrievalService( vectorTask.Result.Count, bm25Task.Result.Count, window, - hasMore); + hasMore, + string.Join(", ", gaps)); var contexts = hits .Where(hit => !string.IsNullOrWhiteSpace(hit.Text)) .Select(hit => ToRetrievalContext(hit, dataSource)) .ToList(); - return new RetrievalPage(contexts, hasMore); + return new RetrievalPage(contexts, hasMore) { Gaps = gaps }; } private async Task> SearchVectorAsync( @@ -141,6 +168,7 @@ public sealed class DataSourceLocalRetrievalService( string query, int maxMatches, string collectionName, + RetrievalRun run, CancellationToken token) { try @@ -153,18 +181,18 @@ public sealed class DataSourceLocalRetrievalService( dataSource.Name, dataSource.Id, vectorStore.Name); - await this.ReportRetrievalGapAsync(dataSource, "no-vector-store", string.Format(TB("The data source '{0}' was left out of the answer: its local index is not available."), dataSource.Name)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "no-vector-store", string.Format(TB("The data source '{0}' was left out of the answer: its local index is not available."), dataSource.Name)); return []; } if (!DataSourceEmbeddingProviders.TryResolve(settingsManager, dataSource, out var embeddingProvider)) { logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the selected embedding provider is not available.", dataSource.Name, dataSource.Id); - await this.ReportRetrievalGapAsync(dataSource, "no-embedding-provider", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider is not available. Please check it in the settings."), dataSource.Name)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "no-embedding-provider", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider is not available. Please check it in the settings."), dataSource.Name)); return []; } - if (!await this.QueryFitsEmbeddingProviderAsync(dataSource, embeddingProvider, query, token)) + if (!await this.QueryFitsEmbeddingProviderAsync(dataSource, embeddingProvider, query, run, token)) return []; var provider = embeddingProvider.CreateProvider(); @@ -174,7 +202,7 @@ public sealed class DataSourceLocalRetrievalService( if (vector is null || vector.Count == 0) { logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because query embedding returned no vector.", dataSource.Name, dataSource.Id); - await this.ReportRetrievalGapAsync(dataSource, "no-query-vector", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector for your message."), dataSource.Name, embeddingProvider.Name)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "no-query-vector", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector to search with."), dataSource.Name, embeddingProvider.Name)); return []; } @@ -200,7 +228,7 @@ public sealed class DataSourceLocalRetrievalService( exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because the embedding provider failed. FailureReason={FailureReason}, StatusCode={StatusCode}.", dataSource.Name, dataSource.Id, exception.FailureReason, exception.StatusCode); - await this.ReportRetrievalGapAsync(dataSource, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage)); return []; } catch (VectorStoreUnreadableException exception) @@ -211,31 +239,40 @@ public sealed class DataSourceLocalRetrievalService( // answer into one the user can do something about. // logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because its vector store cannot be read.", dataSource.Name, dataSource.Id); - await this.ReportRetrievalGapAsync(dataSource, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."), dataSource.Name)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."), dataSource.Name)); return []; } catch (Exception exception) { logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); - await this.ReportRetrievalGapAsync(dataSource, "vector-search-failed", string.Format(TB("The data source '{0}' was left out of the answer because searching it failed."), dataSource.Name)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "vector-search-failed", string.Format(TB("The data source '{0}' was left out of the answer because searching it failed."), dataSource.Name)); return []; } } /// - /// Tells the user once that a data source cannot take part in answering. + /// Records that a data source cannot fully take part in answering, and tells the user once. /// /// /// A failed search is not an error of the chat: the model still answers, only without what /// this data source knows. Saying so once is what keeps somebody from trusting an answer /// which was put together without half of its sources. Saying it with every prompt would be /// worse than saying nothing, which is why every gap is reported once per session. + /// + /// The retrieval records every gap regardless, cf. RetrievalPage.Gaps: whoever asked for the + /// page has to know each time, not once per session. /// /// The data source which could not be searched. + /// The retrieval this gap belongs to. + /// What the gap means for the search. /// What kind of gap this is, so a different problem is reported again. /// What to tell the user. - private async Task ReportRetrievalGapAsync(IInternalDataSource dataSource, string gapKey, string userMessage) + private async Task ReportRetrievalGapAsync(IInternalDataSource dataSource, RetrievalRun run, RetrievalGap gap, string gapKey, string userMessage) { + run.Add(gap); + if (!IsForTheUser(gap, run.QueryWrittenByUser)) + return; + lock (this.retrievalGapLock) { if (!this.reportedRetrievalGaps.Add($"{dataSource.Id}::{gapKey}")) @@ -245,23 +282,38 @@ public sealed class DataSourceLocalRetrievalService( await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.SearchOff, userMessage)); } + /// + /// Whether the user has to hear about a gap. + /// + /// + /// Problems of the data source are for the user, since only the user can fix them. Problems of + /// the query are for whoever wrote it. When the model worked the query out, telling the user + /// their message was too long would be wrong, and the model learns about it from the page and + /// can search with a shorter one. + /// + /// What the gap means for the search. + /// Whether the query is the user's own message. + /// True when the user has to be told. + internal static bool IsForTheUser(RetrievalGap gap, bool queryWrittenByUser) => gap is not RetrievalGap.QUERY_NOT_SEARCHABLE || queryWrittenByUser; + private async Task QueryFitsEmbeddingProviderAsync( IInternalDataSource dataSource, EmbeddingProvider embeddingProvider, string query, + RetrievalRun run, CancellationToken token) { var providerTokenLimit = Math.Max(1, embeddingProvider.EffectiveTokenLimit); if (query.Length > RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH) { logger.LogWarning( - "Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the latest prompt has {CharacterCount} characters and exceeds the safe tokenizer request length of {MaxCharacterCount}. ProviderTokenLimit={ProviderTokenLimit}.", + "Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the query has {CharacterCount} characters and exceeds the safe tokenizer request length of {MaxCharacterCount}. ProviderTokenLimit={ProviderTokenLimit}.", dataSource.Name, dataSource.Id, query.Length, RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH, providerTokenLimit); - await this.ReportRetrievalGapAsync(dataSource, "query-too-long", string.Format(TB("The data source '{0}' was left out of the answer because your message is too long to search with."), dataSource.Name)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.QUERY_NOT_SEARCHABLE, "query-too-long", string.Format(TB("The data source '{0}' was left out of the answer because your message is too long to search with."), dataSource.Name)); return false; } @@ -274,7 +326,7 @@ public sealed class DataSourceLocalRetrievalService( dataSource.Id, embeddingProvider.Name, tokenCountResponse?.Message ?? "No response was returned by the tokenizer service."); - await this.ReportRetrievalGapAsync(dataSource, "no-token-count", string.Format(TB("The data source '{0}' was left out of the answer: the tokenizer of its embedding provider '{1}' is not available."), dataSource.Name, embeddingProvider.Name)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "no-token-count", string.Format(TB("The data source '{0}' was left out of the answer: the tokenizer of its embedding provider '{1}' is not available."), dataSource.Name, embeddingProvider.Name)); return false; } @@ -282,20 +334,20 @@ public sealed class DataSourceLocalRetrievalService( if (queryTokenCount > providerTokenLimit) { logger.LogWarning( - "Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the latest prompt has {QueryTokenCount} tokens, exceeding embedding provider '{EmbeddingProviderName}' limit of {ProviderTokenLimit} tokens.", + "Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the query has {QueryTokenCount} tokens, exceeding embedding provider '{EmbeddingProviderName}' limit of {ProviderTokenLimit} tokens.", dataSource.Name, dataSource.Id, queryTokenCount, embeddingProvider.Name, providerTokenLimit); - await this.ReportRetrievalGapAsync(dataSource, "query-over-token-limit", string.Format(TB("The data source '{0}' was left out of the answer because your message is longer than its embedding provider '{1}' accepts."), dataSource.Name, embeddingProvider.Name)); + await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.QUERY_NOT_SEARCHABLE, "query-over-token-limit", string.Format(TB("The data source '{0}' was left out of the answer because your message is longer than its embedding provider '{1}' accepts."), dataSource.Name, embeddingProvider.Name)); return false; } return true; } - private async Task> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token) + private async Task> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, RetrievalRun run, CancellationToken token) { try { @@ -307,6 +359,7 @@ public sealed class DataSourceLocalRetrievalService( dataSource.Name, dataSource.Id, indexStore.Name); + run.Add(RetrievalGap.PARTLY_SEARCHED); return []; } @@ -325,6 +378,7 @@ public sealed class DataSourceLocalRetrievalService( catch (Exception exception) { logger.LogWarning(exception, "BM25 retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); + run.Add(RetrievalGap.PARTLY_SEARCHED); return []; } } diff --git a/app/Tests/Tools/RetrievalGapTests.cs b/app/Tests/Tools/RetrievalGapTests.cs new file mode 100644 index 00000000..06dc2d05 --- /dev/null +++ b/app/Tests/Tools/RetrievalGapTests.cs @@ -0,0 +1,38 @@ +using AIStudio.Tools.RAG; +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks who hears about what kept a search from covering a local data source. +/// +/// +/// With Semantic Search, the model writes the query, not the user. A warning that the message was +/// too long to search with would then blame the user for a query they never wrote, while the model, +/// which could search with a shorter one, would learn nothing. Problems of the data source itself +/// stay with the user either way, since nobody else can fix a missing embedding provider. +/// +[TestFixture] +public sealed class RetrievalGapTests +{ + [Test] + public void AQueryTheModelWroteIsNotTheUsersProblem() + { + Assert.That(DataSourceLocalRetrievalService.IsForTheUser(RetrievalGap.QUERY_NOT_SEARCHABLE, queryWrittenByUser: false), Is.False, "The model learns about it from the page and can search with a shorter query."); + } + + [Test] + public void AMessageTheUserWroteIsTheirsToShorten() + { + Assert.That(DataSourceLocalRetrievalService.IsForTheUser(RetrievalGap.QUERY_NOT_SEARCHABLE, queryWrittenByUser: true), Is.True); + } + + [TestCase(RetrievalGap.NOT_SEARCHED, false)] + [TestCase(RetrievalGap.NOT_SEARCHED, true)] + [TestCase(RetrievalGap.PARTLY_SEARCHED, false)] + [TestCase(RetrievalGap.PARTLY_SEARCHED, true)] + public void ProblemsOfTheDataSourceAreAlwaysForTheUser(RetrievalGap gap, bool queryWrittenByUser) + { + Assert.That(DataSourceLocalRetrievalService.IsForTheUser(gap, queryWrittenByUser), Is.True, "Only the user can fix an index or an embedding provider."); + } +} \ No newline at end of file