mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-27 00:13:38 +00:00
Tell whoever asked for a page what the search could not cover
This commit is contained in:
parent
92339479d0
commit
90db777ad5
@ -85,7 +85,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
|
||||
_ => string.Empty
|
||||
};
|
||||
|
||||
return await this.RetrieveDataAsync(latestUserPrompt, lastUserPrompt.ToERIContentType, thread, this.MaxMatches, token);
|
||||
return await this.RetrieveDataAsync(latestUserPrompt, lastUserPrompt.ToERIContentType, thread, this.MaxMatches, token) ?? [];
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@ -103,11 +103,15 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
|
||||
// returns fewer matches than asked for ends the paging early, which errs on the safe side.
|
||||
//
|
||||
var contexts = await this.RetrieveDataAsync(query, ContentType.TEXT, thread, window, token);
|
||||
if (contexts is null)
|
||||
return RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] };
|
||||
|
||||
var (pageContexts, hasMore) = RetrievalPaging.Cut(contexts, page, this.MaxMatches);
|
||||
return new RetrievalPage(pageContexts, hasMore);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(string latestUserPrompt, ContentType latestUserPromptType, ChatThread thread, int maxMatches, CancellationToken token)
|
||||
/// <returns>What the ERI server found, or null when it could not be searched.</returns>
|
||||
private async Task<IReadOnlyList<IRetrievalContext>?> RetrieveDataAsync(string latestUserPrompt, ContentType latestUserPromptType, ChatThread thread, int maxMatches, CancellationToken token)
|
||||
{
|
||||
// Important: Do not dispose the RustService here, as it is a singleton.
|
||||
var rustService = Program.SERVICE_PROVIDER.GetRequiredService<RustService>();
|
||||
@ -175,11 +179,11 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
|
||||
}
|
||||
|
||||
logger.LogWarning($"Was not able to retrieve data from the ERI data source '{this.Name}'. Message: {retrievalResponse.Message}");
|
||||
return [];
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.LogWarning($"Was not able to authenticate with the ERI data source '{this.Name}'. Message: {authResponse.Message}");
|
||||
return [];
|
||||
return null;
|
||||
}
|
||||
|
||||
public static bool TryParseConfiguration(int idx, LuaTable table, Guid configPluginId, out DataSourceERI_V1 dataSource)
|
||||
|
||||
@ -43,12 +43,16 @@ public interface IDataSource : IConfigurationObject
|
||||
/// lets the model work it out from the conversation, and search as often as it takes. The first
|
||||
/// page holds what the retrieval above finds for the same text. How the pages are cut is
|
||||
/// described in RetrievalPaging.
|
||||
///
|
||||
/// Since the user did not write the query, the user is not told about problems with it. They
|
||||
/// arrive in RetrievalPage.Gaps instead, together with everything else which kept the search
|
||||
/// from covering the whole data source.
|
||||
/// </remarks>
|
||||
/// <param name="query">What to search for.</param>
|
||||
/// <param name="page">The page to retrieve, from 1 up to RetrievalPaging.GetLastPage for MaxMatches.</param>
|
||||
/// <param name="thread">The chat thread.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The retrieved data contexts of this page, and whether the next page is worth asking for.</returns>
|
||||
/// <returns>The retrieved data contexts of this page, whether the next page is worth asking for, and what the search could not cover.</returns>
|
||||
/// <exception cref="ArgumentOutOfRangeException">The page is below 1 or beyond the last page.</exception>
|
||||
public Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default);
|
||||
}
|
||||
25
app/MindWork AI Studio/Tools/RAG/RetrievalGap.cs
Normal file
25
app/MindWork AI Studio/Tools/RAG/RetrievalGap.cs
Normal file
@ -0,0 +1,25 @@
|
||||
namespace AIStudio.Tools.RAG;
|
||||
|
||||
/// <summary>
|
||||
/// What kept a search from covering the whole data source.
|
||||
/// </summary>
|
||||
public enum RetrievalGap
|
||||
{
|
||||
/// <summary>
|
||||
/// The data source could not be searched at all, e.g., while it is being indexed again, or
|
||||
/// when its ERI server could not be reached.
|
||||
/// </summary>
|
||||
NOT_SEARCHED,
|
||||
|
||||
/// <summary>
|
||||
/// Part of the search failed, e.g., the vector search while the embedding provider is not
|
||||
/// available. The matches came from the rest of it and may be incomplete.
|
||||
/// </summary>
|
||||
PARTLY_SEARCHED,
|
||||
|
||||
/// <summary>
|
||||
/// The query could not be used for part of the search, e.g., because it is longer than the
|
||||
/// embedding model accepts. A shorter query would be searched in full.
|
||||
/// </summary>
|
||||
QUERY_NOT_SEARCHABLE,
|
||||
}
|
||||
@ -19,4 +19,16 @@ public sealed record RetrievalPage(IReadOnlyList<IRetrievalContext> Contexts, bo
|
||||
/// A page without any matches and nothing after it.
|
||||
/// </summary>
|
||||
public static readonly RetrievalPage EMPTY = new([], false);
|
||||
|
||||
/// <summary>
|
||||
/// What kept the search from covering the whole data source. Empty when nothing did.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Without this, a data source which could not be searched would look like one which found
|
||||
/// nothing, and the model would tell the user their documents do not mention what they might
|
||||
/// well mention. A local data source tells the user about its own problems as well, since only
|
||||
/// the user can fix those. Not so about problems of the query: it was written by whoever asked
|
||||
/// for this page, and so is a better one.
|
||||
/// </remarks>
|
||||
public IReadOnlyList<RetrievalGap> Gaps { get; init; } = [];
|
||||
}
|
||||
@ -54,6 +54,31 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
int Rank);
|
||||
// ReSharper restore NotAccessedPositionalProperty.Local
|
||||
|
||||
/// <summary>
|
||||
/// What kept one retrieval from covering the whole data source.
|
||||
/// </summary>
|
||||
/// <param name="queryWrittenByUser">Whether the query is the user's own message, which decides who hears about its problems.</param>
|
||||
private sealed class RetrievalRun(bool queryWrittenByUser)
|
||||
{
|
||||
// Both channels search at the same time:
|
||||
private readonly Lock gapLock = new();
|
||||
private readonly HashSet<RetrievalGap> gaps = [];
|
||||
|
||||
public bool QueryWrittenByUser => queryWrittenByUser;
|
||||
|
||||
public void Add(RetrievalGap gap)
|
||||
{
|
||||
lock (this.gapLock)
|
||||
this.gaps.Add(gap);
|
||||
}
|
||||
|
||||
public IReadOnlyList<RetrievalGap> GetGaps()
|
||||
{
|
||||
lock (this.gapLock)
|
||||
return this.gaps.Order().ToList();
|
||||
}
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(DataSourceLocalFile dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
|
||||
this.RetrieveDataAsync(dataSource, lastUserPrompt, token);
|
||||
|
||||
@ -61,19 +86,19 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
this.RetrieveDataAsync(dataSource, lastUserPrompt, token);
|
||||
|
||||
public Task<RetrievalPage> RetrieveDataAsync(DataSourceLocalFile dataSource, string query, int page, ChatThread thread, CancellationToken token = default) =>
|
||||
this.RetrievePageAsync(dataSource, query, page, token);
|
||||
this.RetrievePageAsync(dataSource, query, page, new RetrievalRun(queryWrittenByUser: false), token);
|
||||
|
||||
public Task<RetrievalPage> RetrieveDataAsync(DataSourceLocalDirectory dataSource, string query, int page, ChatThread thread, CancellationToken token = default) =>
|
||||
this.RetrievePageAsync(dataSource, query, page, token);
|
||||
this.RetrievePageAsync(dataSource, query, page, new RetrievalRun(queryWrittenByUser: false), token);
|
||||
|
||||
private async Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IInternalDataSource dataSource, IContent lastUserPrompt, CancellationToken token)
|
||||
{
|
||||
// The first page is what this retrieval has always returned:
|
||||
var firstPage = await this.RetrievePageAsync(dataSource, GetQueryText(lastUserPrompt), 1, token);
|
||||
var firstPage = await this.RetrievePageAsync(dataSource, GetQueryText(lastUserPrompt), 1, new RetrievalRun(queryWrittenByUser: true), token);
|
||||
return firstPage.Contexts;
|
||||
}
|
||||
|
||||
private async Task<RetrievalPage> RetrievePageAsync(IInternalDataSource dataSource, string query, int page, CancellationToken token)
|
||||
private async Task<RetrievalPage> RetrievePageAsync(IInternalDataSource dataSource, string query, int page, RetrievalRun run, CancellationToken token)
|
||||
{
|
||||
var pageSize = (int)dataSource.MaxMatches;
|
||||
var window = RetrievalPaging.GetWindowSize(page, pageSize);
|
||||
@ -99,13 +124,13 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
if (await embeddingService.IsAwaitingReindexAsync(dataSource, token))
|
||||
{
|
||||
logger.LogWarning("Skipping local retrieval for data source '{DataSourceName}' ({DataSourceId}) because its index has to be built anew.", dataSource.Name, dataSource.Id);
|
||||
await this.ReportRetrievalGapAsync(dataSource, "index-rebuilding", string.Format(TB("The data source '{0}' was left out of the answer: it is being indexed again and cannot be searched until that is finished."), dataSource.Name));
|
||||
return RetrievalPage.EMPTY;
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.NOT_SEARCHED, "index-rebuilding", string.Format(TB("The data source '{0}' was left out of the answer: it is being indexed again and cannot be searched until that is finished."), dataSource.Name));
|
||||
return RetrievalPage.EMPTY with { Gaps = run.GetGaps() };
|
||||
}
|
||||
|
||||
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
|
||||
var vectorTask = this.SearchVectorAsync(dataSource, query, window, collectionName, token);
|
||||
var bm25Task = this.SearchBm25Async(dataSource, query, window, token);
|
||||
var vectorTask = this.SearchVectorAsync(dataSource, query, window, collectionName, run, token);
|
||||
var bm25Task = this.SearchBm25Async(dataSource, query, window, run, token);
|
||||
|
||||
await Task.WhenAll(vectorTask, bm25Task);
|
||||
token.ThrowIfCancellationRequested();
|
||||
@ -117,8 +142,9 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
page,
|
||||
pageSize);
|
||||
|
||||
var gaps = run.GetGaps();
|
||||
logger.LogInformation(
|
||||
"Retrieved {MergedHits} local RAG hits on page {Page} for data source '{DataSourceName}' ({DataSourceId}). VectorCandidates={VectorHits}, BM25Candidates={BM25Hits}, RequestedPerChannel={RequestedPerChannel}, HasMore={HasMore}.",
|
||||
"Retrieved {MergedHits} local RAG hits on page {Page} for data source '{DataSourceName}' ({DataSourceId}). VectorCandidates={VectorHits}, BM25Candidates={BM25Hits}, RequestedPerChannel={RequestedPerChannel}, HasMore={HasMore}, Gaps=[{Gaps}].",
|
||||
hits.Count,
|
||||
page,
|
||||
dataSource.Name,
|
||||
@ -126,14 +152,15 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
vectorTask.Result.Count,
|
||||
bm25Task.Result.Count,
|
||||
window,
|
||||
hasMore);
|
||||
hasMore,
|
||||
string.Join(", ", gaps));
|
||||
|
||||
var contexts = hits
|
||||
.Where(hit => !string.IsNullOrWhiteSpace(hit.Text))
|
||||
.Select(hit => ToRetrievalContext(hit, dataSource))
|
||||
.ToList();
|
||||
|
||||
return new RetrievalPage(contexts, hasMore);
|
||||
return new RetrievalPage(contexts, hasMore) { Gaps = gaps };
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<VectorSearchResult>> SearchVectorAsync(
|
||||
@ -141,6 +168,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
string query,
|
||||
int maxMatches,
|
||||
string collectionName,
|
||||
RetrievalRun run,
|
||||
CancellationToken token)
|
||||
{
|
||||
try
|
||||
@ -153,18 +181,18 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
dataSource.Name,
|
||||
dataSource.Id,
|
||||
vectorStore.Name);
|
||||
await this.ReportRetrievalGapAsync(dataSource, "no-vector-store", string.Format(TB("The data source '{0}' was left out of the answer: its local index is not available."), dataSource.Name));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "no-vector-store", string.Format(TB("The data source '{0}' was left out of the answer: its local index is not available."), dataSource.Name));
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!DataSourceEmbeddingProviders.TryResolve(settingsManager, dataSource, out var embeddingProvider))
|
||||
{
|
||||
logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the selected embedding provider is not available.", dataSource.Name, dataSource.Id);
|
||||
await this.ReportRetrievalGapAsync(dataSource, "no-embedding-provider", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider is not available. Please check it in the settings."), dataSource.Name));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "no-embedding-provider", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider is not available. Please check it in the settings."), dataSource.Name));
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!await this.QueryFitsEmbeddingProviderAsync(dataSource, embeddingProvider, query, token))
|
||||
if (!await this.QueryFitsEmbeddingProviderAsync(dataSource, embeddingProvider, query, run, token))
|
||||
return [];
|
||||
|
||||
var provider = embeddingProvider.CreateProvider();
|
||||
@ -174,7 +202,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
if (vector is null || vector.Count == 0)
|
||||
{
|
||||
logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because query embedding returned no vector.", dataSource.Name, dataSource.Id);
|
||||
await this.ReportRetrievalGapAsync(dataSource, "no-query-vector", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector for your message."), dataSource.Name, embeddingProvider.Name));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "no-query-vector", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector to search with."), dataSource.Name, embeddingProvider.Name));
|
||||
return [];
|
||||
}
|
||||
|
||||
@ -200,7 +228,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
exception,
|
||||
"Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because the embedding provider failed. FailureReason={FailureReason}, StatusCode={StatusCode}.",
|
||||
dataSource.Name, dataSource.Id, exception.FailureReason, exception.StatusCode);
|
||||
await this.ReportRetrievalGapAsync(dataSource, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage));
|
||||
return [];
|
||||
}
|
||||
catch (VectorStoreUnreadableException exception)
|
||||
@ -211,31 +239,40 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
// answer into one the user can do something about.
|
||||
//
|
||||
logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because its vector store cannot be read.", dataSource.Name, dataSource.Id);
|
||||
await this.ReportRetrievalGapAsync(dataSource, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."), dataSource.Name));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."), dataSource.Name));
|
||||
return [];
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
|
||||
await this.ReportRetrievalGapAsync(dataSource, "vector-search-failed", string.Format(TB("The data source '{0}' was left out of the answer because searching it failed."), dataSource.Name));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "vector-search-failed", string.Format(TB("The data source '{0}' was left out of the answer because searching it failed."), dataSource.Name));
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tells the user once that a data source cannot take part in answering.
|
||||
/// Records that a data source cannot fully take part in answering, and tells the user once.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A failed search is not an error of the chat: the model still answers, only without what
|
||||
/// this data source knows. Saying so once is what keeps somebody from trusting an answer
|
||||
/// which was put together without half of its sources. Saying it with every prompt would be
|
||||
/// worse than saying nothing, which is why every gap is reported once per session.
|
||||
///
|
||||
/// The retrieval records every gap regardless, cf. RetrievalPage.Gaps: whoever asked for the
|
||||
/// page has to know each time, not once per session.
|
||||
/// </remarks>
|
||||
/// <param name="dataSource">The data source which could not be searched.</param>
|
||||
/// <param name="run">The retrieval this gap belongs to.</param>
|
||||
/// <param name="gap">What the gap means for the search.</param>
|
||||
/// <param name="gapKey">What kind of gap this is, so a different problem is reported again.</param>
|
||||
/// <param name="userMessage">What to tell the user.</param>
|
||||
private async Task ReportRetrievalGapAsync(IInternalDataSource dataSource, string gapKey, string userMessage)
|
||||
private async Task ReportRetrievalGapAsync(IInternalDataSource dataSource, RetrievalRun run, RetrievalGap gap, string gapKey, string userMessage)
|
||||
{
|
||||
run.Add(gap);
|
||||
if (!IsForTheUser(gap, run.QueryWrittenByUser))
|
||||
return;
|
||||
|
||||
lock (this.retrievalGapLock)
|
||||
{
|
||||
if (!this.reportedRetrievalGaps.Add($"{dataSource.Id}::{gapKey}"))
|
||||
@ -245,23 +282,38 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.SearchOff, userMessage));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the user has to hear about a gap.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Problems of the data source are for the user, since only the user can fix them. Problems of
|
||||
/// the query are for whoever wrote it. When the model worked the query out, telling the user
|
||||
/// their message was too long would be wrong, and the model learns about it from the page and
|
||||
/// can search with a shorter one.
|
||||
/// </remarks>
|
||||
/// <param name="gap">What the gap means for the search.</param>
|
||||
/// <param name="queryWrittenByUser">Whether the query is the user's own message.</param>
|
||||
/// <returns>True when the user has to be told.</returns>
|
||||
internal static bool IsForTheUser(RetrievalGap gap, bool queryWrittenByUser) => gap is not RetrievalGap.QUERY_NOT_SEARCHABLE || queryWrittenByUser;
|
||||
|
||||
private async Task<bool> QueryFitsEmbeddingProviderAsync(
|
||||
IInternalDataSource dataSource,
|
||||
EmbeddingProvider embeddingProvider,
|
||||
string query,
|
||||
RetrievalRun run,
|
||||
CancellationToken token)
|
||||
{
|
||||
var providerTokenLimit = Math.Max(1, embeddingProvider.EffectiveTokenLimit);
|
||||
if (query.Length > RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the latest prompt has {CharacterCount} characters and exceeds the safe tokenizer request length of {MaxCharacterCount}. ProviderTokenLimit={ProviderTokenLimit}.",
|
||||
"Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the query has {CharacterCount} characters and exceeds the safe tokenizer request length of {MaxCharacterCount}. ProviderTokenLimit={ProviderTokenLimit}.",
|
||||
dataSource.Name,
|
||||
dataSource.Id,
|
||||
query.Length,
|
||||
RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH,
|
||||
providerTokenLimit);
|
||||
await this.ReportRetrievalGapAsync(dataSource, "query-too-long", string.Format(TB("The data source '{0}' was left out of the answer because your message is too long to search with."), dataSource.Name));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.QUERY_NOT_SEARCHABLE, "query-too-long", string.Format(TB("The data source '{0}' was left out of the answer because your message is too long to search with."), dataSource.Name));
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -274,7 +326,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
dataSource.Id,
|
||||
embeddingProvider.Name,
|
||||
tokenCountResponse?.Message ?? "No response was returned by the tokenizer service.");
|
||||
await this.ReportRetrievalGapAsync(dataSource, "no-token-count", string.Format(TB("The data source '{0}' was left out of the answer: the tokenizer of its embedding provider '{1}' is not available."), dataSource.Name, embeddingProvider.Name));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.PARTLY_SEARCHED, "no-token-count", string.Format(TB("The data source '{0}' was left out of the answer: the tokenizer of its embedding provider '{1}' is not available."), dataSource.Name, embeddingProvider.Name));
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -282,20 +334,20 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
if (queryTokenCount > providerTokenLimit)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the latest prompt has {QueryTokenCount} tokens, exceeding embedding provider '{EmbeddingProviderName}' limit of {ProviderTokenLimit} tokens.",
|
||||
"Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the query has {QueryTokenCount} tokens, exceeding embedding provider '{EmbeddingProviderName}' limit of {ProviderTokenLimit} tokens.",
|
||||
dataSource.Name,
|
||||
dataSource.Id,
|
||||
queryTokenCount,
|
||||
embeddingProvider.Name,
|
||||
providerTokenLimit);
|
||||
await this.ReportRetrievalGapAsync(dataSource, "query-over-token-limit", string.Format(TB("The data source '{0}' was left out of the answer because your message is longer than its embedding provider '{1}' accepts."), dataSource.Name, embeddingProvider.Name));
|
||||
await this.ReportRetrievalGapAsync(dataSource, run, RetrievalGap.QUERY_NOT_SEARCHABLE, "query-over-token-limit", string.Format(TB("The data source '{0}' was left out of the answer because your message is longer than its embedding provider '{1}' accepts."), dataSource.Name, embeddingProvider.Name));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<IndexStoreSearchResult>> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token)
|
||||
private async Task<IReadOnlyList<IndexStoreSearchResult>> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, RetrievalRun run, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
@ -307,6 +359,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
dataSource.Name,
|
||||
dataSource.Id,
|
||||
indexStore.Name);
|
||||
run.Add(RetrievalGap.PARTLY_SEARCHED);
|
||||
return [];
|
||||
}
|
||||
|
||||
@ -325,6 +378,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "BM25 retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
|
||||
run.Add(RetrievalGap.PARTLY_SEARCHED);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
38
app/Tests/Tools/RetrievalGapTests.cs
Normal file
38
app/Tests/Tools/RetrievalGapTests.cs
Normal file
@ -0,0 +1,38 @@
|
||||
using AIStudio.Tools.RAG;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Checks who hears about what kept a search from covering a local data source.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// With Semantic Search, the model writes the query, not the user. A warning that the message was
|
||||
/// too long to search with would then blame the user for a query they never wrote, while the model,
|
||||
/// which could search with a shorter one, would learn nothing. Problems of the data source itself
|
||||
/// stay with the user either way, since nobody else can fix a missing embedding provider.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class RetrievalGapTests
|
||||
{
|
||||
[Test]
|
||||
public void AQueryTheModelWroteIsNotTheUsersProblem()
|
||||
{
|
||||
Assert.That(DataSourceLocalRetrievalService.IsForTheUser(RetrievalGap.QUERY_NOT_SEARCHABLE, queryWrittenByUser: false), Is.False, "The model learns about it from the page and can search with a shorter query.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMessageTheUserWroteIsTheirsToShorten()
|
||||
{
|
||||
Assert.That(DataSourceLocalRetrievalService.IsForTheUser(RetrievalGap.QUERY_NOT_SEARCHABLE, queryWrittenByUser: true), Is.True);
|
||||
}
|
||||
|
||||
[TestCase(RetrievalGap.NOT_SEARCHED, false)]
|
||||
[TestCase(RetrievalGap.NOT_SEARCHED, true)]
|
||||
[TestCase(RetrievalGap.PARTLY_SEARCHED, false)]
|
||||
[TestCase(RetrievalGap.PARTLY_SEARCHED, true)]
|
||||
public void ProblemsOfTheDataSourceAreAlwaysForTheUser(RetrievalGap gap, bool queryWrittenByUser)
|
||||
{
|
||||
Assert.That(DataSourceLocalRetrievalService.IsForTheUser(gap, queryWrittenByUser), Is.True, "Only the user can fix an index or an embedding provider.");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user