Added a retrieval context validation agent (#289)

README.md

@@ -20,7 +20,7 @@ Things we are currently working on:
   - [ ] Runtime: Integration of the vector database [LanceDB](https://github.com/lancedb/lancedb)
   - [ ] App: Implement the continuous process of vectorizing data
   - [x] ~~App: Define a common retrieval context interface for the integration of RAG processes in chats (PR [#281](https://github.com/MindWorkAI/AI-Studio/pull/281), [#284](https://github.com/MindWorkAI/AI-Studio/pull/284), [#286](https://github.com/MindWorkAI/AI-Studio/pull/286), [#287](https://github.com/MindWorkAI/AI-Studio/pull/287))~~
-  - [x] ~~App: Define a common augmentation interface for the integration of RAG processes in chats (PR [#288](https://github.com/MindWorkAI/AI-Studio/pull/288))~~
+  - [x] ~~App: Define a common augmentation interface for the integration of RAG processes in chats (PR [#288](https://github.com/MindWorkAI/AI-Studio/pull/288), [#289](https://github.com/MindWorkAI/AI-Studio/pull/289))~~
   - [x] ~~App: Integrate data sources in chats (PR [#282](https://github.com/MindWorkAI/AI-Studio/pull/282))~~
 
 

app/MindWork AI Studio/Agents/AgentBase.cs

@@ -11,6 +11,14 @@ namespace AIStudio.Agents;
 
 public abstract class AgentBase(ILogger<AgentBase> logger, SettingsManager settingsManager, DataSourceService dataSourceService, ThreadSafeRandom rng) : IAgent
 {
+    protected static readonly ContentBlock EMPTY_BLOCK = new()
+    {
+        Content = null,
+        ContentType = ContentType.NONE,
+        Role = ChatRole.AGENT,
+        Time = DateTimeOffset.UtcNow,
+    };
+    
     protected static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
     {
         PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
@@ -24,8 +32,6 @@ public abstract class AgentBase(ILogger<AgentBase> logger, SettingsManager setti
     
     protected ILogger<AgentBase> Logger { get; init; } = logger;
     
-    protected IContent? lastUserPrompt;
-
     /// <summary>
     /// Represents the type or category of this agent.
     /// </summary>
@@ -72,10 +78,10 @@ public abstract class AgentBase(ILogger<AgentBase> logger, SettingsManager setti
         Blocks = [],
     };
 
-    protected DateTimeOffset AddUserRequest(ChatThread thread, string request)
+    protected UserRequest AddUserRequest(ChatThread thread, string request)
     {
         var time = DateTimeOffset.Now;
-        this.lastUserPrompt = new ContentText
+        var lastUserPrompt = new ContentText
         {
             Text = request,
         };
@@ -85,13 +91,17 @@ public abstract class AgentBase(ILogger<AgentBase> logger, SettingsManager setti
             Time = time,
             ContentType = ContentType.TEXT,
             Role = ChatRole.USER,
-            Content = this.lastUserPrompt,
+            Content = lastUserPrompt,
         });
-        
+
-        return time;
+        return new()
+        {
+            Time = time,
+            UserPrompt = lastUserPrompt,
+        };
     }
     
-    protected async Task AddAIResponseAsync(ChatThread thread, DateTimeOffset time)
+    protected async Task AddAIResponseAsync(ChatThread thread, IContent lastUserPrompt, DateTimeOffset time)
     {
         if(this.ProviderSettings is null)
             return;
@@ -117,6 +127,6 @@ public abstract class AgentBase(ILogger<AgentBase> logger, SettingsManager setti
         // Use the selected provider to get the AI response.
         // By awaiting this line, we wait for the entire
         // content to be streamed.
-        await aiText.CreateFromProviderAsync(providerSettings.CreateProvider(this.Logger), providerSettings.Model, this.lastUserPrompt, thread);
+        await aiText.CreateFromProviderAsync(providerSettings.CreateProvider(this.Logger), providerSettings.Model, lastUserPrompt, thread);
     }
 }

app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs

@@ -12,14 +12,6 @@ namespace AIStudio.Agents;
 
 public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection> logger, ILogger<AgentBase> baseLogger, SettingsManager settingsManager, DataSourceService dataSourceService, ThreadSafeRandom rng) : AgentBase(baseLogger, settingsManager, dataSourceService, rng)
 {
-    private static readonly ContentBlock EMPTY_BLOCK = new()
-    {
-        Content = null,
-        ContentType = ContentType.NONE,
-        Role = ChatRole.AGENT,
-        Time = DateTimeOffset.UtcNow,
-    };
-    
     private readonly List<ContentBlock> answers = new();
     
     #region Overrides of AgentBase
@@ -119,8 +111,8 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
             return EMPTY_BLOCK;
         
         var thread = this.CreateChatThread(this.SystemPrompt(availableDataSources));
-        var time = this.AddUserRequest(thread, text.Text);
+        var userRequest = this.AddUserRequest(thread, text.Text);
-        await this.AddAIResponseAsync(thread, time);
+        await this.AddAIResponseAsync(thread, userRequest.UserPrompt, userRequest.Time);
         
         var answer = thread.Blocks[^1];
         
@@ -306,7 +298,7 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
                 // We know how bad LLM may be in generating JSON without surrounding text.
                 // Thus, we expect the worst and try to extract the JSON list from the text:
                 //
-                var json = this.ExtractJson(selectedDataSourcesJson);
+                var json = ExtractJson(selectedDataSourcesJson);
                 
                 try
                 {
@@ -352,7 +344,7 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
     /// </remarks>
     /// <param name="text">The text that may contain the JSON list.</param>
     /// <returns>The extracted JSON list.</returns>
-    private string ExtractJson(string text) => ExtractJson(text.AsSpan()).ToString();
+    private static string ExtractJson(string text) => ExtractJson(text.AsSpan()).ToString();
 
     /// <summary>
     /// Extracts the JSON list from the given text. The text may contain additional

app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs

@@ -0,0 +1,393 @@
+using System.Text.Json;
+
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Agents;
+
+public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalContextValidation> logger, ILogger<AgentBase> baseLogger, SettingsManager settingsManager, DataSourceService dataSourceService, ThreadSafeRandom rng) : AgentBase(baseLogger, settingsManager, dataSourceService, rng)
+{
+    #region Overrides of AgentBase
+
+    /// <inheritdoc />
+    protected override Type Type => Type.WORKER;
+    
+    /// <inheritdoc />
+    public override string Id => "Retrieval Context Validation";
+
+    /// <inheritdoc />
+    protected override string JobDescription =>
+        """
+        You receive a system and user prompt as well as a retrieval context as input. Your task is to decide whether this
+        retrieval context is helpful in processing the prompts or not. You respond with the decision (true or false),
+        your reasoning, and your confidence in this decision.
+
+        Your response is only one JSON object in the following format:
+
+        ```
+        {"decision": true, "reason": "Why did you choose this source?", "confidence": 0.87}
+        ```
+
+        You express your confidence as a floating-point number between 0.0 (maximum uncertainty) and
+        1.0 (you are absolutely certain that this retrieval context is needed).
+
+        The JSON schema is:
+
+        ```
+        {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "type": "object",
+          "properties": {
+            "decision": {
+              "type": "boolean"
+            },
+            "reason": {
+              "type": "string"
+            },
+            "confidence": {
+              "type": "number"
+            }
+          },
+          "required": [
+            "decision",
+            "reason",
+            "confidence"
+          ]
+        }
+        ```
+
+        You do not ask any follow-up questions. You do not address the user. Your response consists solely of
+        that one JSON object.
+        """;
+    
+    /// <inheritdoc />
+    protected override string SystemPrompt(string retrievalContext) => $"""
+                                                                        {this.JobDescription}
+
+                                                                        {retrievalContext}
+                                                                        """;
+
+    /// <inheritdoc />
+    public override Settings.Provider? ProviderSettings { get; set; }
+    
+    /// <summary>
+    /// The retrieval context validation agent does not work with context. Use
+    /// the process input method instead.
+    /// </summary>
+    /// <returns>The chat thread without any changes.</returns>
+    public override Task<ChatThread> ProcessContext(ChatThread chatThread, IDictionary<string, string> additionalData) => Task.FromResult(chatThread);
+
+    /// <inheritdoc />
+    public override async Task<ContentBlock> ProcessInput(ContentBlock input, IDictionary<string, string> additionalData)
+    {
+        if (input.Content is not ContentText text)
+            return EMPTY_BLOCK;
+          
+        if(text.InitialRemoteWait || text.IsStreaming)
+            return EMPTY_BLOCK;
+          
+        if(string.IsNullOrWhiteSpace(text.Text))
+            return EMPTY_BLOCK;
+          
+        if(!additionalData.TryGetValue("retrievalContext", out var retrievalContext) || string.IsNullOrWhiteSpace(retrievalContext))
+            return EMPTY_BLOCK;
+          
+        var thread = this.CreateChatThread(this.SystemPrompt(retrievalContext));
+        var userRequest = this.AddUserRequest(thread, text.Text);
+        await this.AddAIResponseAsync(thread, userRequest.UserPrompt, userRequest.Time);
+          
+        return thread.Blocks[^1];
+    }
+
+    /// <inheritdoc />
+    public override Task<bool> MadeDecision(ContentBlock input) => Task.FromResult(true);
+
+    /// <summary>
+    /// We do not provide any context. This agent will process many retrieval contexts.
+    /// This would block a huge amount of memory.
+    /// </summary>
+    /// <returns>An empty list.</returns>
+    public override IReadOnlyCollection<ContentBlock> GetContext() => [];
+
+    /// <summary>
+    /// We do not provide any answers. This agent will process many retrieval contexts.
+    /// This would block a huge amount of memory.
+    /// </summary>
+    /// <returns>An empty list.</returns>
+    public override IReadOnlyCollection<ContentBlock> GetAnswers() => [];
+
+    #endregion
+
+    /// <summary>
+    /// Sets the LLM provider for the agent.
+    /// </summary>
+    /// <remarks>
+    /// When you have to call the validation in parallel for many retrieval contexts,
+    /// you can set the provider once and then call the validation method in parallel.
+    /// </remarks>
+    /// <param name="provider">The current LLM provider. When the user doesn't preselect an agent provider, the agent uses this provider.</param>
+    public void SetLLMProvider(IProvider provider)
+    {
+        // We start with the provider currently selected by the user:
+        var agentProvider = this.SettingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == provider.Id);
+
+        // If the user preselected an agent provider, we try to use this one:
+        if (this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectAgentOptions)
+        {
+            var configuredAgentProvider = this.SettingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectedAgentProvider);
+
+            // If the configured agent provider is available, we use it:
+            if (configuredAgentProvider != default)
+                agentProvider = configuredAgentProvider;
+        }
+
+        // Assign the provider settings to the agent:
+        logger.LogInformation($"The agent for the retrieval context validation uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()}).");
+        this.ProviderSettings = agentProvider;
+    }
+    
+    /// <summary>
+    /// Validate all retrieval contexts against the last user and the system prompt.
+    /// </summary>
+    /// <param name="lastPrompt">The last user prompt.</param>
+    /// <param name="chatThread">The chat thread.</param>
+    /// <param name="retrievalContexts">All retrieval contexts to validate.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The validation results.</returns>
+    public async Task<IReadOnlyList<RetrievalContextValidationResult>> ValidateRetrievalContextsAsync(IContent lastPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default)
+    {
+        // Check if the retrieval context validation is enabled:
+        if (!this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation)
+            return [];
+        
+        // Prepare the list of validation tasks:
+        var validationTasks = new List<Task<RetrievalContextValidationResult>>(retrievalContexts.Count);
+        
+        // Read the number of parallel validations:
+        var numParallelValidations = this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.NumParallelValidations;
+        numParallelValidations = Math.Max(1, numParallelValidations);
+        
+        // Use a semaphore to limit the number of parallel validations:
+        using var semaphore = new SemaphoreSlim(numParallelValidations);
+        foreach (var retrievalContext in retrievalContexts)
+        {
+            // Wait for an available slot in the semaphore:
+            await semaphore.WaitAsync(token);
+            
+            // Start the next validation task:
+            validationTasks.Add(this.ValidateRetrievalContextAsync(lastPrompt, chatThread, retrievalContext, token, semaphore));
+        }
+
+        // Wait for all validation tasks to complete:
+        return await Task.WhenAll(validationTasks);
+    }
+
+    /// <summary>
+    /// Validates the retrieval context against the last user and the system prompt.
+    /// </summary>
+    /// <remarks>
+    /// Probably, you have a lot of retrieval contexts to validate. In this case, you
+    /// can call this method in parallel for each retrieval context. You might use
+    /// the ValidateRetrievalContextsAsync method to validate all retrieval contexts.
+    /// </remarks>
+    /// <param name="lastPrompt">The last user prompt.</param>
+    /// <param name="chatThread">The chat thread.</param>
+    /// <param name="retrievalContext">The retrieval context to validate.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <param name="semaphore">The optional semaphore to limit the number of parallel validations.</param>
+    /// <returns>The validation result.</returns>
+    public async Task<RetrievalContextValidationResult> ValidateRetrievalContextAsync(IContent lastPrompt, ChatThread chatThread, IRetrievalContext retrievalContext, CancellationToken token = default, SemaphoreSlim? semaphore = null)
+    {
+        try
+        {
+            //
+            // Check if the validation was canceled. This could happen when the user
+            // canceled the validation process or when the validation process took
+            // too long:
+            //
+            if(token.IsCancellationRequested)
+                return new(false, "The validation was canceled.", 1.0f, retrievalContext);
+            
+            //
+            // 1. Prepare the current system and user prompts as input for the agent: 
+            //
+            var lastPromptContent = lastPrompt switch
+            {
+                ContentText text => text.Text,
+
+                // Image prompts may be empty, e.g., when the image is too large:
+                ContentImage image => await image.AsBase64(token),
+
+                // Other content types are not supported yet:
+                _ => string.Empty,
+            };
+
+            if (string.IsNullOrWhiteSpace(lastPromptContent))
+            {
+                logger.LogWarning("The last prompt is empty. The AI cannot validate the retrieval context.");
+                return new(false, "The last prompt was empty.", 1.0f, retrievalContext);
+            }
+
+            //
+            // 2. Prepare the retrieval context for the agent:
+            //
+            var additionalData = new Dictionary<string, string>();
+            var markdownRetrievalContext = await retrievalContext.AsMarkdown(token: token);
+            additionalData.Add("retrievalContext", markdownRetrievalContext);
+
+            //
+            // 3. Let the agent validate the retrieval context:
+            //
+            var prompt = $"""
+                          The system prompt is:
+
+                          ```
+                          {chatThread.SystemPrompt}
+                          ```
+
+                          The user prompt is:
+
+                          ```
+                          {lastPromptContent}
+                          ```
+                          """;
+
+            // Call the agent:
+            var aiResponse = await this.ProcessInput(new ContentBlock
+            {
+                Time = DateTimeOffset.UtcNow,
+                ContentType = ContentType.TEXT,
+                Role = ChatRole.USER,
+                Content = new ContentText
+                {
+                    Text = prompt,
+                },
+            }, additionalData);
+
+            if (aiResponse.Content is null)
+            {
+                logger.LogWarning("The agent did not return a response.");
+                return new(false, "The agent did not return a response.", 1.0f, retrievalContext);
+            }
+
+            switch (aiResponse)
+            {
+
+                //
+                // 4. Parse the agent response:
+                //
+                case { ContentType: ContentType.TEXT, Content: ContentText textContent }:
+                {
+                    //
+                    // What we expect is one JSON object:
+                    //
+                    var validationJson = textContent.Text;
+
+                    //
+                    // We know how bad LLM may be in generating JSON without surrounding text.
+                    // Thus, we expect the worst and try to extract the JSON list from the text:
+                    //
+                    var json = ExtractJson(validationJson);
+
+                    try
+                    {
+                        var result = JsonSerializer.Deserialize<RetrievalContextValidationResult>(json, JSON_SERIALIZER_OPTIONS);
+                        return result with { RetrievalContext = retrievalContext };
+                    }
+                    catch
+                    {
+                        logger.LogWarning("The agent answered with an invalid or unexpected JSON format.");
+                        return new(false, "The agent answered with an invalid or unexpected JSON format.", 1.0f, retrievalContext);
+                    }
+                }
+
+                case { ContentType: ContentType.TEXT }:
+                    logger.LogWarning("The agent answered with an unexpected inner content type.");
+                    return new(false, "The agent answered with an unexpected inner content type.", 1.0f, retrievalContext);
+
+                case { ContentType: ContentType.NONE }:
+                    logger.LogWarning("The agent did not return a response.");
+                    return new(false, "The agent did not return a response.", 1.0f, retrievalContext);
+
+                default:
+                    logger.LogWarning($"The agent answered with an unexpected content type '{aiResponse.ContentType}'.");
+                    return new(false, $"The agent answered with an unexpected content type '{aiResponse.ContentType}'.", 1.0f, retrievalContext);
+            }
+        }
+        finally
+        {
+            // Release the semaphore slot:
+            semaphore?.Release();
+        }
+    }
+    
+    // A wrapper around the span version, because we need to call this method from an async context.
+    private static string ExtractJson(string text) => ExtractJson(text.AsSpan()).ToString();
+    
+    private static ReadOnlySpan<char> ExtractJson(ReadOnlySpan<char> input)
+    {
+        //
+        // 1. Expect the best case ;-)
+        //
+        if (CheckJsonObjectStart(input))
+            return ExtractJsonPart(input);
+
+        //
+        // 2. Okay, we have some garbage before the
+        // JSON object. We expected that...
+        //
+        for (var index = 0; index < input.Length; index++)
+        {
+            if (input[index] is '{' && CheckJsonObjectStart(input[index..]))
+                return ExtractJsonPart(input[index..]);
+        }
+
+        return [];
+    }
+    
+    private static bool CheckJsonObjectStart(ReadOnlySpan<char> area)
+    {
+        char[] expectedSymbols = ['{', '"', 'd'];
+        var symbolIndex = 0;
+
+        foreach (var c in area)
+        {
+            if (symbolIndex >= expectedSymbols.Length)
+                return true;
+
+            if (char.IsWhiteSpace(c))
+                continue;
+
+            if (c == expectedSymbols[symbolIndex++])
+                continue;
+
+            return false;
+        }
+
+        return true;
+    }
+
+    private static ReadOnlySpan<char> ExtractJsonPart(ReadOnlySpan<char> input)
+    {
+        var insideString = false;
+        for (var index = 0; index < input.Length; index++)
+        {
+            if (input[index] is '"')
+            {
+                insideString = !insideString;
+                continue;
+            }
+
+            if (insideString)
+                continue;
+
+            if (input[index] is '}')
+                return input[..++index];
+        }
+
+        return [];
+    }
+}

app/MindWork AI Studio/Agents/AgentTextContentCleaner.cs

@@ -6,14 +6,6 @@ namespace AIStudio.Agents;
 
 public sealed class AgentTextContentCleaner(ILogger<AgentBase> logger, SettingsManager settingsManager, DataSourceService dataSourceService, ThreadSafeRandom rng) : AgentBase(logger, settingsManager, dataSourceService, rng)
 {
-    private static readonly ContentBlock EMPTY_BLOCK = new()
-    {
-        Content = null,
-        ContentType = ContentType.NONE,
-        Role = ChatRole.AGENT,
-        Time = DateTimeOffset.UtcNow,
-    };
-    
     private readonly List<ContentBlock> context = new();
     private readonly List<ContentBlock> answers = new();
     
@@ -73,8 +65,8 @@ public sealed class AgentTextContentCleaner(ILogger<AgentBase> logger, SettingsM
             return EMPTY_BLOCK;
         
         var thread = this.CreateChatThread(this.SystemPrompt(sourceURL));
-        var time = this.AddUserRequest(thread, text.Text);
+        var userRequest = this.AddUserRequest(thread, text.Text);
-        await this.AddAIResponseAsync(thread, time);
+        await this.AddAIResponseAsync(thread, userRequest.UserPrompt, userRequest.Time);
         
         var answer = thread.Blocks[^1];
         this.answers.Add(answer);

app/MindWork AI Studio/Agents/RetrievalContextValidationResult.cs

@@ -0,0 +1,12 @@
+using AIStudio.Tools.RAG;
+
+namespace AIStudio.Agents;
+
+/// <summary>
+/// Represents the result of a retrieval context validation.
+/// </summary>
+/// <param name="Decision">Whether the retrieval context is useful or not.</param>
+/// <param name="Reason">The reason for the decision.</param>
+/// <param name="Confidence">The confidence of the decision.</param>
+/// <param name="RetrievalContext">The retrieval context that was validated.</param>
+public readonly record struct RetrievalContextValidationResult(bool Decision, string Reason, float Confidence, IRetrievalContext? RetrievalContext) : IConfidence;

app/MindWork AI Studio/Agents/SelectedDataSource.cs

@@ -6,4 +6,4 @@ namespace AIStudio.Agents;
 /// <param name="Id">The data source ID.</param>
 /// <param name="Reason">The reason for selecting the data source.</param>
 /// <param name="Confidence">The confidence of the agent in the selection.</param>
-public readonly record struct SelectedDataSource(string Id, string Reason, float Confidence);
+public readonly record struct SelectedDataSource(string Id, string Reason, float Confidence) : IConfidence;

app/MindWork AI Studio/Agents/UserRequest.cs

@@ -0,0 +1,19 @@
+using AIStudio.Chat;
+
+namespace AIStudio.Agents;
+
+/// <summary>
+/// The created user request.
+/// </summary>
+public sealed class UserRequest
+{
+    /// <summary>
+    /// The time when the request was created.
+    /// </summary>
+    public required DateTimeOffset Time { get; init; }
+
+    /// <summary>
+    /// The user prompt.
+    /// </summary>
+    public required IContent UserPrompt { get; init; }
+}

app/MindWork AI Studio/Components/Settings/SettingsPanelAgentRetrievalContextValidation.razor

@@ -0,0 +1,18 @@
+@inherits SettingsPanelBase
+
+<ExpansionPanel HeaderIcon="@Icons.Material.Filled.Assessment" HeaderText="Agent: Retrieval Context Validation Options">
+    <MudText Typo="Typo.body1" Class="mb-3">
+        Use Case: this agent is used to validate any retrieval context of any retrieval process. Perhaps there are many of these
+        retrieval contexts and you want to validate them all. Therefore, you might want to use a cheap and fast LLM for this
+        job. When using a local or self-hosted LLM, look for a small (e.g. 3B) and fast model.
+    </MudText>
+    <ConfigurationOption OptionDescription="Enable the retrieval context validation agent?" LabelOn="The validation agent is enabled" LabelOff="No validation is performed" State="@(() => this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation = updatedState)" OptionHelp="When enabled, the retrieval context validation agent will check each retrieval context of any retrieval process, whether a context makes sense for the given prompt."/>
+    @if (this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation)
+    {
+        <MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
+            <ConfigurationOption OptionDescription="Preselect retrieval context validation options?" LabelOn="Options are preselected" LabelOff="No options are preselected" State="@(() => this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectAgentOptions)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectAgentOptions = updatedState)" OptionHelp="When enabled, you can preselect some agent options. This is might be useful when you prefer a LLM."/>
+            <ConfigurationSlider T="int" OptionDescription="How many validation agents should work simultaneously?" Min="1" Max="100" Step="1" Unit="agents" Value="@(() => this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.NumParallelValidations)" ValueUpdate="@(updatedValue => this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.NumParallelValidations = updatedValue)" OptionHelp="More active agents also mean that a corresponding number of requests are made simultaneously. Some providers limit the number of requests per minute. When you are unsure, choose a low setting between 1 to 6 agents."/>
+            <ConfigurationProviderSelection Data="@this.AvailableLLMProvidersFunc()" Disabled="@(() => !this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectAgentOptions)" SelectedValue="@(() => this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectedAgentProvider)" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectedAgentProvider = selectedValue)"/>
+        </MudPaper>
+    }
+</ExpansionPanel>

app/MindWork AI Studio/Components/Settings/SettingsPanelAgentRetrievalContextValidation.razor.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Components.Settings;
+
+public partial class SettingsPanelAgentRetrievalContextValidation : SettingsPanelBase;

app/MindWork AI Studio/Pages/Settings.razor

@@ -1,5 +1,6 @@
 @attribute [Route(Routes.SETTINGS)]
 @using AIStudio.Components.Settings
+@using AIStudio.Settings.DataModel
 
 <div class="inner-scrolling-context">
     <MudText Typo="Typo.h3" Class="mb-12">Settings</MudText>
@@ -7,8 +8,12 @@
     <InnerScrolling>
         <MudExpansionPanels Class="mb-3" MultiExpansion="@false">
             <SettingsPanelProviders @bind-AvailableLLMProviders="@this.availableLLMProviders" />
-            <SettingsPanelEmbeddings AvailableLLMProvidersFunc="() => this.availableLLMProviders" @bind-AvailableEmbeddingProviders="@this.availableEmbeddingProviders" />
+            
-            <SettingsPanelDataSources AvailableLLMProvidersFunc="() => this.availableLLMProviders" AvailableEmbeddingsFunc="() => this.availableEmbeddingProviders" @bind-AvailableDataSources="@this.availableDataSources" />
+            @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
+            {
+                <SettingsPanelEmbeddings AvailableLLMProvidersFunc="() => this.availableLLMProviders" @bind-AvailableEmbeddingProviders="@this.availableEmbeddingProviders"/>
+                <SettingsPanelDataSources AvailableLLMProvidersFunc="() => this.availableLLMProviders" AvailableEmbeddingsFunc="() => this.availableEmbeddingProviders" @bind-AvailableDataSources="@this.availableDataSources"/>
+            }
             <SettingsPanelProfiles AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
             <SettingsPanelApp AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
             <SettingsPanelChat AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
@@ -16,7 +21,12 @@
             <SettingsPanelIconFinder AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
             <SettingsPanelTranslation AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
             <SettingsPanelCoding AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
-            <SettingsPanelERIServer AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
+            
+            @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
+            {
+                <SettingsPanelERIServer AvailableLLMProvidersFunc="() => this.availableLLMProviders"/>
+            }
+            
             <SettingsPanelTextSummarizer AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
             <SettingsPanelAgenda AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
             <SettingsPanelGrammarSpelling AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
@@ -27,7 +37,13 @@
             <SettingsPanelSynonyms AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
             <SettingsPanelMyTasks AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
             <SettingsPanelAssistantBias AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
-            <SettingsPanelAgentDataSourceSelection AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
+            
+            @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
+            {
+                <SettingsPanelAgentDataSourceSelection AvailableLLMProvidersFunc="() => this.availableLLMProviders"/>
+                <SettingsPanelAgentRetrievalContextValidation AvailableLLMProvidersFunc="() => this.availableLLMProviders"/>
+            }
+            
             <SettingsPanelAgentContentCleaner AvailableLLMProvidersFunc="() => this.availableLLMProviders" />
         </MudExpansionPanels>
     </InnerScrolling>

app/MindWork AI Studio/Pages/Settings.razor.cs

@@ -6,6 +6,9 @@ namespace AIStudio.Pages;
 
 public partial class Settings : ComponentBase, IMessageBusReceiver, IDisposable
 {
+    [Inject]
+    private SettingsManager SettingsManager { get; init; } = null!;
+    
     [Inject]
     private MessageBus MessageBus { get; init; } = null!;
     

app/MindWork AI Studio/Program.cs

@@ -119,6 +119,7 @@ internal sealed class Program
         builder.Services.AddSingleton<DataSourceService>();
         builder.Services.AddTransient<HTMLParser>();
         builder.Services.AddTransient<AgentDataSourceSelection>();
+        builder.Services.AddTransient<AgentRetrievalContextValidation>();
         builder.Services.AddTransient<AgentTextContentCleaner>();
         builder.Services.AddHostedService<UpdateService>();
         builder.Services.AddHostedService<TemporaryChatService>();

app/MindWork AI Studio/Settings/DataModel/Data.cs

@@ -76,6 +76,8 @@ public sealed class Data
     
     public DataAgentDataSourceSelection AgentDataSourceSelection { get; init; } = new();
     
+    public DataAgentRetrievalContextValidation AgentRetrievalContextValidation { get; init; } = new();
+    
     public DataAgenda Agenda { get; init; } = new();
     
     public DataGrammarSpelling GrammarSpelling { get; init; } = new();

app/MindWork AI Studio/Settings/DataModel/DataAgentDataSourceSelection.cs

@@ -3,12 +3,12 @@ namespace AIStudio.Settings.DataModel;
 public sealed class DataAgentDataSourceSelection
 {
     /// <summary>
-    /// Preselect any text content cleaner options?
+    /// Preselect any data source selection options?
     /// </summary>
     public bool PreselectAgentOptions { get; set; }
     
     /// <summary>
-    /// Preselect a text content cleaner provider?
+    /// Preselect a data source selection provider?
     /// </summary>
     public string PreselectedAgentProvider { get; set; } = string.Empty;
 }

app/MindWork AI Studio/Settings/DataModel/DataAgentRetrievalContextValidation.cs

@@ -0,0 +1,24 @@
+namespace AIStudio.Settings.DataModel;
+
+public sealed class DataAgentRetrievalContextValidation
+{
+    /// <summary>
+    /// Enable the retrieval context validation agent?
+    /// </summary>
+    public bool EnableRetrievalContextValidation { get; set; }
+    
+    /// <summary>
+    /// Preselect any retrieval context validation options?
+    /// </summary>
+    public bool PreselectAgentOptions { get; set; }
+    
+    /// <summary>
+    /// Preselect a retrieval context validation provider?
+    /// </summary>
+    public string PreselectedAgentProvider { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Configure how many parallel validations to run.
+    /// </summary>
+    public int NumParallelValidations { get; set; } = 3;
+}

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -95,7 +95,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                                 Path = eriContext.Path ?? string.Empty,
                                 Type = eriContext.ToRetrievalContentType(),
                                 Links = eriContext.Links,
-                                Category = RetrievalContentCategory.TEXT,
+                                Category = eriContext.Type.ToRetrievalContentCategory(),
                                 MatchedText = eriContext.MatchedContent,
                                 DataSourceName = eriContext.Name,
                                 SurroundingContent = eriContext.SurroundingContent,
@@ -109,7 +109,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                                 Type = eriContext.ToRetrievalContentType(),
                                 Links = eriContext.Links,
                                 Source = eriContext.MatchedContent,
-                                Category = RetrievalContentCategory.IMAGE,
+                                Category = eriContext.Type.ToRetrievalContentCategory(),
                                 SourceType = ContentImageSource.BASE64,
                                 DataSourceName = eriContext.Name,
                             });

app/MindWork AI Studio/Tools/IConfidence.cs

@@ -0,0 +1,16 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// A contract for data classes with a confidence value.
+/// </summary>
+/// <remarks>
+/// Using this confidence contract allows us to provide
+/// algorithms based on confidence values.
+/// </remarks>
+public interface IConfidence
+{
+    /// <summary>
+    /// How confident is the AI in this task or decision?
+    /// </summary>
+    public float Confidence { get; init; }
+}

app/MindWork AI Studio/Tools/IConfidenceExtensions.cs

@@ -0,0 +1,101 @@
+namespace AIStudio.Tools;
+
+public static class IConfidenceExtensions
+{
+    public static TargetWindow DetermineTargetWindow<T>(this IReadOnlyList<T> items, TargetWindowStrategy strategy, int numMaximumItems = 30) where T : IConfidence 
+    {
+        switch (strategy)
+        {
+            case TargetWindowStrategy.A_FEW_GOOD_ONES:
+                return new(1, 2, 3, 0f);
+            
+            case TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING:
+                var numItemsBetterThanGuessing = items.Count(x => x.Confidence > 0.5f);
+                if(numItemsBetterThanGuessing < 3)
+                    return new(1, 2, 3, 0.5f);
+
+                // We want the top 10% of items better than guessing:
+                var numTop10Percent = (int) MathF.Floor(numItemsBetterThanGuessing * 0.1f);
+
+                // When these 10% are just a few items, we take them all:
+                if (numTop10Percent <= 10)
+                {
+                    var diff = numItemsBetterThanGuessing - numTop10Percent;
+                    var num50Percent = (int) MathF.Floor(numItemsBetterThanGuessing * 0.5f);
+                    return new(num50Percent, num50Percent + 1, Math.Max(numItemsBetterThanGuessing, diff), 0.5f);
+                }
+
+                // Let's define the size of the window:
+                const int MIN_NUM_ITEMS = 3;
+                var windowMin = Math.Max(MIN_NUM_ITEMS + 1, numTop10Percent);
+                windowMin = Math.Min(windowMin, numMaximumItems - 1);
+                var totalMin = Math.Max(MIN_NUM_ITEMS, windowMin - 3);
+                var windowSize = (int)MathF.Max(MathF.Floor(numTop10Percent * 0.1f), MathF.Min(10, numTop10Percent));
+                var windowMax = Math.Min(numMaximumItems, numTop10Percent + windowSize);
+                return new(totalMin, windowMin, windowMax, 0.5f);
+
+            case TargetWindowStrategy.NONE:
+            default:
+                return new(-1, -1, -1, 0f);
+        }
+    }
+
+    /// <summary>
+    /// Determine the optimal confidence threshold for a list of items
+    /// in order to match a target window of number of items.
+    /// </summary>
+    /// <param name="items">The list of confidence items to analyze.</param>
+    /// <param name="targetWindow">The target window for the number of items.</param>
+    /// <param name="maxSteps">The maximum number of steps to search for the threshold.</param>
+    /// <typeparam name="T">The type of items in the list.</typeparam>
+    /// <returns>The confidence threshold.</returns>
+    public static float GetConfidenceThreshold<T>(this IReadOnlyList<T> items, TargetWindow targetWindow, int maxSteps = 10) where T : IConfidence
+    {
+        if(!targetWindow.IsValid())
+        {
+            var logger = Program.SERVICE_PROVIDER.GetService<ILogger<IConfidence>>()!;
+            logger.LogWarning("The target window is invalid. Returning 0f as threshold.");
+            return 0f;
+        }
+
+        var confidenceValues = items.Select(x => x.Confidence).ToList();
+        var minConfidence = confidenceValues.Min();
+        var lowerBound = MathF.Max(minConfidence, targetWindow.MinThreshold);
+        var upperBound = confidenceValues.Max();
+
+        //
+        // We search for a threshold so that we have between
+        // targetWindowMin and targetWindowMax items. When not
+        // possible, we take all items (e.g., threshold = 0f; depends on the used window strategy)
+        //
+        var threshold = 0.0f;
+
+        // Check the case where the confidence values are too close:
+        if (upperBound - minConfidence >= 0.01)
+        {
+            var previousThreshold = threshold;
+            for (var i = 0; i < maxSteps; i++)
+            {
+                threshold = lowerBound + (upperBound - lowerBound) * i / maxSteps;
+                var numMatches = items.Count(x => x.Confidence >= threshold);
+                if (numMatches <= targetWindow.NumMinItems)
+                {
+                    threshold = previousThreshold;
+                    break;
+                }
+
+                if (targetWindow.InsideWindow(numMatches))
+                    break;
+
+                previousThreshold = threshold;
+            }
+        }
+        else
+        {
+            var logger = Program.SERVICE_PROVIDER.GetService<ILogger<IConfidence>>()!;
+            logger.LogWarning("The confidence values are too close. Returning 0f as threshold.");
+        }
+        
+        return threshold;
+    }
+}

app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

@@ -1,7 +1,9 @@
 using System.Text;
 
+using AIStudio.Agents;
 using AIStudio.Chat;
 using AIStudio.Provider;
+using AIStudio.Settings;
 
 namespace AIStudio.Tools.RAG.AugmentationProcesses;
 
@@ -22,13 +24,36 @@ public sealed class AugmentationOne : IAugmentationProcess
     public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default)
     {
         var logger = Program.SERVICE_PROVIDER.GetService<ILogger<AugmentationOne>>()!;
+        var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
+        
         if(retrievalContexts.Count == 0)
         {
             logger.LogWarning("No retrieval contexts were issued. Skipping the augmentation process.");
             return chatThread;
         }
-        
+
         var numTotalRetrievalContexts = retrievalContexts.Count;
+        
+        // Want the user to validate all retrieval contexts?
+        if (settings.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation)
+        {
+            // Let's get the validation agent & set up its provider:
+            var validationAgent = Program.SERVICE_PROVIDER.GetService<AgentRetrievalContextValidation>()!;
+            validationAgent.SetLLMProvider(provider);
+            
+            // Let's validate all retrieval contexts:
+            var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastPrompt, chatThread, retrievalContexts, token);
+         
+            //
+            // Now, filter the retrieval contexts to the most relevant ones:
+            //
+            var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING);
+            var threshold = validationResults.GetConfidenceThreshold(targetWindow);
+            
+            // Filter the retrieval contexts:
+            retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
+        }
+        
         logger.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts.");
         
         //
@@ -38,63 +63,8 @@ public sealed class AugmentationOne : IAugmentationProcess
         sb.AppendLine("The following useful information will help you in processing the user prompt:");
         sb.AppendLine();
         
-        var index = 0;
+        // Let's convert all retrieval contexts to Markdown:
-        foreach(var retrievalContext in retrievalContexts)
+        await retrievalContexts.AsMarkdown(sb, token);
-        {
-            index++;
-            sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
-            sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
-            sb.AppendLine($"Content category: {retrievalContext.Category}");
-            sb.AppendLine($"Content type: {retrievalContext.Type}");
-            sb.AppendLine($"Content path: {retrievalContext.Path}");
-            
-            if(retrievalContext.Links.Count > 0)
-            {
-                sb.AppendLine("Additional links:");
-                foreach(var link in retrievalContext.Links)
-                    sb.AppendLine($"- {link}");
-            }
-            
-            switch(retrievalContext)
-            {
-                case RetrievalTextContext textContext:
-                    sb.AppendLine();
-                    sb.AppendLine("Matched text content:");
-                    sb.AppendLine("````");
-                    sb.AppendLine(textContext.MatchedText);
-                    sb.AppendLine("````");
-                    
-                    if(textContext.SurroundingContent.Count > 0)
-                    {
-                        sb.AppendLine();
-                        sb.AppendLine("Surrounding text content:");
-                        foreach(var surrounding in textContext.SurroundingContent)
-                        {
-                            sb.AppendLine();
-                            sb.AppendLine("````");
-                            sb.AppendLine(surrounding);
-                            sb.AppendLine("````");
-                        }
-                    }
-                    
-                    
-                    break;
-                
-                case RetrievalImageContext imageContext:
-                    sb.AppendLine();
-                    sb.AppendLine("Matched image content as base64-encoded data:");
-                    sb.AppendLine("````");
-                    sb.AppendLine(await imageContext.AsBase64(token));
-                    sb.AppendLine("````");
-                    break;
-                
-                default:
-                    logger.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
-                    break;
-            }
-            
-            sb.AppendLine();
-        }
         
         //
         // Append the entire augmentation to the chat thread,

app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs

@@ -70,41 +70,9 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
 
             if (aiSelectedDataSources.Count > 3)
             {
-                //
+                // We have more than 3 data sources. Let's filter by confidence:
-                // We have more than 3 data sources. Let's filter by confidence.
+                var targetWindow = aiSelectedDataSources.DetermineTargetWindow(TargetWindowStrategy.A_FEW_GOOD_ONES);
-                // In order to do that, we must identify the lower and upper
+                var threshold = aiSelectedDataSources.GetConfidenceThreshold(targetWindow);
-                // bounds of the confidence interval:
-                //
-                var confidenceValues = aiSelectedDataSources.Select(x => x.Confidence).ToList();
-                var lowerBound = confidenceValues.Min();
-                var upperBound = confidenceValues.Max();
-
-                //
-                // Next, we search for a threshold so that we have between 2 and 3
-                // data sources. When not possible, we take all data sources.
-                //
-                var threshold = 0.0f;
-
-                // Check the case where the confidence values are too close:
-                if (upperBound - lowerBound >= 0.01)
-                {
-                    var previousThreshold = 0.0f;
-                    for (var i = 0; i < 10; i++)
-                    {
-                        threshold = lowerBound + (upperBound - lowerBound) * i / 10;
-                        var numMatches = aiSelectedDataSources.Count(x => x.Confidence >= threshold);
-                        if (numMatches <= 1)
-                        {
-                            threshold = previousThreshold;
-                            break;
-                        }
-
-                        if (numMatches is <= 3 and >= 2)
-                            break;
-
-                        previousThreshold = threshold;
-                    }
-                }
 
                 //
                 // Filter the data sources by the threshold:

app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs

@@ -0,0 +1,96 @@
+using System.Text;
+
+using AIStudio.Chat;
+
+namespace AIStudio.Tools.RAG;
+
+public static class IRetrievalContextExtensions
+{
+    private static readonly ILogger<IRetrievalContext> LOGGER = Program.SERVICE_PROVIDER.GetService<ILogger<IRetrievalContext>>()!;
+    
+    public static async Task<string> AsMarkdown(this IReadOnlyList<IRetrievalContext> retrievalContexts, StringBuilder? sb = null, CancellationToken token = default)
+    {
+        sb ??= new StringBuilder();
+        var index = 0;
+        
+        foreach(var retrievalContext in retrievalContexts)
+        {
+            index++;
+            await retrievalContext.AsMarkdown(sb, index, retrievalContexts.Count, token);
+        }
+        
+        return sb.ToString();
+    }
+    
+    public static async Task<string> AsMarkdown(this IRetrievalContext retrievalContext, StringBuilder? sb = null, int index = -1, int numTotalRetrievalContexts = -1, CancellationToken token = default)
+    {
+        sb ??= new StringBuilder();
+        switch (index)
+        {
+            case > 0 when numTotalRetrievalContexts is -1:
+                sb.AppendLine($"# Retrieval context {index}");
+                break;
+            
+            case > 0 when numTotalRetrievalContexts > 0:
+                sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
+                break;
+
+            default:
+                sb.AppendLine("# Retrieval context");
+                break;
+        }
+        
+        sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
+        sb.AppendLine($"Content category: {retrievalContext.Category}");
+        sb.AppendLine($"Content type: {retrievalContext.Type}");
+        sb.AppendLine($"Content path: {retrievalContext.Path}");
+            
+        if(retrievalContext.Links.Count > 0)
+        {
+            sb.AppendLine("Additional links:");
+            foreach(var link in retrievalContext.Links)
+                sb.AppendLine($"- {link}");
+        }
+            
+        switch(retrievalContext)
+        {
+            case RetrievalTextContext textContext:
+                sb.AppendLine();
+                sb.AppendLine("Matched text content:");
+                sb.AppendLine("````");
+                sb.AppendLine(textContext.MatchedText);
+                sb.AppendLine("````");
+                    
+                if(textContext.SurroundingContent.Count > 0)
+                {
+                    sb.AppendLine();
+                    sb.AppendLine("Surrounding text content:");
+                    foreach(var surrounding in textContext.SurroundingContent)
+                    {
+                        sb.AppendLine();
+                        sb.AppendLine("````");
+                        sb.AppendLine(surrounding);
+                        sb.AppendLine("````");
+                    }
+                }
+                    
+                    
+                break;
+                
+            case RetrievalImageContext imageContext:
+                sb.AppendLine();
+                sb.AppendLine("Matched image content as base64-encoded data:");
+                sb.AppendLine("````");
+                sb.AppendLine(await imageContext.AsBase64(token));
+                sb.AppendLine("````");
+                break;
+                
+            default:
+                LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
+                break;
+        }
+            
+        sb.AppendLine();
+        return sb.ToString();
+    }
+}

app/MindWork AI Studio/Tools/RAG/RetrievalContentCategoryExtensions.cs

@@ -9,7 +9,7 @@ public static class RetrievalContentCategoryExtensions
     /// </summary>
     /// <param name="contentType">The content type yielded by the ERI server.</param>
     /// <returns>The corresponding retrieval content category.</returns>
-    public static RetrievalContentCategory ToRetrievalContentCategory(ContentType contentType) => contentType switch
+    public static RetrievalContentCategory ToRetrievalContentCategory(this ContentType contentType) => contentType switch
     {
         ContentType.NONE => RetrievalContentCategory.NONE,
         ContentType.UNKNOWN => RetrievalContentCategory.UNKNOWN,

app/MindWork AI Studio/Tools/TargetWindow.cs

@@ -0,0 +1,38 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Represents a target window for the number of items to match a threshold.
+/// </summary>
+/// <param name="NumMinItems">The minimum number of items to match the threshold. Should be at least one and less than targetWindowMin.</param>
+/// <param name="TargetWindowMin">The minimum number of items in the target window. Should be at least 2 and more than numMinItems.</param>
+/// <param name="TargetWindowMax">The maximum number of items in the target window.</param>
+public readonly record struct TargetWindow(int NumMinItems, int TargetWindowMin, int TargetWindowMax, float MinThreshold)
+{
+    /// <summary>
+    /// Determines if the target window is valid.
+    /// </summary>
+    /// <returns>True when the target window is valid; otherwise, false.</returns>
+    public bool IsValid()
+    {
+        if(this.NumMinItems < 1)
+            return false;
+        
+        if(this.TargetWindowMin < this.NumMinItems)
+            return false;
+        
+        if(this.TargetWindowMax < this.TargetWindowMin)
+            return false;
+        
+        if(this.MinThreshold is < 0f or > 1f)
+            return false;
+        
+        return true;
+    }
+
+    /// <summary>
+    /// Determines if the number of items is inside the target window.
+    /// </summary>
+    /// <param name="numItems">The number of items to check.</param>
+    /// <returns>True when the number of items is inside the target window; otherwise, false.</returns>
+    public bool InsideWindow(int numItems) => numItems >= this.TargetWindowMin && numItems <= this.TargetWindowMax;
+}

app/MindWork AI Studio/Tools/TargetWindowStrategy.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Tools;
+
+public enum TargetWindowStrategy
+{
+    /// <summary>
+    /// Means no target window strategy, which will effectively return all items.
+    /// </summary>
+    NONE,
+    
+    /// <summary>
+    /// Searches for two up-to-three items but at least one.
+    /// </summary>
+    A_FEW_GOOD_ONES,
+    
+    /// <summary>
+    /// Searches for the top 10% items that are better than guessing, i.e., with confidence greater than 0.5f.
+    /// </summary>
+    TOP10_BETTER_THAN_GUESSING,
+}

app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md

@@ -3,6 +3,7 @@
 - Added an option to all data sources to select a local security policy. This preview feature is hidden behind the RAG feature flag.
 - Added an option to preselect data sources and options for new chats. This preview feature is hidden behind the RAG feature flag.
 - Added an agent to select the appropriate data sources for any prompt. This preview feature is hidden behind the RAG feature flag.
+- Added an agent to validate whether a retrieval context makes sense for the given prompt. This preview feature is hidden behind the RAG feature flag.
 - Added a generic RAG process to integrate possibly any data in your chats. Although the generic RAG process is now implemented, the retrieval part is working only with external data sources using the ERI interface. That means that you could integrate your company's data from the corporate network by now. The retrieval process for your local data is still under development and will take several weeks to be released. In order to use data through ERI, you (or your company) have to develop an ERI server. You might use the ERI server assistant within AI Studio to do so. This preview feature is hidden behind the RAG feature flag.
 - Improved confidence card for small spaces.
 - Fixed a bug in which 'APP_SETTINGS' appeared as a valid destination in the "send to" menu.