Added an agent to select the appropriate data sources for any prompt (#284)

app/MindWork AI Studio/Chat/ChatThread.cs

@@ -1,3 +1,4 @@
+using AIStudio.Components;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 
@@ -33,6 +34,11 @@ public sealed record ChatThread
     /// </summary>
     public DataSourceOptions DataSourceOptions { get; set; } = new();
 
+    /// <summary>
+    /// The AI-selected data sources for this chat thread.
+    /// </summary>
+    public IReadOnlyList<DataSourceAgentSelected> AISelectedDataSources { get; set; } = [];
+
     /// <summary>
     /// The name of the chat thread. Usually generated by an AI model or manually edited by the user.
     /// </summary>

app/MindWork AI Studio/Chat/ContentImage.cs

@@ -49,4 +49,62 @@ public sealed class ContentImage : IContent
     /// The image source.
     /// </summary>
     public required string Source { get; set; }
+
+    /// <summary>
+    /// Read the image content as a base64 string.
+    /// </summary>
+    /// <remarks>
+    /// The images are directly converted to base64 strings. The maximum
+    /// size of the image is around 10 MB. If the image is larger, the method
+    /// returns an empty string.
+    ///
+    /// As of now, this method does no sort of image processing. LLMs usually
+    /// do not work with arbitrary image sizes. In the future, we might have
+    /// to resize the images before sending them to the model.
+    /// </remarks>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The image content as a base64 string; might be empty.</returns>
+    public async Task<string> AsBase64(CancellationToken token = default)
+    {
+        switch (this.SourceType)
+        {
+            case ContentImageSource.BASE64:
+                return this.Source;
+            
+            case ContentImageSource.URL:
+            {
+                using var httpClient = new HttpClient();
+                using var response = await httpClient.GetAsync(this.Source, HttpCompletionOption.ResponseHeadersRead, token);
+                if(response.IsSuccessStatusCode)
+                {
+                    // Read the length of the content:
+                    var lengthBytes = response.Content.Headers.ContentLength;
+                    if(lengthBytes > 10_000_000)
+                        return string.Empty;
+                    
+                    var bytes = await response.Content.ReadAsByteArrayAsync(token);
+                    return Convert.ToBase64String(bytes);
+                }
+
+                return string.Empty;
+            }
+
+            case ContentImageSource.LOCAL_PATH:
+                if(File.Exists(this.Source))
+                {
+                    // Read the content length:
+                    var length = new FileInfo(this.Source).Length;
+                    if(length > 10_000_000)
+                        return string.Empty;
+                    
+                    var bytes = await File.ReadAllBytesAsync(this.Source, token);
+                    return Convert.ToBase64String(bytes);
+                }
+
+                return string.Empty;
+            
+            default:
+                return string.Empty;
+        }
+    }
 }

app/MindWork AI Studio/Chat/ContentText.cs

@@ -1,5 +1,7 @@
 using System.Text.Json.Serialization;
 
+using AIStudio.Agents;
+using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Tools.Services;
@@ -41,11 +43,19 @@ public sealed class ContentText : IContent
         if(chatThread is null)
             return;
 
+        var logger = Program.SERVICE_PROVIDER.GetService<ILogger<ContentText>>()!;
+        
         //
-        // Check if the user wants to bind any data sources to the chat:
+        // 1. Check if the user wants to bind any data sources to the chat:
         //
-        if (chatThread.DataSourceOptions.IsEnabled())
+        if (chatThread.DataSourceOptions.IsEnabled() && lastPrompt is not null)
         {
+            logger.LogInformation("Data sources are enabled for this chat.");
+            
+            // Across the different code-branches, we keep track of whether it
+            // makes sense to proceed with the RAG process:
+            var proceedWithRAG = true;
+            
             //
             // When the user wants to bind data sources to the chat, we
             // have to check if the data sources are available for the
@@ -61,18 +71,146 @@ public sealed class ContentText : IContent
             //
             if (chatThread.DataSourceOptions.AutomaticDataSourceSelection)
             {
-                // TODO: Start agent based on allowed data sources.
+                // Get the agent for the data source selection:
+                var selectionAgent = Program.SERVICE_PROVIDER.GetService<AgentDataSourceSelection>()!;
+                
+                // Let the AI agent do its work:
+                IReadOnlyList<DataSourceAgentSelected> finalAISelection = [];
+                var aiSelectedDataSources = await selectionAgent.PerformSelectionAsync(provider, lastPrompt, chatThread, dataSources, token);
+
+                // Check if the AI selected any data sources:
+                if(aiSelectedDataSources.Count is 0)
+                {
+                    logger.LogWarning("The AI did not select any data sources. The RAG process is skipped.");
+                    proceedWithRAG = false;
+                    
+                    // Send the selected data sources to the data source selection component.
+                    // Then, the user can see which data sources were selected by the AI.
+                    await MessageBus.INSTANCE.SendMessage(null, Event.RAG_AUTO_DATA_SOURCES_SELECTED, finalAISelection);
+                    chatThread.AISelectedDataSources = finalAISelection;
+                }
+                else
+                {
+                    // Log the selected data sources:
+                    var selectedDataSourceInfo = aiSelectedDataSources.Select(ds => $"[Id={ds.Id}, reason={ds.Reason}, confidence={ds.Confidence}]").Aggregate((a, b) => $"'{a}', '{b}'");
+                    logger.LogInformation($"The AI selected the data sources automatically. {aiSelectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
+
+                    //
+                    // Check how many data sources were hallucinated by the AI:
+                    //
+                    var totalAISelectedDataSources = aiSelectedDataSources.Count;
+                    
+                    // Filter out the data sources that are not available:
+                    aiSelectedDataSources = aiSelectedDataSources.Where(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id) is not null).ToList();
+                    
+                    // Store the real AI-selected data sources:
+                    finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = settings.ConfigurationData.DataSources.First(ds => ds.Id == x.Id), AIDecision = x, Selected = false }).ToList();
+                    
+                    var numHallucinatedSources = totalAISelectedDataSources - aiSelectedDataSources.Count;
+                    if(numHallucinatedSources > 0)
+                        logger.LogWarning($"The AI hallucinated {numHallucinatedSources} data source(s). We ignore them.");
+                    
+                    if (aiSelectedDataSources.Count > 3)
+                    {
+                        //
+                        // We have more than 3 data sources. Let's filter by confidence.
+                        // In order to do that, we must identify the lower and upper
+                        // bounds of the confidence interval:
+                        //
+                        var confidenceValues = aiSelectedDataSources.Select(x => x.Confidence).ToList();
+                        var lowerBound = confidenceValues.Min();
+                        var upperBound = confidenceValues.Max();
+                        
+                        //
+                        // Next, we search for a threshold so that we have between 2 and 3
+                        // data sources. When not possible, we take all data sources.
+                        //
+                        var threshold = 0.0f;
+                        
+                        // Check the case where the confidence values are too close:
+                        if (upperBound - lowerBound >= 0.01)
+                        {
+                            var previousThreshold = 0.0f;
+                            for (var i = 0; i < 10; i++)
+                            {
+                                threshold = lowerBound + (upperBound - lowerBound) * i / 10;
+                                var numMatches = aiSelectedDataSources.Count(x => x.Confidence >= threshold);
+                                if (numMatches <= 1)
+                                {
+                                    threshold = previousThreshold;
+                                    break;
+                                }
+		
+                                if (numMatches is <= 3 and >= 2)
+                                    break;
+                                
+                                previousThreshold = threshold;
+                            }
+                        }
+                        
+                        //
+                        // Filter the data sources by the threshold:
+                        //
+                        aiSelectedDataSources = aiSelectedDataSources.Where(x => x.Confidence >= threshold).ToList();
+                        foreach (var dataSource in finalAISelection)
+                            if(aiSelectedDataSources.Any(x => x.Id == dataSource.DataSource.Id))
+                                dataSource.Selected = true;
+                        
+                        logger.LogInformation($"The AI selected {aiSelectedDataSources.Count} data source(s) with a confidence of at least {threshold}.");
+                        
+                        // Transform the final data sources to the actual data sources:
+                        selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;
+                    }
+                    
+                    // We have max. 3 data sources. We take all of them:
+                    else
+                    {
+                        // Transform the selected data sources to the actual data sources:
+                        selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;
+                        
+                        // Mark the data sources as selected:
+                        foreach (var dataSource in finalAISelection)
+                            dataSource.Selected = true;
+                    }
+                    
+                    // Send the selected data sources to the data source selection component.
+                    // Then, the user can see which data sources were selected by the AI.
+                    await MessageBus.INSTANCE.SendMessage(null, Event.RAG_AUTO_DATA_SOURCES_SELECTED, finalAISelection);
+                    chatThread.AISelectedDataSources = finalAISelection;
+                }
+            }
+            else
+            {
+                //
+                // No, the user made the choice manually:
+                //
+                var selectedDataSourceInfo = selectedDataSources.Select(ds => ds.Name).Aggregate((a, b) => $"'{a}', '{b}'");
+                logger.LogInformation($"The user selected the data sources manually. {selectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
             }
 
+            if(selectedDataSources.Count == 0)
+            {
+                logger.LogWarning("No data sources are selected. The RAG process is skipped.");
+                proceedWithRAG = false;
+            }
+            
             //
             // Trigger the retrieval part of the (R)AG process:
             //
+            if (proceedWithRAG)
+            {
+                
+            }
 
             //
             // Perform the augmentation of the R(A)G process:
             //
+            if (proceedWithRAG)
+            {
+                
+            }
         }
-
+        
         // Store the last time we got a response. We use this later
         // to determine whether we should notify the UI about the
         // new content or not. Depends on the energy saving mode