Showing sources coming from data providers (#559)

app/MindWork AI Studio/Tools/ISource.cs

@@ -0,0 +1,22 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Data model for a source used in the response.
+/// </summary>
+public interface ISource
+{
+    /// <summary>
+    /// The title of the source.
+    /// </summary>
+    public string Title { get; }
+    
+    /// <summary>
+    /// The URL of the source.
+    /// </summary>
+    public string URL { get; }
+    
+    /// <summary>
+    /// The origin of the source, whether it was provided by the AI or by the RAG process.
+    /// </summary>
+    public SourceOrigin Origin { get; }
+}

app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

@@ -10,6 +10,8 @@ namespace AIStudio.Tools.RAG.AugmentationProcesses;
 
 public sealed class AugmentationOne : IAugmentationProcess
 {
+    private static readonly ILogger<AugmentationOne> LOGGER = Program.LOGGER_FACTORY.CreateLogger<AugmentationOne>();
+    
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(AugmentationOne).Namespace, nameof(AugmentationOne));
     
     #region Implementation of IAugmentationProcess
@@ -24,14 +26,13 @@ public sealed class AugmentationOne : IAugmentationProcess
     public string Description => TB("This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread.");
     
     /// <inheritdoc />
-    public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default)
+    public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default)
     {
-        var logger = Program.SERVICE_PROVIDER.GetService<ILogger<AugmentationOne>>()!;
         var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
         
         if(retrievalContexts.Count == 0)
         {
-            logger.LogWarning("No retrieval contexts were issued. Skipping the augmentation process.");
+            LOGGER.LogWarning("No retrieval contexts were issued. Skipping the augmentation process.");
             return chatThread;
         }
 
@@ -45,7 +46,7 @@ public sealed class AugmentationOne : IAugmentationProcess
             validationAgent.SetLLMProvider(provider);
             
             // Let's validate all retrieval contexts:
-            var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastPrompt, chatThread, retrievalContexts, token);
+            var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token);
          
             //
             // Now, filter the retrieval contexts to the most relevant ones:
@@ -57,7 +58,7 @@ public sealed class AugmentationOne : IAugmentationProcess
             retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
         }
         
-        logger.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts.");
+        LOGGER.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts.");
         
         //
         // We build a huge prompt from all retrieval contexts:

app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs

@@ -9,6 +9,8 @@ namespace AIStudio.Tools.RAG.DataSourceSelectionProcesses;
 
 public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
 {
+    private static readonly ILogger<AgenticSrcSelWithDynHeur> LOGGER = Program.LOGGER_FACTORY.CreateLogger<AgenticSrcSelWithDynHeur>();
+    
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(AgenticSrcSelWithDynHeur).Namespace, nameof(AgenticSrcSelWithDynHeur));
     
     #region Implementation of IDataSourceSelectionProcess
@@ -23,15 +25,12 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
     public string Description => TB("Automatically selects the appropriate data sources based on the last prompt. Applies a heuristic reduction at the end to reduce the number of data sources.");
     
     /// <inheritdoc />
-    public async Task<DataSelectionResult> SelectDataSourcesAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, AllowedSelectedDataSources dataSources, CancellationToken token = default)
+    public async Task<DataSelectionResult> SelectDataSourcesAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, AllowedSelectedDataSources dataSources, CancellationToken token = default)
     {
         var proceedWithRAG = true;
         IReadOnlyList<IDataSource> selectedDataSources = [];
         IReadOnlyList<DataSourceAgentSelected> finalAISelection = [];
         
-        // Get the logger:
-        var logger = Program.SERVICE_PROVIDER.GetService<ILogger<AgenticSrcSelWithDynHeur>>()!;
-        
         // Get the settings manager:
         var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
         
@@ -41,12 +40,12 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
         try
         {
             // Let the AI agent do its work:
-            var aiSelectedDataSources = await selectionAgent.PerformSelectionAsync(provider, lastPrompt, chatThread, dataSources, token);
+            var aiSelectedDataSources = await selectionAgent.PerformSelectionAsync(provider, lastUserPrompt, chatThread, dataSources, token);
 
             // Check if the AI selected any data sources:
             if (aiSelectedDataSources.Count is 0)
             {
-                logger.LogWarning("The AI did not select any data sources. The RAG process is skipped.");
+                LOGGER.LogWarning("The AI did not select any data sources. The RAG process is skipped.");
                 proceedWithRAG = false;
                 
                 return new(proceedWithRAG, selectedDataSources);
@@ -54,7 +53,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
 
             // Log the selected data sources:
             var selectedDataSourceInfo = aiSelectedDataSources.Select(ds => $"[Id={ds.Id}, reason={ds.Reason}, confidence={ds.Confidence}]").Aggregate((a, b) => $"'{a}', '{b}'");
-            logger.LogInformation($"The AI selected the data sources automatically. {aiSelectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
+            LOGGER.LogInformation($"The AI selected the data sources automatically. {aiSelectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
 
             //
             // Check how many data sources were hallucinated by the AI:
@@ -69,7 +68,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
 
             var numHallucinatedSources = totalAISelectedDataSources - aiSelectedDataSources.Count;
             if (numHallucinatedSources > 0)
-                logger.LogWarning($"The AI hallucinated {numHallucinatedSources} data source(s). We ignore them.");
+                LOGGER.LogWarning($"The AI hallucinated {numHallucinatedSources} data source(s). We ignore them.");
 
             if (aiSelectedDataSources.Count > 3)
             {
@@ -85,7 +84,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
                     if (aiSelectedDataSources.Any(x => x.Id == dataSource.DataSource.Id))
                         dataSource.Selected = true;
 
-                logger.LogInformation($"The AI selected {aiSelectedDataSources.Count} data source(s) with a confidence of at least {threshold}.");
+                LOGGER.LogInformation($"The AI selected {aiSelectedDataSources.Count} data source(s) with a confidence of at least {threshold}.");
 
                 // Transform the final data sources to the actual data sources:
                 selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;

app/MindWork AI Studio/Tools/RAG/IAugmentationProcess.cs

@@ -24,10 +24,10 @@ public interface IAugmentationProcess
     /// Starts the augmentation process.
     /// </summary>
     /// <param name="provider">The LLM provider. Gets used, e.g., for automatic retrieval context validation.</param>
-    /// <param name="lastPrompt">The last prompt that was issued by the user.</param>
+    /// <param name="lastUserPrompt">The last user prompt that was issued by the user.</param>
     /// <param name="chatThread">The chat thread.</param>
     /// <param name="retrievalContexts">The retrieval contexts that were issued by the retrieval process.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>The altered chat thread.</returns>
-    public Task<ChatThread> ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default);
+    public Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default);
 }

app/MindWork AI Studio/Tools/RAG/IDataSourceSelectionProcess.cs

@@ -24,10 +24,10 @@ public interface IDataSourceSelectionProcess
     /// Starts the data source selection process.
     /// </summary>
     /// <param name="provider">The LLM provider. Used as default for data selection agents.</param>
-    /// <param name="lastPrompt">The last prompt that was issued by the user.</param>
+    /// <param name="lastUserPrompt">The last prompt that was issued by the user.</param>
     /// <param name="chatThread">The chat thread.</param>
     /// <param name="dataSources">The allowed data sources yielded by the data source service.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns></returns>
-    public Task<DataSelectionResult> SelectDataSourcesAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, AllowedSelectedDataSources dataSources, CancellationToken token = default);
+    public Task<DataSelectionResult> SelectDataSourcesAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, AllowedSelectedDataSources dataSources, CancellationToken token = default);
 }

app/MindWork AI Studio/Tools/RAG/IRagProcess.cs

@@ -24,9 +24,9 @@ public interface IRagProcess
     /// Starts the RAG process.
     /// </summary>
     /// <param name="provider">The LLM provider. Used to check whether the data sources are allowed to be used by this LLM.</param>
-    /// <param name="lastPrompt">The last prompt that was issued by the user.</param>
+    /// <param name="lastUserPrompt">The last user prompt that was issued by the user.</param>
     /// <param name="chatThread">The chat thread.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>The altered chat thread.</returns>
-    public Task<ChatThread> ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, CancellationToken token = default);
+    public Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, CancellationToken token = default);
 }

app/MindWork AI Studio/Tools/RAG/IRetrievalContext.cs

@@ -9,9 +9,9 @@ public interface IRetrievalContext
     /// The name of the data source.
     /// </summary>
     /// <remarks>
-    /// Depending on the configuration, the AI is selecting the appropriate data source.
-    /// In order to inform the user about where the information is coming from, the data
-    /// source name is necessary.
+    /// This is not the name the user chooses but the name of the source where
+    /// the match was found. This could be a document or database name, a website
+    /// or a directory on a remote server, etc.
     /// </remarks>
     public string DataSourceName { get; init; }
 

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -11,6 +11,8 @@ namespace AIStudio.Tools.RAG.RAGProcesses;
 
 public sealed class AISrcSelWithRetCtxVal : IRagProcess
 {
+    private static readonly ILogger<AISrcSelWithRetCtxVal> LOGGER = Program.LOGGER_FACTORY.CreateLogger<AISrcSelWithRetCtxVal>();
+    
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(AISrcSelWithRetCtxVal).Namespace, nameof(AISrcSelWithRetCtxVal));
     
     #region Implementation of IRagProcess
@@ -25,9 +27,8 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
     public string Description => TB("This RAG process filters data sources, automatically selects appropriate sources, optionally allows manual source selection, retrieves data, and automatically validates the retrieval context.");
 
     /// <inheritdoc />
-    public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, CancellationToken token = default)
+    public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, CancellationToken token = default)
     {
-        var logger = Program.SERVICE_PROVIDER.GetService<ILogger<AISrcSelWithRetCtxVal>>()!;
         var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
         var dataSourceService = Program.SERVICE_PROVIDER.GetService<DataSourceService>()!;
         
@@ -36,7 +37,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
         //
         if (chatThread.DataSourceOptions.IsEnabled())
         {
-            logger.LogInformation("Data sources are enabled for this chat.");
+            LOGGER.LogInformation("Data sources are enabled for this chat.");
             
             // Across the different code-branches, we keep track of whether it
             // makes sense to proceed with the RAG process:
@@ -49,13 +50,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             //
             if(chatThread.Blocks.Count == 0)
             {
-                logger.LogError("The chat thread is empty. Skipping the RAG process.");
+                LOGGER.LogError("The chat thread is empty. Skipping the RAG process.");
                 return chatThread;
             }
             
             if (chatThread.Blocks.Last().Role != ChatRole.AI)
             {
-                logger.LogError("The last block in the chat thread is not the AI block. There is something wrong with the chat thread. Skipping the RAG process.");
+                LOGGER.LogError("The last block in the chat thread is not the AI block. There is something wrong with the chat thread. Skipping the RAG process.");
                 return chatThread;
             }
             
@@ -82,7 +83,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             if (chatThread.DataSourceOptions.AutomaticDataSourceSelection)
             {
                 var dataSourceSelectionProcess = new AgenticSrcSelWithDynHeur();
-                var result = await dataSourceSelectionProcess.SelectDataSourcesAsync(provider, lastPrompt, chatThread, dataSources, token);
+                var result = await dataSourceSelectionProcess.SelectDataSourcesAsync(provider, lastUserPrompt, chatThread, dataSources, token);
                 proceedWithRAG = result.ProceedWithRAG;
                 selectedDataSources = result.SelectedDataSources;
             }
@@ -92,12 +93,12 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 // No, the user made the choice manually:
                 //
                 var selectedDataSourceInfo = selectedDataSources.Select(ds => ds.Name).Aggregate((a, b) => $"'{a}', '{b}'");
-                logger.LogInformation($"The user selected the data sources manually. {selectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
+                LOGGER.LogInformation($"The user selected the data sources manually. {selectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
             }
 
             if(selectedDataSources.Count == 0)
             {
-                logger.LogWarning("No data sources are selected. The RAG process is skipped.");
+                LOGGER.LogWarning("No data sources are selected. The RAG process is skipped.");
                 proceedWithRAG = false;
             }
             else
@@ -148,7 +149,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 };
                 
                 if (previousDataSecurity != chatThread.DataSecurity)
-                    logger.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'.");
+                    LOGGER.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'.");
             }
             
             //
@@ -162,7 +163,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 //
                 var retrievalTasks = new List<Task<IReadOnlyList<IRetrievalContext>>>(selectedDataSources.Count);
                 foreach (var dataSource in selectedDataSources)
-                    retrievalTasks.Add(dataSource.RetrieveDataAsync(lastPrompt, chatThreadWithoutWaitingAIBlock, token));
+                    retrievalTasks.Add(dataSource.RetrieveDataAsync(lastUserPrompt, chatThreadWithoutWaitingAIBlock, token));
             
                 //
                 // Wait for all retrieval tasks to finish:
@@ -175,7 +176,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                     }
                     catch (Exception e)
                     {
-                        logger.LogError(e, "An error occurred during the retrieval process.");
+                        LOGGER.LogError(e, "An error occurred during the retrieval process.");
                     }
                 }
             }
@@ -186,8 +187,38 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             if (proceedWithRAG)
             {
                 var augmentationProcess = new AugmentationOne();
-                chatThread = await augmentationProcess.ProcessAsync(provider, lastPrompt, chatThread, dataContexts, token);
+                chatThread = await augmentationProcess.ProcessAsync(provider, lastUserPrompt, chatThread, dataContexts, token);
             }
+            
+            //
+            // Add sources from the selected data
+            //
+            
+            // We know that the last block is the AI answer block (cf. check above):
+            var aiAnswerBlock = chatThread.Blocks.Last();
+            var aiAnswerSources = aiAnswerBlock.Content?.Sources;
+            
+            // It should never happen that the AI answer block does not contain a content part.
+            // Just in case, we check this:
+            if(aiAnswerSources is null)
+                return chatThread;
+            
+            var ragSources = new List<ISource>();
+            foreach (var retrievalContext in dataContexts)
+            {
+                var title = retrievalContext.DataSourceName;
+                if(string.IsNullOrWhiteSpace(title))
+                    continue;
+                
+                var link = retrievalContext.Path;
+                if(!link.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+                    continue;
+
+                ragSources.Add(new Source(title, link, SourceOrigin.RAG));
+            }
+
+            // Merge the sources, avoiding duplicates:
+            aiAnswerSources.MergeSources(ragSources);
         }
         
         return chatThread;

app/MindWork AI Studio/Tools/Services/UpdateService.cs

@@ -115,7 +115,14 @@ public sealed class UpdateService : BackgroundService, IMessageBusReceiver
         var response = await this.rust.CheckForUpdate();
         if (response.UpdateIsAvailable)
         {
-            if (this.settingsManager.ConfigurationData.App.UpdateInstallation is UpdateInstallation.AUTOMATIC)
+            // ReSharper disable RedundantAssignment
+            var isDevEnvironment = false;
+            #if DEBUG
+            isDevEnvironment = true;
+            #endif
+            // ReSharper restore RedundantAssignment
+            
+            if (!isDevEnvironment && this.settingsManager.ConfigurationData.App.UpdateInstallation is UpdateInstallation.AUTOMATIC)
             {
                 try
                 {

app/MindWork AI Studio/Tools/Source.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Data model for a source used in the response.
+/// </summary>
+/// <param name="Title">The title of the source.</param>
+/// <param name="URL">The URL of the source.</param>
+public record Source(string Title, string URL, SourceOrigin Origin) : ISource;

app/MindWork AI Studio/Tools/SourceExtensions.cs

@@ -0,0 +1,79 @@
+using System.Text;
+
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools;
+
+public static class SourceExtensions
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SourceExtensions).Namespace, nameof(SourceExtensions));
+    
+    /// <summary>
+    /// Converts a list of sources to a markdown-formatted string.
+    /// </summary>
+    /// <param name="sources">The list of sources to convert.</param>
+    /// <returns>A markdown-formatted string representing the sources.</returns>
+    public static string ToMarkdown(this IList<Source> sources)
+    {
+        var sb = new StringBuilder();
+        var ragSources = new List<ISource>();
+        var sourceNum = 0;
+        var addedLLMHeaders = false;
+        foreach (var source in sources)
+        {
+            switch (source.Origin)
+            {
+                case SourceOrigin.RAG: 
+                    ragSources.Add(source);
+                    break;
+                
+                case SourceOrigin.LLM:
+                    if (!addedLLMHeaders)
+                    {
+                        sb.Append("## ");
+                        sb.AppendLine(TB("Sources provided by the AI"));
+                        addedLLMHeaders = true;
+                    }
+                    
+                    sb.Append($"- [{++sourceNum}] ");
+                    sb.Append('[');
+                    sb.Append(source.Title);
+                    sb.Append("](");
+                    sb.Append(source.URL);
+                    sb.AppendLine(")");
+                    break;
+            }
+        }
+        
+        if(ragSources.Count == 0)
+            return sb.ToString();
+        
+        sb.AppendLine();
+        sb.Append("## ");
+        sb.AppendLine(TB("Sources provided by the data providers"));
+        
+        foreach (var source in ragSources)
+        {
+            sb.Append($"- [{++sourceNum}] ");
+            sb.Append('[');
+            sb.Append(source.Title);
+            sb.Append("](");
+            sb.Append(source.URL);
+            sb.AppendLine(")");
+        }
+        
+        return sb.ToString();
+    }
+    
+    /// <summary>
+    /// Merges a list of added sources into an existing list of sources, avoiding duplicates based on URL and Title.
+    /// </summary>
+    /// <param name="sources">The existing list of sources to merge into.</param>
+    /// <param name="addedSources">The list of sources to add.</param>
+    public static void MergeSources(this IList<Source> sources, IList<ISource> addedSources)
+    {
+        foreach (var addedSource in addedSources)
+            if (sources.All(s => s.URL != addedSource.URL && s.Title != addedSource.Title))
+                sources.Add((Source)addedSource);
+    }
+}

app/MindWork AI Studio/Tools/SourceOrigin.cs

@@ -0,0 +1,17 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Represents the origin of a source, whether it was provided by the LLM or by the RAG process.
+/// </summary>
+public enum SourceOrigin
+{
+    /// <summary>
+    /// The LLM provided the source.
+    /// </summary>
+    LLM,
+    
+    /// <summary>
+    /// The source was provided by the RAG process.
+    /// </summary>
+    RAG,
+}