Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

@@ -43,22 +43,40 @@ public sealed class AugmentationOne : IAugmentationProcess
         {
             // Let's get the validation agent & set up its provider:
             var validationAgent = Program.SERVICE_PROVIDER.GetService<AgentRetrievalContextValidation>()!;
-            validationAgent.SetLLMProvider(provider);
-            
-            // Let's validate all retrieval contexts:
-            var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token);
-         
-            //
-            // Now, filter the retrieval contexts to the most relevant ones:
-            //
-            var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING);
-            var threshold = validationResults.GetConfidenceThreshold(targetWindow);
-            
-            // Filter the retrieval contexts:
-            retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
+            if (validationAgent.SetLLMProvider(provider, chatThread.DataSecurity, chatThread.RequiredProviderConfidence))
+            {
+                try
+                {
+                    // Let's validate all retrieval contexts:
+                    var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token);
+                    if (validationResults.Count == 0)
+                        LOGGER.LogWarning("Retrieval context validation returned no results. Continuing augmentation with all retrieved contexts.");
+                    else
+                    {
+                        //
+                        // Now, filter the retrieval contexts to the most relevant ones:
+                        //
+                        var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING);
+                        var threshold = validationResults.GetConfidenceThreshold(targetWindow);
+
+                        // Filter the retrieval contexts:
+                        retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
+                    }
+                }
+                catch (OperationCanceledException) when (token.IsCancellationRequested)
+                {
+                    throw;
+                }
+                catch (Exception exception)
+                {
+                    LOGGER.LogError(exception, "Retrieval context validation failed. Continuing augmentation with all retrieved contexts.");
+                }
+            }
+            else
+                LOGGER.LogWarning("Skipping retrieval context validation because no sufficiently trusted validation agent provider is available. Continuing augmentation with all retrieved contexts.");
         }
         
-        LOGGER.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts.");
+        LOGGER.LogInformation($"Starting the augmentation process over {retrievalContexts.Count:###,###,###,###} of {numTotalRetrievalContexts:###,###,###,###} retrieved contexts.");
         
         //
         // We build a huge prompt from all retrieval contexts:

app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs

@@ -31,11 +31,9 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
         IReadOnlyList<IDataSource> selectedDataSources = [];
         IReadOnlyList<DataSourceAgentSelected> finalAISelection = [];
         
-        // Get the settings manager:
-        var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
-        
         // Get the agent for the data source selection:
         var selectionAgent = Program.SERVICE_PROVIDER.GetService<AgentDataSourceSelection>()!;
+        var allowedDataSources = dataSources.AllowedDataSources.ToDictionary(ds => ds.Id, StringComparer.Ordinal);
 
         try
         {
@@ -61,14 +59,14 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
             var totalAISelectedDataSources = aiSelectedDataSources.Count;
 
             // Filter out the data sources that are not available:
-            aiSelectedDataSources = aiSelectedDataSources.Where(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id) is not null).ToList();
+            aiSelectedDataSources = aiSelectedDataSources.Where(x => allowedDataSources.ContainsKey(x.Id)).ToList();
 
             // Store the real AI-selected data sources:
-            finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = settings.ConfigurationData.DataSources.First(ds => ds.Id == x.Id), AIDecision = x, Selected = false }).ToList();
+            finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = allowedDataSources[x.Id], AIDecision = x, Selected = false }).ToList();
 
             var numHallucinatedSources = totalAISelectedDataSources - aiSelectedDataSources.Count;
             if (numHallucinatedSources > 0)
-                LOGGER.LogWarning($"The AI hallucinated {numHallucinatedSources} data source(s). We ignore them.");
+                LOGGER.LogWarning($"The AI selected {numHallucinatedSources} unavailable data source(s). We ignore them.");
 
             if (aiSelectedDataSources.Count > 3)
             {
@@ -87,7 +85,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
                 LOGGER.LogInformation($"The AI selected {aiSelectedDataSources.Count} data source(s) with a confidence of at least {threshold}.");
 
                 // Transform the final data sources to the actual data sources:
-                selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;
+                selectedDataSources = aiSelectedDataSources.Select(x => allowedDataSources[x.Id]).ToList();
                 return new(proceedWithRAG, selectedDataSources);
             }
 
@@ -96,7 +94,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
             //
 
             // Transform the selected data sources to the actual data sources:
-            selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;
+            selectedDataSources = aiSelectedDataSources.Select(x => allowedDataSources[x.Id]).ToList();
 
             // Mark the data sources as selected:
             foreach (var dataSource in finalAISelection)

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -35,7 +35,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
         //
         // 1. Check if the user wants to bind any data sources to the chat:
         //
-        if (chatThread.DataSourceOptions.IsEnabled())
+        //
+        // Data sources are a preview feature. The check belongs here rather than in the options
+        // themselves: a chat keeps its data source options while the feature is switched off, and
+        // organizations may preselect data sources through a configuration plugin. Without this,
+        // such a chat would still run the entire RAG process with the feature disabled.
+        //
+        if (PreviewFeatures.PRE_RAG_2024.IsEnabled(settings) && chatThread.DataSourceOptions.IsEnabled())
         {
             LOGGER.LogInformation("Data sources are enabled for this chat.");
             
@@ -74,7 +80,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             // data sources changed its security requirements.
             //
             List<IDataSource> preselectedDataSources = chatThread.DataSourceOptions.PreselectedDataSourceIds.Select(id => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == id)).Where(ds => ds is not null).ToList()!;
-            var dataSources = await dataSourceService.GetDataSources(provider, preselectedDataSources);
+            var dataSources = await dataSourceService.GetDataSources(provider, chatThread.DataSourceOptions, preselectedDataSources);
             var selectedDataSources = dataSources.SelectedDataSources;
             
             //
@@ -92,7 +98,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 //
                 // No, the user made the choice manually:
                 //
-                var selectedDataSourceInfo = selectedDataSources.Select(ds => ds.Name).Aggregate((a, b) => $"'{a}', '{b}'");
+                var selectedDataSourceInfo = string.Join(", ", selectedDataSources.Select(ds => $"'{ds.Name}'"));
                 LOGGER.LogInformation($"The user selected the data sources manually. {selectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
             }
 
@@ -104,12 +110,15 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             else
             {
                 var previousDataSecurity = chatThread.DataSecurity;
+                var previousRequiredProviderConfidence = chatThread.RequiredProviderConfidence;
                 
                 //
                 // Update the data security of the chat thread. We consider the current data security
                 // of the chat thread and the data security of the selected data sources:
                 //
-                var dataSecurityRestrictedToSelfHosted = selectedDataSources.Any(x => x.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
+                var dataSecurityRestrictedToSelfHosted = selectedDataSources
+                    .OfType<IExternalDataSource>()
+                    .Any(dataSource => dataSource.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
                 chatThread.DataSecurity = dataSecurityRestrictedToSelfHosted switch
                 {
                     //
@@ -150,6 +159,12 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 
                 if (previousDataSecurity != chatThread.DataSecurity)
                     LOGGER.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'.");
+
+                foreach (var dataSource in selectedDataSources.OfType<IInternalDataSource>())
+                    chatThread.RequireProviderConfidence(dataSource.ConfidenceLevel);
+
+                if (previousRequiredProviderConfidence != chatThread.RequiredProviderConfidence)
+                    LOGGER.LogInformation($"The required provider confidence of the chat thread was updated from '{previousRequiredProviderConfidence.GetName()}' to '{chatThread.RequiredProviderConfidence.GetName()}'.");
             }
             
             //
@@ -205,17 +220,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             
             var ragSources = new List<ISource>();
             foreach (var retrievalContext in dataContexts)
-            {
-                var title = retrievalContext.DataSourceName;
-                if(string.IsNullOrWhiteSpace(title))
-                    continue;
-                
-                var link = retrievalContext.Path;
-                if(!link.StartsWith("http", StringComparison.OrdinalIgnoreCase))
-                    continue;
-
-                ragSources.Add(new Source(title, link, SourceOrigin.RAG));
-            }
+                ragSources.AddRange(CreateSources(retrievalContext));
 
             // Merge the sources, avoiding duplicates:
             aiAnswerSources.MergeSources(ragSources);
@@ -225,4 +230,63 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
     }
 
     #endregion
-}
+
+    private static IReadOnlyList<ISource> CreateSources(IRetrievalContext retrievalContext)
+    {
+        var sources = new List<ISource>();
+        AddSource(sources, GetReferenceTitle(retrievalContext), GetReferenceLink(retrievalContext));
+        foreach (var link in retrievalContext.Links)
+            AddSource(sources, retrievalContext.DataSourceName, link);
+
+        return sources;
+    }
+
+    private static void AddSource(ICollection<ISource> sources, string title, string link)
+    {
+        if (string.IsNullOrWhiteSpace(title) || !TryNormalizeSourceLink(link, out var normalizedLink))
+            return;
+
+        sources.Add(new Source(title, normalizedLink, SourceOrigin.RAG));
+    }
+
+    private static string GetReferenceTitle(IRetrievalContext retrievalContext) =>
+        retrievalContext is RetrievalTextContext { ReferenceTitle: { Length: > 0 } referenceTitle }
+            ? referenceTitle
+            : retrievalContext.DataSourceName;
+
+    private static string GetReferenceLink(IRetrievalContext retrievalContext) =>
+        retrievalContext is RetrievalTextContext { ReferenceLink: { Length: > 0 } referenceLink }
+            ? referenceLink
+            : retrievalContext.Path;
+
+    private static bool TryNormalizeSourceLink(string link, out string normalizedLink)
+    {
+        normalizedLink = string.Empty;
+        if (string.IsNullOrWhiteSpace(link))
+            return false;
+
+        if (Uri.TryCreate(link, UriKind.Absolute, out var absoluteUri) && IsSupportedSourceUri(absoluteUri))
+        {
+            normalizedLink = absoluteUri.AbsoluteUri;
+            return true;
+        }
+
+        try
+        {
+            if (!Path.IsPathRooted(link))
+                return false;
+
+            normalizedLink = new Uri(Path.GetFullPath(link)).AbsoluteUri;
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    private static bool IsSupportedSourceUri(Uri uri) =>
+        string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
+        || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)
+        || string.Equals(uri.Scheme, Uri.UriSchemeFile, StringComparison.OrdinalIgnoreCase);
+}

app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs

@@ -40,4 +40,14 @@ public sealed class RetrievalTextContext : IRetrievalContext
     /// For example, one sentence or paragraph before and after the matched text.
     /// </remarks>
     public IReadOnlyList<string> SurroundingContent { get; set; } = [];
+
+    /// <summary>
+    /// Optional title used when this context is displayed as a source reference.
+    /// </summary>
+    public string ReferenceTitle { get; init; } = string.Empty;
+
+    /// <summary>
+    /// Optional link used when this context is displayed as a source reference.
+    /// </summary>
+    public string ReferenceLink { get; init; } = string.Empty;
 }