Merge

app/MindWork AI Studio/Tools/ERIClient/ERIClientBase.cs

@@ -9,9 +9,10 @@ public abstract class ERIClientBase(string baseAddress) : IDisposable
     {
         WriteIndented = true,
         AllowTrailingCommas = true,
-        PropertyNamingPolicy = null,
-        DictionaryKeyPolicy = JsonNamingPolicy.CamelCase,
         PropertyNameCaseInsensitive = true,
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        DictionaryKeyPolicy = JsonNamingPolicy.CamelCase,
+        UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow,
         Converters =
         {
             new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseUpper),

app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

@@ -66,22 +66,8 @@ public sealed class AugmentationOne : IAugmentationProcess
         // Let's convert all retrieval contexts to Markdown:
         await retrievalContexts.AsMarkdown(sb, token);
         
-        //
-        // Append the entire augmentation to the chat thread,
-        // just before the user prompt:
-        //
-        chatThread.Blocks.Insert(chatThread.Blocks.Count - 1, new()
-        {
-            Role = ChatRole.RAG,
-            Time = DateTimeOffset.UtcNow,
-            ContentType = ContentType.TEXT,
-            HideFromUser = true,
-            Content = new ContentText
-            {
-                Text = sb.ToString(),
-            }
-        });
-        
+        // Add the augmented data to the chat thread:
+        chatThread.AugmentedData = sb.ToString();
         return chatThread;
     }
 

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -1,6 +1,7 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
 using AIStudio.Tools.RAG.AugmentationProcesses;
 using AIStudio.Tools.RAG.DataSourceSelectionProcesses;
 using AIStudio.Tools.Services;
@@ -38,6 +39,30 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             // makes sense to proceed with the RAG process:
             var proceedWithRAG = true;
             
+            //
+            // We read the last block in the chat thread. We need to re-arrange
+            // the order of blocks later, after the augmentation process takes
+            // place:
+            //
+            if(chatThread.Blocks.Count == 0)
+            {
+                logger.LogError("The chat thread is empty. Skipping the RAG process.");
+                return chatThread;
+            }
+            
+            if (chatThread.Blocks.Last().Role != ChatRole.AI)
+            {
+                logger.LogError("The last block in the chat thread is not the AI block. There is something wrong with the chat thread. Skipping the RAG process.");
+                return chatThread;
+            }
+            
+            //
+            // At this point in time, the chat thread contains already the
+            // last block, which is the waiting AI block. We need to remove
+            // this block before we call some parts of the RAG process:
+            //
+            var chatThreadWithoutWaitingAIBlock = chatThread with { Blocks = chatThread.Blocks[..^1] };
+            
             //
             // When the user wants to bind data sources to the chat, we
             // have to check if the data sources are available for the
@@ -72,6 +97,56 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 logger.LogWarning("No data sources are selected. The RAG process is skipped.");
                 proceedWithRAG = false;
             }
+            else
+            {
+                var previousDataSecurity = chatThread.DataSecurity;
+                
+                //
+                // Update the data security of the chat thread. We consider the current data security
+                // of the chat thread and the data security of the selected data sources:
+                //
+                var dataSecurityRestrictedToSelfHosted = selectedDataSources.Any(x => x.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
+                chatThread.DataSecurity = dataSecurityRestrictedToSelfHosted switch
+                {
+                    //
+                    //
+                    // Case: the data sources which are selected have a security policy
+                    // of SELF_HOSTED (at least one data source).
+                    //
+                    // When the policy was already set to ALLOW_ANY, we restrict it
+                    // to SELF_HOSTED.
+                    //
+                    true => DataSourceSecurity.SELF_HOSTED,
+                    
+                    //
+                    // Case: the data sources which are selected have a security policy
+                    // of ALLOW_ANY (none of the data sources has a SELF_HOSTED policy).
+                    //
+                    // When the policy was already set to SELF_HOSTED, we must keep that.
+                    //
+                    false => chatThread.DataSecurity switch
+                    {
+                        //
+                        // When the policy was not specified yet, we set it to ALLOW_ANY.
+                        //
+                        DataSourceSecurity.NOT_SPECIFIED => DataSourceSecurity.ALLOW_ANY,
+                        DataSourceSecurity.ALLOW_ANY => DataSourceSecurity.ALLOW_ANY,
+                        
+                        //
+                        // When the policy was already set to SELF_HOSTED, we must keep that.
+                        // This is important since the thread might already contain data
+                        // from a data source with a SELF_HOSTED policy.
+                        //
+                        DataSourceSecurity.SELF_HOSTED => DataSourceSecurity.SELF_HOSTED,
+                        
+                        // Default case: we use the current data security of the chat thread.
+                        _ => chatThread.DataSecurity,
+                    }
+                };
+                
+                if (previousDataSecurity != chatThread.DataSecurity)
+                    logger.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'.");
+            }
             
             //
             // Trigger the retrieval part of the (R)AG process:
@@ -84,7 +159,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 //
                 var retrievalTasks = new List<Task<IReadOnlyList<IRetrievalContext>>>(selectedDataSources.Count);
                 foreach (var dataSource in selectedDataSources)
-                    retrievalTasks.Add(dataSource.RetrieveDataAsync(lastPrompt, chatThread, token));
+                    retrievalTasks.Add(dataSource.RetrieveDataAsync(lastPrompt, chatThreadWithoutWaitingAIBlock, token));
             
                 //
                 // Wait for all retrieval tasks to finish:

app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs

@@ -92,6 +92,14 @@ public sealed class DataSourceValidation
         
         return null;
     }
+
+    public string? ValidateRetrievalProcess(RetrievalInfo retrievalInfo)
+    {
+        if(retrievalInfo == default)
+            return "Please select one retrieval process.";
+        
+        return null;
+    }
     
     public string? ValidatingName(string dataSourceName)
     {