Improved the augmentation & generation of RAG (#314)

app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

@@ -66,22 +66,8 @@ public sealed class AugmentationOne : IAugmentationProcess
         // Let's convert all retrieval contexts to Markdown:
         await retrievalContexts.AsMarkdown(sb, token);
         
-        //
-        // Append the entire augmentation to the chat thread,
-        // just before the user prompt:
-        //
-        chatThread.Blocks.Insert(chatThread.Blocks.Count - 1, new()
-        {
-            Role = ChatRole.RAG,
-            Time = DateTimeOffset.UtcNow,
-            ContentType = ContentType.TEXT,
-            HideFromUser = true,
-            Content = new ContentText
-            {
-                Text = sb.ToString(),
-            }
-        });
-        
+        // Add the augmented data to the chat thread:
+        chatThread.AugmentedData = sb.ToString();
         return chatThread;
     }
 

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -38,6 +38,30 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             // makes sense to proceed with the RAG process:
             var proceedWithRAG = true;
             
+            //
+            // We read the last block in the chat thread. We need to re-arrange
+            // the order of blocks later, after the augmentation process takes
+            // place:
+            //
+            if(chatThread.Blocks.Count == 0)
+            {
+                logger.LogError("The chat thread is empty. Skipping the RAG process.");
+                return chatThread;
+            }
+            
+            if (chatThread.Blocks.Last().Role != ChatRole.AI)
+            {
+                logger.LogError("The last block in the chat thread is not the AI block. There is something wrong with the chat thread. Skipping the RAG process.");
+                return chatThread;
+            }
+            
+            //
+            // At this point in time, the chat thread contains already the
+            // last block, which is the waiting AI block. We need to remove
+            // this block before we call some parts of the RAG process:
+            //
+            var chatThreadWithoutWaitingAIBlock = chatThread with { Blocks = chatThread.Blocks[..^1] };
+            
             //
             // When the user wants to bind data sources to the chat, we
             // have to check if the data sources are available for the
@@ -84,7 +108,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 //
                 var retrievalTasks = new List<Task<IReadOnlyList<IRetrievalContext>>>(selectedDataSources.Count);
                 foreach (var dataSource in selectedDataSources)
-                    retrievalTasks.Add(dataSource.RetrieveDataAsync(lastPrompt, chatThread, token));
+                    retrievalTasks.Add(dataSource.RetrieveDataAsync(lastPrompt, chatThreadWithoutWaitingAIBlock, token));
             
                 //
                 // Wait for all retrieval tasks to finish: