From 50c1b457354c09dc9b6629d536735a2c3e481f22 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 18 Feb 2025 13:40:31 +0100 Subject: [PATCH] Refactored the conversion from retrieval contexts to Markdown --- .../AugmentationProcesses/AugmentationOne.cs | 59 +----------- .../Tools/RAG/IRetrievalContextExtensions.cs | 96 +++++++++++++++++++ 2 files changed, 98 insertions(+), 57 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs diff --git a/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs b/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs index 1d0fc0e1..fda8cf37 100644 --- a/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs +++ b/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs @@ -38,63 +38,8 @@ public sealed class AugmentationOne : IAugmentationProcess sb.AppendLine("The following useful information will help you in processing the user prompt:"); sb.AppendLine(); - var index = 0; - foreach(var retrievalContext in retrievalContexts) - { - index++; - sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}"); - sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}"); - sb.AppendLine($"Content category: {retrievalContext.Category}"); - sb.AppendLine($"Content type: {retrievalContext.Type}"); - sb.AppendLine($"Content path: {retrievalContext.Path}"); - - if(retrievalContext.Links.Count > 0) - { - sb.AppendLine("Additional links:"); - foreach(var link in retrievalContext.Links) - sb.AppendLine($"- {link}"); - } - - switch(retrievalContext) - { - case RetrievalTextContext textContext: - sb.AppendLine(); - sb.AppendLine("Matched text content:"); - sb.AppendLine("````"); - sb.AppendLine(textContext.MatchedText); - sb.AppendLine("````"); - - if(textContext.SurroundingContent.Count > 0) - { - sb.AppendLine(); - sb.AppendLine("Surrounding text content:"); - foreach(var surrounding in textContext.SurroundingContent) - { - sb.AppendLine(); - sb.AppendLine("````"); - sb.AppendLine(surrounding); - sb.AppendLine("````"); - } - } - - - break; - - case RetrievalImageContext imageContext: - sb.AppendLine(); - sb.AppendLine("Matched image content as base64-encoded data:"); - sb.AppendLine("````"); - sb.AppendLine(await imageContext.AsBase64(token)); - sb.AppendLine("````"); - break; - - default: - logger.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet."); - break; - } - - sb.AppendLine(); - } + // Let's convert all retrieval contexts to Markdown: + await retrievalContexts.AsMarkdown(sb, token); // // Append the entire augmentation to the chat thread, diff --git a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs new file mode 100644 index 00000000..ee5e6cb5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs @@ -0,0 +1,96 @@ +using System.Text; + +using AIStudio.Chat; + +namespace AIStudio.Tools.RAG; + +public static class IRetrievalContextExtensions +{ + private static readonly ILogger LOGGER = Program.SERVICE_PROVIDER.GetService>()!; + + public static async Task AsMarkdown(this IReadOnlyList retrievalContexts, StringBuilder? sb = null, CancellationToken token = default) + { + sb ??= new StringBuilder(); + var index = 0; + + foreach(var retrievalContext in retrievalContexts) + { + index++; + await retrievalContext.AsMarkdown(sb, index, retrievalContexts.Count, token); + } + + return sb.ToString(); + } + + public static async Task AsMarkdown(this IRetrievalContext retrievalContext, StringBuilder? sb = null, int index = -1, int numTotalRetrievalContexts = -1, CancellationToken token = default) + { + sb ??= new StringBuilder(); + switch (index) + { + case > 0 when numTotalRetrievalContexts is -1: + sb.AppendLine($"# Retrieval context {index}"); + break; + + case > 0 when numTotalRetrievalContexts > 0: + sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}"); + break; + + default: + sb.AppendLine("# Retrieval context"); + break; + } + + sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}"); + sb.AppendLine($"Content category: {retrievalContext.Category}"); + sb.AppendLine($"Content type: {retrievalContext.Type}"); + sb.AppendLine($"Content path: {retrievalContext.Path}"); + + if(retrievalContext.Links.Count > 0) + { + sb.AppendLine("Additional links:"); + foreach(var link in retrievalContext.Links) + sb.AppendLine($"- {link}"); + } + + switch(retrievalContext) + { + case RetrievalTextContext textContext: + sb.AppendLine(); + sb.AppendLine("Matched text content:"); + sb.AppendLine("````"); + sb.AppendLine(textContext.MatchedText); + sb.AppendLine("````"); + + if(textContext.SurroundingContent.Count > 0) + { + sb.AppendLine(); + sb.AppendLine("Surrounding text content:"); + foreach(var surrounding in textContext.SurroundingContent) + { + sb.AppendLine(); + sb.AppendLine("````"); + sb.AppendLine(surrounding); + sb.AppendLine("````"); + } + } + + + break; + + case RetrievalImageContext imageContext: + sb.AppendLine(); + sb.AppendLine("Matched image content as base64-encoded data:"); + sb.AppendLine("````"); + sb.AppendLine(await imageContext.AsBase64(token)); + sb.AppendLine("````"); + break; + + default: + LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet."); + break; + } + + sb.AppendLine(); + return sb.ToString(); + } +} \ No newline at end of file