diff --git a/README.md b/README.md index 257852d..04c7dbe 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Things we are currently working on: - [ ] Runtime: Integration of the vector database [LanceDB](https://github.com/lancedb/lancedb) - [ ] App: Implement the continuous process of vectorizing data - [x] ~~App: Define a common retrieval context interface for the integration of RAG processes in chats (PR [#281](https://github.com/MindWorkAI/AI-Studio/pull/281), [#284](https://github.com/MindWorkAI/AI-Studio/pull/284), [#286](https://github.com/MindWorkAI/AI-Studio/pull/286), [#287](https://github.com/MindWorkAI/AI-Studio/pull/287))~~ - - [ ] App: Define a common augmentation interface for the integration of RAG processes in chats + - [x] ~~App: Define a common augmentation interface for the integration of RAG processes in chats (PR [#288](https://github.com/MindWorkAI/AI-Studio/pull/288))~~ - [x] ~~App: Integrate data sources in chats (PR [#282](https://github.com/MindWorkAI/AI-Studio/pull/282))~~ diff --git a/app/MindWork AI Studio.sln.DotSettings b/app/MindWork AI Studio.sln.DotSettings index 3f07930..4038c0a 100644 --- a/app/MindWork AI Studio.sln.DotSettings +++ b/app/MindWork AI Studio.sln.DotSettings @@ -7,6 +7,7 @@ MSG RAG UI + True True True True \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ChatRole.cs b/app/MindWork AI Studio/Chat/ChatRole.cs index 339be97..d4e61e4 100644 --- a/app/MindWork AI Studio/Chat/ChatRole.cs +++ b/app/MindWork AI Studio/Chat/ChatRole.cs @@ -12,6 +12,7 @@ public enum ChatRole USER, AI, AGENT, + RAG, } /// diff --git a/app/MindWork AI Studio/Chat/ContentImage.cs b/app/MindWork AI Studio/Chat/ContentImage.cs index 997e998..9f09e87 100644 --- a/app/MindWork AI Studio/Chat/ContentImage.cs +++ b/app/MindWork AI Studio/Chat/ContentImage.cs @@ -7,7 +7,7 @@ namespace AIStudio.Chat; /// /// Represents an image inside the chat. /// -public sealed class ContentImage : IContent +public sealed class ContentImage : IContent, IImageSource { #region Implementation of IContent @@ -47,62 +47,4 @@ public sealed class ContentImage : IContent /// The image source. /// public required string Source { get; set; } - - /// - /// Read the image content as a base64 string. - /// - /// - /// The images are directly converted to base64 strings. The maximum - /// size of the image is around 10 MB. If the image is larger, the method - /// returns an empty string. - /// - /// As of now, this method does no sort of image processing. LLMs usually - /// do not work with arbitrary image sizes. In the future, we might have - /// to resize the images before sending them to the model. - /// - /// The cancellation token. - /// The image content as a base64 string; might be empty. - public async Task AsBase64(CancellationToken token = default) - { - switch (this.SourceType) - { - case ContentImageSource.BASE64: - return this.Source; - - case ContentImageSource.URL: - { - using var httpClient = new HttpClient(); - using var response = await httpClient.GetAsync(this.Source, HttpCompletionOption.ResponseHeadersRead, token); - if(response.IsSuccessStatusCode) - { - // Read the length of the content: - var lengthBytes = response.Content.Headers.ContentLength; - if(lengthBytes > 10_000_000) - return string.Empty; - - var bytes = await response.Content.ReadAsByteArrayAsync(token); - return Convert.ToBase64String(bytes); - } - - return string.Empty; - } - - case ContentImageSource.LOCAL_PATH: - if(File.Exists(this.Source)) - { - // Read the content length: - var length = new FileInfo(this.Source).Length; - if(length > 10_000_000) - return string.Empty; - - var bytes = await File.ReadAllBytesAsync(this.Source, token); - return Convert.ToBase64String(bytes); - } - - return string.Empty; - - default: - return string.Empty; - } - } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/IImageSource.cs b/app/MindWork AI Studio/Chat/IImageSource.cs new file mode 100644 index 0000000..1931630 --- /dev/null +++ b/app/MindWork AI Studio/Chat/IImageSource.cs @@ -0,0 +1,17 @@ +namespace AIStudio.Chat; + +public interface IImageSource +{ + /// + /// The type of the image source. + /// + /// + /// Is the image source a URL, a local file path, a base64 string, etc.? + /// + public ContentImageSource SourceType { get; init; } + + /// + /// The image source. + /// + public string Source { get; set; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs b/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs new file mode 100644 index 0000000..836df53 --- /dev/null +++ b/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs @@ -0,0 +1,63 @@ +namespace AIStudio.Chat; + +public static class IImageSourceExtensions +{ + /// + /// Read the image content as a base64 string. + /// + /// + /// The images are directly converted to base64 strings. The maximum + /// size of the image is around 10 MB. If the image is larger, the method + /// returns an empty string. + /// + /// As of now, this method does no sort of image processing. LLMs usually + /// do not work with arbitrary image sizes. In the future, we might have + /// to resize the images before sending them to the model. + /// + /// The image source. + /// The cancellation token. + /// The image content as a base64 string; might be empty. + public static async Task AsBase64(this IImageSource image, CancellationToken token = default) + { + switch (image.SourceType) + { + case ContentImageSource.BASE64: + return image.Source; + + case ContentImageSource.URL: + { + using var httpClient = new HttpClient(); + using var response = await httpClient.GetAsync(image.Source, HttpCompletionOption.ResponseHeadersRead, token); + if(response.IsSuccessStatusCode) + { + // Read the length of the content: + var lengthBytes = response.Content.Headers.ContentLength; + if(lengthBytes > 10_000_000) + return string.Empty; + + var bytes = await response.Content.ReadAsByteArrayAsync(token); + return Convert.ToBase64String(bytes); + } + + return string.Empty; + } + + case ContentImageSource.LOCAL_PATH: + if(File.Exists(image.Source)) + { + // Read the content length: + var length = new FileInfo(image.Source).Length; + if(length > 10_000_000) + return string.Empty; + + var bytes = await File.ReadAllBytesAsync(image.Source, token); + return Convert.ToBase64String(bytes); + } + + return string.Empty; + + default: + return string.Empty; + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index d0e20d3..6f3ef54 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -37,6 +37,7 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap ChatRole.USER => "user", ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", + ChatRole.RAG => "assistant", _ => "user", }, diff --git a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs index 66817fc..88801a1 100644 --- a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs +++ b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs @@ -49,6 +49,7 @@ public class ProviderFireworks(ILogger logger) : BaseProvider("https://api.firew ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", ChatRole.SYSTEM => "system", + ChatRole.RAG => "assistant", _ => "user", }, diff --git a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs index 942cb24..4fd12ab 100644 --- a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs +++ b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs @@ -50,6 +50,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", ChatRole.SYSTEM => "system", + ChatRole.RAG => "assistant", _ => "user", }, diff --git a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs index f32a31b..10e8899 100644 --- a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs +++ b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs @@ -50,6 +50,7 @@ public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/o ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", ChatRole.SYSTEM => "system", + ChatRole.RAG => "assistant", _ => "user", }, diff --git a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs index 024f60d..4395920 100644 --- a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs +++ b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs @@ -48,6 +48,7 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api. ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", ChatRole.SYSTEM => "system", + ChatRole.RAG => "assistant", _ => "user", }, diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index ed09217..f901925 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -76,6 +76,7 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o ChatRole.USER => "user", ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", + ChatRole.RAG => "assistant", ChatRole.SYSTEM => systemPromptRole, _ => "user", diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs index 4ba45c6..73c101a 100644 --- a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs +++ b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs @@ -46,6 +46,7 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", ChatRole.SYSTEM => "system", + ChatRole.RAG => "assistant", _ => "user", }, diff --git a/app/MindWork AI Studio/Provider/X/ProviderX.cs b/app/MindWork AI Studio/Provider/X/ProviderX.cs index a8334c8..df927d7 100644 --- a/app/MindWork AI Studio/Provider/X/ProviderX.cs +++ b/app/MindWork AI Studio/Provider/X/ProviderX.cs @@ -50,6 +50,7 @@ public sealed class ProviderX(ILogger logger) : BaseProvider("https://api.x.ai/v ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", ChatRole.SYSTEM => "system", + ChatRole.RAG => "assistant", _ => "user", }, diff --git a/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs b/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs new file mode 100644 index 0000000..1d0fc0e --- /dev/null +++ b/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs @@ -0,0 +1,119 @@ +using System.Text; + +using AIStudio.Chat; +using AIStudio.Provider; + +namespace AIStudio.Tools.RAG.AugmentationProcesses; + +public sealed class AugmentationOne : IAugmentationProcess +{ + #region Implementation of IAugmentationProcess + + /// + public string TechnicalName => "AugmentationOne"; + + /// + public string UIName => "Standard augmentation process"; + + /// + public string Description => "This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread."; + + /// + public async Task ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, IReadOnlyList retrievalContexts, CancellationToken token = default) + { + var logger = Program.SERVICE_PROVIDER.GetService>()!; + if(retrievalContexts.Count == 0) + { + logger.LogWarning("No retrieval contexts were issued. Skipping the augmentation process."); + return chatThread; + } + + var numTotalRetrievalContexts = retrievalContexts.Count; + logger.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts."); + + // + // We build a huge prompt from all retrieval contexts: + // + var sb = new StringBuilder(); + sb.AppendLine("The following useful information will help you in processing the user prompt:"); + sb.AppendLine(); + + var index = 0; + foreach(var retrievalContext in retrievalContexts) + { + index++; + sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}"); + sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}"); + sb.AppendLine($"Content category: {retrievalContext.Category}"); + sb.AppendLine($"Content type: {retrievalContext.Type}"); + sb.AppendLine($"Content path: {retrievalContext.Path}"); + + if(retrievalContext.Links.Count > 0) + { + sb.AppendLine("Additional links:"); + foreach(var link in retrievalContext.Links) + sb.AppendLine($"- {link}"); + } + + switch(retrievalContext) + { + case RetrievalTextContext textContext: + sb.AppendLine(); + sb.AppendLine("Matched text content:"); + sb.AppendLine("````"); + sb.AppendLine(textContext.MatchedText); + sb.AppendLine("````"); + + if(textContext.SurroundingContent.Count > 0) + { + sb.AppendLine(); + sb.AppendLine("Surrounding text content:"); + foreach(var surrounding in textContext.SurroundingContent) + { + sb.AppendLine(); + sb.AppendLine("````"); + sb.AppendLine(surrounding); + sb.AppendLine("````"); + } + } + + + break; + + case RetrievalImageContext imageContext: + sb.AppendLine(); + sb.AppendLine("Matched image content as base64-encoded data:"); + sb.AppendLine("````"); + sb.AppendLine(await imageContext.AsBase64(token)); + sb.AppendLine("````"); + break; + + default: + logger.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet."); + break; + } + + sb.AppendLine(); + } + + // + // Append the entire augmentation to the chat thread, + // just before the user prompt: + // + chatThread.Blocks.Insert(chatThread.Blocks.Count - 1, new() + { + Role = ChatRole.RAG, + Time = DateTimeOffset.UtcNow, + ContentType = ContentType.TEXT, + HideFromUser = true, + Content = new ContentText + { + Text = sb.ToString(), + } + }); + + return chatThread; + } + + #endregion +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/RAG/IAugmentationProcess.cs b/app/MindWork AI Studio/Tools/RAG/IAugmentationProcess.cs new file mode 100644 index 0000000..81bb6da --- /dev/null +++ b/app/MindWork AI Studio/Tools/RAG/IAugmentationProcess.cs @@ -0,0 +1,33 @@ +using AIStudio.Chat; +using AIStudio.Provider; + +namespace AIStudio.Tools.RAG; + +public interface IAugmentationProcess +{ + /// + /// How is the augmentation process called? + /// + public string TechnicalName { get; } + + /// + /// How is the augmentation process called in the UI? + /// + public string UIName { get; } + + /// + /// How works the augmentation process? + /// + public string Description { get; } + + /// + /// Starts the augmentation process. + /// + /// The LLM provider. Gets used, e.g., for automatic retrieval context validation. + /// The last prompt that was issued by the user. + /// The chat thread. + /// The retrieval contexts that were issued by the retrieval process. + /// The cancellation token. + /// The altered chat thread. + public Task ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, IReadOnlyList retrievalContexts, CancellationToken token = default); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs index 8581a3e..13e3250 100644 --- a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs +++ b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs @@ -1,6 +1,7 @@ using AIStudio.Chat; using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.RAG.AugmentationProcesses; using AIStudio.Tools.RAG.DataSourceSelectionProcesses; using AIStudio.Tools.Services; @@ -106,7 +107,8 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess // if (proceedWithRAG) { - + var augmentationProcess = new AugmentationOne(); + chatThread = await augmentationProcess.ProcessAsync(provider, lastPrompt, chatThread, dataContexts, token); } } diff --git a/app/MindWork AI Studio/Tools/RAG/RetrievalImageContext.cs b/app/MindWork AI Studio/Tools/RAG/RetrievalImageContext.cs index 37cadb9..973892d 100644 --- a/app/MindWork AI Studio/Tools/RAG/RetrievalImageContext.cs +++ b/app/MindWork AI Studio/Tools/RAG/RetrievalImageContext.cs @@ -2,7 +2,7 @@ using AIStudio.Chat; namespace AIStudio.Tools.RAG; -public sealed class RetrievalImageContext : IRetrievalContext +public sealed class RetrievalImageContext : IRetrievalContext, IImageSource { #region Implementation of IRetrievalContext diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md b/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md index 358146c..3d86143 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md @@ -3,5 +3,6 @@ - Added an option to all data sources to select a local security policy. This preview feature is hidden behind the RAG feature flag. - Added an option to preselect data sources and options for new chats. This preview feature is hidden behind the RAG feature flag. - Added an agent to select the appropriate data sources for any prompt. This preview feature is hidden behind the RAG feature flag. +- Added a generic RAG process to integrate possibly any data in your chats. Although the generic RAG process is now implemented, the retrieval part is working only with external data sources using the ERI interface. That means that you could integrate your company's data from the corporate network by now. The retrieval process for your local data is still under development and will take several weeks to be released. In order to use data through ERI, you (or your company) have to develop an ERI server. You might use the ERI server assistant within AI Studio to do so. This preview feature is hidden behind the RAG feature flag. - Improved confidence card for small spaces. - Fixed a bug in which 'APP_SETTINGS' appeared as a valid destination in the "send to" menu. \ No newline at end of file