diff --git a/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs b/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs new file mode 100644 index 00000000..53875f13 --- /dev/null +++ b/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs @@ -0,0 +1,319 @@ +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools.RAG; +using AIStudio.Tools.Services; + +namespace AIStudio.Agents; + +public sealed class AgentRetrievalContextValidation (ILogger logger, ILogger baseLogger, SettingsManager settingsManager, DataSourceService dataSourceService, ThreadSafeRandom rng) : AgentBase(baseLogger, settingsManager, dataSourceService, rng) +{ + #region Overrides of AgentBase + + /// + protected override Type Type => Type.WORKER; + + /// + public override string Id => "Retrieval Context Validation"; + + /// + protected override string JobDescription => + """ + You receive a system and user prompt as well as a retrieval context as input. Your task is to decide whether this + retrieval context is helpful in processing the prompts or not. You respond with the decision (true or false), + your reasoning, and your confidence in this decision. + + Your response is only one JSON object in the following format: + + ``` + {"decision": true, "reason": "Why did you choose this source?", "confidence": 0.87} + ``` + + You express your confidence as a floating-point number between 0.0 (maximum uncertainty) and + 1.0 (you are absolutely certain that this retrieval context is needed). + + The JSON schema is: + + ``` + { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "decision": { + "type": "boolean" + }, + "reason": { + "type": "string" + }, + "confidence": { + "type": "number" + } + }, + "required": [ + "decision", + "reason", + "confidence" + ] + } + ``` + + You do not ask any follow-up questions. You do not address the user. Your response consists solely of + that one JSON object. + """; + + /// + protected override string SystemPrompt(string retrievalContext) => $""" + {this.JobDescription} + + {retrievalContext} + """; + + /// + public override Settings.Provider? ProviderSettings { get; set; } + + /// + /// The retrieval context validation agent does not work with context. Use + /// the process input method instead. + /// + /// The chat thread without any changes. + public override Task ProcessContext(ChatThread chatThread, IDictionary additionalData) => Task.FromResult(chatThread); + + /// + public override async Task ProcessInput(ContentBlock input, IDictionary additionalData) + { + if (input.Content is not ContentText text) + return EMPTY_BLOCK; + + if(text.InitialRemoteWait || text.IsStreaming) + return EMPTY_BLOCK; + + if(string.IsNullOrWhiteSpace(text.Text)) + return EMPTY_BLOCK; + + if(!additionalData.TryGetValue("retrievalContext", out var retrievalContext) || string.IsNullOrWhiteSpace(retrievalContext)) + return EMPTY_BLOCK; + + var thread = this.CreateChatThread(this.SystemPrompt(retrievalContext)); + var time = this.AddUserRequest(thread, text.Text); + await this.AddAIResponseAsync(thread, time); + + return thread.Blocks[^1]; + } + + /// + public override Task MadeDecision(ContentBlock input) => Task.FromResult(true); + + /// + /// We do not provide any context. This agent will process many retrieval contexts. + /// This would block a huge amount of memory. + /// + /// An empty list. + public override IReadOnlyCollection GetContext() => []; + + /// + /// We do not provide any answers. This agent will process many retrieval contexts. + /// This would block a huge amount of memory. + /// + /// An empty list. + public override IReadOnlyCollection GetAnswers() => []; + + #endregion + + public async Task ValidateDataSampleAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, IRetrievalContext dataContext, CancellationToken token = default) + { + // + // 1. Which LLM provider should the agent use? + // + + // We start with the provider currently selected by the user: + var agentProvider = this.SettingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == provider.Id); + + // If the user preselected an agent provider, we try to use this one: + if (this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectAgentOptions) + { + var configuredAgentProvider = this.SettingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.PreselectedAgentProvider); + + // If the configured agent provider is available, we use it: + if (configuredAgentProvider != default) + agentProvider = configuredAgentProvider; + } + + // Assign the provider settings to the agent: + logger.LogInformation($"The agent for the retrieval context validation uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()})."); + this.ProviderSettings = agentProvider; + + // + // 2. Prepare the current system and user prompts as input for the agent: + // + var lastPromptContent = lastPrompt switch + { + ContentText text => text.Text, + + // Image prompts may be empty, e.g., when the image is too large: + ContentImage image => await image.AsBase64(token), + + // Other content types are not supported yet: + _ => string.Empty, + }; + + if (string.IsNullOrWhiteSpace(lastPromptContent)) + { + logger.LogWarning("The last prompt is empty. The AI cannot validate the retrieval context."); + return new(false, "The last prompt was empty.", 1.0f); + } + + // + // 3. Prepare the retrieval context for the agent: + // + var additionalData = new Dictionary(); + var markdownRetrievalContext = await dataContext.AsMarkdown(token: token); + additionalData.Add("retrievalContext", markdownRetrievalContext); + + // + // 4. Let the agent validate the retrieval context: + // + var prompt = $""" + The system prompt is: + + ``` + {chatThread.SystemPrompt} + ``` + + The user prompt is: + + ``` + {lastPromptContent} + ``` + """; + + // Call the agent: + var aiResponse = await this.ProcessInput(new ContentBlock + { + Time = DateTimeOffset.UtcNow, + ContentType = ContentType.TEXT, + Role = ChatRole.USER, + Content = new ContentText + { + Text = prompt, + }, + }, additionalData); + + if(aiResponse.Content is null) + { + logger.LogWarning("The agent did not return a response."); + return new(false, "The agent did not return a response.", 1.0f); + } + + switch (aiResponse) + { + + // + // 5. Parse the agent response: + // + case { ContentType: ContentType.TEXT, Content: ContentText textContent }: + { + // + // What we expect is one JSON object: + // + var validationJson = textContent.Text; + + // + // We know how bad LLM may be in generating JSON without surrounding text. + // Thus, we expect the worst and try to extract the JSON list from the text: + // + var json = ExtractJson(validationJson); + + try + { + return JsonSerializer.Deserialize(json, JSON_SERIALIZER_OPTIONS); + } + catch + { + logger.LogWarning("The agent answered with an invalid or unexpected JSON format."); + return new(false, "The agent answered with an invalid or unexpected JSON format.", 1.0f); + } + } + + case { ContentType: ContentType.TEXT }: + logger.LogWarning("The agent answered with an unexpected inner content type."); + return new(false, "The agent answered with an unexpected inner content type.", 1.0f); + + case { ContentType: ContentType.NONE }: + logger.LogWarning("The agent did not return a response."); + return new(false, "The agent did not return a response.", 1.0f); + + default: + logger.LogWarning($"The agent answered with an unexpected content type '{aiResponse.ContentType}'."); + return new(false, $"The agent answered with an unexpected content type '{aiResponse.ContentType}'.", 1.0f); + } + } + + // A wrapper around the span version, because we need to call this method from an async context. + private static string ExtractJson(string text) => ExtractJson(text.AsSpan()).ToString(); + + private static ReadOnlySpan ExtractJson(ReadOnlySpan input) + { + // + // 1. Expect the best case ;-) + // + if (CheckJsonObjectStart(input)) + return ExtractJsonPart(input); + + // + // 2. Okay, we have some garbage before the + // JSON object. We expected that... + // + for (var index = 0; index < input.Length; index++) + { + if (input[index] is '{' && CheckJsonObjectStart(input[index..])) + return ExtractJsonPart(input[index..]); + } + + return []; + } + + private static bool CheckJsonObjectStart(ReadOnlySpan area) + { + char[] expectedSymbols = ['{', '"', 'd']; + var symbolIndex = 0; + + foreach (var c in area) + { + if (symbolIndex >= expectedSymbols.Length) + return true; + + if (char.IsWhiteSpace(c)) + continue; + + if (c == expectedSymbols[symbolIndex++]) + continue; + + return false; + } + + return true; + } + + private static ReadOnlySpan ExtractJsonPart(ReadOnlySpan input) + { + var insideString = false; + for (var index = 0; index < input.Length; index++) + { + if (input[index] is '"') + { + insideString = !insideString; + continue; + } + + if (insideString) + continue; + + if (input[index] is '}') + return input[..++index]; + } + + return []; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Agents/RetrievalContextValidationResult.cs b/app/MindWork AI Studio/Agents/RetrievalContextValidationResult.cs new file mode 100644 index 00000000..4b3bd810 --- /dev/null +++ b/app/MindWork AI Studio/Agents/RetrievalContextValidationResult.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Agents; + +/// +/// Represents the result of a retrieval context validation. +/// +/// Whether the retrieval context is useful or not. +/// The reason for the decision. +/// The confidence of the decision. +public readonly record struct RetrievalContextValidationResult(bool Decision, string Reason, float Confidence); \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelAgentRetrievalContextValidation.razor b/app/MindWork AI Studio/Components/Settings/SettingsPanelAgentRetrievalContextValidation.razor new file mode 100644 index 00000000..bc8f78c3 --- /dev/null +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelAgentRetrievalContextValidation.razor @@ -0,0 +1,13 @@ +@inherits SettingsPanelBase + + + + + Use Case: this agent is used to validate any retrieval context of the retrieval process. Perhaps there are many of these + retrieval contexts and you want to validate them all. Therefore, you might want to use a cheap and fast LLM for this + job. When using a local or self-hosted LLM, look for a small (e.g. 3B) and fast model. + + + + + \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelAgentRetrievalContextValidation.razor.cs b/app/MindWork AI Studio/Components/Settings/SettingsPanelAgentRetrievalContextValidation.razor.cs new file mode 100644 index 00000000..aaf0d938 --- /dev/null +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelAgentRetrievalContextValidation.razor.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Components.Settings; + +public partial class SettingsPanelAgentRetrievalContextValidation : SettingsPanelBase; \ No newline at end of file diff --git a/app/MindWork AI Studio/Pages/Settings.razor b/app/MindWork AI Studio/Pages/Settings.razor index 34a8a186..4170eb81 100644 --- a/app/MindWork AI Studio/Pages/Settings.razor +++ b/app/MindWork AI Studio/Pages/Settings.razor @@ -41,6 +41,7 @@ @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager)) { + } diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index a9ae7a97..61261b16 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -119,6 +119,7 @@ internal sealed class Program builder.Services.AddSingleton(); builder.Services.AddTransient(); builder.Services.AddTransient(); + builder.Services.AddTransient(); builder.Services.AddTransient(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); diff --git a/app/MindWork AI Studio/Settings/DataModel/Data.cs b/app/MindWork AI Studio/Settings/DataModel/Data.cs index 729cfe48..b7139c72 100644 --- a/app/MindWork AI Studio/Settings/DataModel/Data.cs +++ b/app/MindWork AI Studio/Settings/DataModel/Data.cs @@ -76,6 +76,8 @@ public sealed class Data public DataAgentDataSourceSelection AgentDataSourceSelection { get; init; } = new(); + public DataAgentRetrievalContextValidation AgentRetrievalContextValidation { get; init; } = new(); + public DataAgenda Agenda { get; init; } = new(); public DataGrammarSpelling GrammarSpelling { get; init; } = new(); diff --git a/app/MindWork AI Studio/Settings/DataModel/DataAgentRetrievalContextValidation.cs b/app/MindWork AI Studio/Settings/DataModel/DataAgentRetrievalContextValidation.cs new file mode 100644 index 00000000..5cf3b186 --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataAgentRetrievalContextValidation.cs @@ -0,0 +1,14 @@ +namespace AIStudio.Settings.DataModel; + +public sealed class DataAgentRetrievalContextValidation +{ + /// + /// Preselect any retrieval context validation options? + /// + public bool PreselectAgentOptions { get; set; } + + /// + /// Preselect a retrieval context validation provider? + /// + public string PreselectedAgentProvider { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md b/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md index 3d861432..c9037877 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md @@ -3,6 +3,7 @@ - Added an option to all data sources to select a local security policy. This preview feature is hidden behind the RAG feature flag. - Added an option to preselect data sources and options for new chats. This preview feature is hidden behind the RAG feature flag. - Added an agent to select the appropriate data sources for any prompt. This preview feature is hidden behind the RAG feature flag. +- Added an agent to validate whether a retrieval context makes sense for the given prompt. This preview feature is hidden behind the RAG feature flag. - Added a generic RAG process to integrate possibly any data in your chats. Although the generic RAG process is now implemented, the retrieval part is working only with external data sources using the ERI interface. That means that you could integrate your company's data from the corporate network by now. The retrieval process for your local data is still under development and will take several weeks to be released. In order to use data through ERI, you (or your company) have to develop an ERI server. You might use the ERI server assistant within AI Studio to do so. This preview feature is hidden behind the RAG feature flag. - Improved confidence card for small spaces. - Fixed a bug in which 'APP_SETTINGS' appeared as a valid destination in the "send to" menu. \ No newline at end of file