AI-Studio/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG.AugmentationProcesses;
using AIStudio.Tools.RAG.DataSourceSelectionProcesses;
using AIStudio.Tools.Services;

namespace AIStudio.Tools.RAG.RAGProcesses;

public sealed class AISrcSelWithRetCtxVal : IRagProcess
{
    private static readonly ILogger<AISrcSelWithRetCtxVal> LOGGER = Program.LOGGER_FACTORY.CreateLogger<AISrcSelWithRetCtxVal>();
    
    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(AISrcSelWithRetCtxVal).Namespace, nameof(AISrcSelWithRetCtxVal));
    
    #region Implementation of IRagProcess

    /// <inheritdoc />
    public string TechnicalName => "AISrcSelWithRetCtxVal";

    /// <inheritdoc />
    public string UIName => TB("AI source selection with AI retrieval context validation");
    
    /// <inheritdoc />
    public string Description => TB("This RAG process filters data sources, automatically selects appropriate sources, optionally allows manual source selection, retrieves data, and automatically validates the retrieval context.");

    /// <inheritdoc />
    public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, CancellationToken token = default)
    {
        var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
        var dataSourceService = Program.SERVICE_PROVIDER.GetService<DataSourceService>()!;
        
        //
        // What an earlier message retrieved must not travel along with this one. The augmented
        // data and the AI-selected data sources describe the last retrieval, not a certain
        // message, and only the steps below ever write them. Without starting from empty, every
        // message which retrieves nothing -- because the data sources were switched off, none of
        // them can be used right now, or the search found nothing -- would still send the
        // passages of the last message which did, cf. ChatThread.RollBackTo.
        //
        // The data security and the required provider confidence stay as they are: both only
        // ever tighten, because the data which raised them was seen by this thread.
        //
        chatThread.AugmentedData = string.Empty;
        chatThread.AISelectedDataSources = [];

        //
        // 1. Check if the user wants to bind any data sources to the chat:
        //
        //
        // Data sources are a preview feature. The check belongs here rather than in the options
        // themselves: a chat keeps its data source options while the feature is switched off, and
        // organizations may preselect data sources through a configuration plugin. Without this,
        // such a chat would still run the entire RAG process with the feature disabled.
        //
        if (PreviewFeatures.PRE_RAG_2024.IsEnabled(settings) && chatThread.DataSourceOptions.IsEnabled())
        {
            LOGGER.LogInformation("Data sources are enabled for this chat.");
            
            // Across the different code-branches, we keep track of whether it
            // makes sense to proceed with the RAG process:
            var proceedWithRAG = true;
            
            //
            // We read the last block in the chat thread. We need to re-arrange
            // the order of blocks later, after the augmentation process takes
            // place:
            //
            if(chatThread.Blocks.Count == 0)
            {
                LOGGER.LogError("The chat thread is empty. Skipping the RAG process.");
                return chatThread;
            }
            
            if (chatThread.Blocks.Last().Role != ChatRole.AI)
            {
                LOGGER.LogError("The last block in the chat thread is not the AI block. There is something wrong with the chat thread. Skipping the RAG process.");
                return chatThread;
            }
            
            //
            // At this point in time, the chat thread contains already the
            // last block, which is the waiting AI block. We need to remove
            // this block before we call some parts of the RAG process:
            //
            var chatThreadWithoutWaitingAIBlock = chatThread with { Blocks = chatThread.Blocks[..^1] };
            
            //
            // When the user wants to bind data sources to the chat, we
            // have to check if the data sources are available for the
            // selected provider. Also, we have to check if any ERI
            // data sources changed its security requirements.
            //
            List<IDataSource> preselectedDataSources = chatThread.DataSourceOptions.PreselectedDataSourceIds.Select(id => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == id)).Where(ds => ds is not null).ToList()!;
            var dataSources = await dataSourceService.GetDataSources(provider, chatThread.DataSourceOptions, preselectedDataSources);
            var selectedDataSources = dataSources.SelectedDataSources;
            
            //
            // Should the AI select the data sources?
            //
            if (chatThread.DataSourceOptions.AutomaticDataSourceSelection)
            {
                var dataSourceSelectionProcess = new AgenticSrcSelWithDynHeur();
                var result = await dataSourceSelectionProcess.SelectDataSourcesAsync(provider, lastUserPrompt, chatThread, dataSources, token);
                proceedWithRAG = result.ProceedWithRAG;
                selectedDataSources = result.SelectedDataSources;
            }
            else
            {
                //
                // No, the user made the choice manually:
                //
                var selectedDataSourceInfo = string.Join(", ", selectedDataSources.Select(ds => $"'{ds.Name}'"));
                LOGGER.LogInformation($"The user selected the data sources manually. {selectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
            }

            if(selectedDataSources.Count == 0)
            {
                //
                // Reaching this point means the user never saw a source of theirs selected: the
                // selection shows what survived the filters, so an empty result there is an empty
                // selection on screen as well. Telling them per answer that their sources were
                // lost would announce a loss they were never shown in the first place. This state
                // belongs into the selection instead, which names the preselected sources it
                // cannot use.
                //
                LOGGER.LogWarning("No data sources are selected. The RAG process is skipped.");
                proceedWithRAG = false;
            }
            else
            {
                var previousDataSecurity = chatThread.DataSecurity;
                var previousRequiredProviderConfidence = chatThread.RequiredProviderConfidence;
                
                //
                // Update the data security of the chat thread. We consider the current data security
                // of the chat thread and the data security of the selected data sources: at least
                // one data source with a SELF_HOSTED policy restricts the chat to self-hosted
                // providers. A restriction set earlier stays either way, because the thread might
                // already contain data from a data source with a SELF_HOSTED policy:
                //
                var dataSecurityRestrictedToSelfHosted = selectedDataSources
                    .OfType<IExternalDataSource>()
                    .Any(dataSource => dataSource.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
                chatThread.RequireDataSecurity(dataSecurityRestrictedToSelfHosted ? DataSourceSecurity.SELF_HOSTED : DataSourceSecurity.ALLOW_ANY);
                
                if (previousDataSecurity != chatThread.DataSecurity)
                    LOGGER.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'.");

                foreach (var dataSource in selectedDataSources.OfType<IInternalDataSource>())
                    chatThread.RequireProviderConfidence(dataSource.ConfidenceLevel);

                if (previousRequiredProviderConfidence != chatThread.RequiredProviderConfidence)
                    LOGGER.LogInformation($"The required provider confidence of the chat thread was updated from '{previousRequiredProviderConfidence.GetName()}' to '{chatThread.RequiredProviderConfidence.GetName()}'.");
            }
            
            //
            // Trigger the retrieval part of the (R)AG process:
            //
            var dataContexts = new List<IRetrievalContext>();
            if (proceedWithRAG)
            {
                //
                // We kick off the retrieval process for each data source in parallel:
                //
                var retrievalTasks = new List<Task<IReadOnlyList<IRetrievalContext>>>(selectedDataSources.Count);
                foreach (var dataSource in selectedDataSources)
                    retrievalTasks.Add(dataSource.RetrieveDataAsync(lastUserPrompt, chatThreadWithoutWaitingAIBlock, token));
            
                //
                // Wait for all retrieval tasks to finish:
                //
                foreach (var retrievalTask in retrievalTasks)
                {
                    try
                    {
                        dataContexts.AddRange(await retrievalTask);
                    }
                    catch (Exception e)
                    {
                        LOGGER.LogError(e, "An error occurred during the retrieval process.");
                    }
                }
            }

            //
            // Perform the augmentation of the R(A)G process:
            //
            if (proceedWithRAG)
            {
                var augmentationProcess = new AugmentationOne();
                chatThread = await augmentationProcess.ProcessAsync(provider, lastUserPrompt, chatThread, dataContexts, token);
            }
            
            //
            // Add sources from the selected data
            //
            
            // We know that the last block is the AI answer block (cf. check above):
            var aiAnswerBlock = chatThread.Blocks.Last();
            var aiAnswerSources = aiAnswerBlock.Content?.Sources;
            
            // It should never happen that the AI answer block does not contain a content part.
            // Just in case, we check this:
            if(aiAnswerSources is null)
                return chatThread;
            
            var ragSources = new List<ISource>();
            foreach (var retrievalContext in dataContexts)
                ragSources.AddRange(retrievalContext.ToSources());

            // Merge the sources, avoiding duplicates:
            aiAnswerSources.MergeSources(ragSources);
        }
        
        return chatThread;
    }

    #endregion
}