Added a semantic search tool (#1005)

app/MindWork AI Studio/Components/DataSourceSelection.razor.cs

@@ -3,6 +3,7 @@ using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.Services;
+using AIStudio.Tools.ToolCallingSystem;
 
 using Microsoft.AspNetCore.Components;
 
@@ -62,7 +63,10 @@ public partial class DataSourceSelection : MSGComponentBase
 
     [Inject]
     private DataSourceService DataSourceService { get; init; } = null!;
-    
+
+    [Inject]
+    private ToolRegistry ToolRegistry { get; init; } = null!;
+
     [Inject]
     private IDialogService DialogService { get; init; } = null!;
 
@@ -78,6 +82,8 @@ public partial class DataSourceSelection : MSGComponentBase
     private bool aiBasedSourceSelection;
     private bool aiBasedValidation;
     private bool areDataSourcesEnabled;
+    private DataSourceRetrievalMode retrievalMode;
+    private EffectiveRetrievalMode effectiveRetrievalMode;
     private uint loadAndApplyFiltersGeneration;
     
     #region Overrides of ComponentBase
@@ -92,6 +98,7 @@ public partial class DataSourceSelection : MSGComponentBase
         this.aiBasedSourceSelection = this.DataSourceOptions.AutomaticDataSourceSelection;
         this.aiBasedValidation = this.DataSourceOptions.AutomaticValidation;
         this.areDataSourcesEnabled = !this.DataSourceOptions.DisableDataSources;
+        this.retrievalMode = this.DataSourceOptions.RetrievalMode;
         this.waitingForDataSources = this.areDataSourcesEnabled && this.SelectionMode is not DataSourceSelectionMode.CONFIGURATION_MODE;
 
         //
@@ -113,6 +120,7 @@ public partial class DataSourceSelection : MSGComponentBase
             this.aiBasedSourceSelection = this.DataSourceOptions.AutomaticDataSourceSelection;
             this.aiBasedValidation = this.DataSourceOptions.AutomaticValidation;
             this.areDataSourcesEnabled = !this.DataSourceOptions.DisableDataSources;
+            this.retrievalMode = this.DataSourceOptions.RetrievalMode;
             this.selectedDataSources = this.GetDataSourcesFromConfiguredIds();
         }
 
@@ -178,6 +186,7 @@ public partial class DataSourceSelection : MSGComponentBase
         this.aiBasedSourceSelection = this.DataSourceOptions.AutomaticDataSourceSelection;
         this.aiBasedValidation = this.DataSourceOptions.AutomaticValidation;
         this.areDataSourcesEnabled = !this.DataSourceOptions.DisableDataSources;
+        this.retrievalMode = this.DataSourceOptions.RetrievalMode;
         this.selectedDataSources = this.GetDataSourcesFromConfiguredIds();
         this.waitingForDataSources = false;
 
@@ -246,10 +255,16 @@ public partial class DataSourceSelection : MSGComponentBase
         // that field holds what was usable the last time we looked, so a source filtered out once
         // would never come back, while the RAG process keeps reading it from the preselection.
         //
-        var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, this.GetDataSourcesFromConfiguredIds());
+        // How the chat searches decides whether the agents of the RAG process see the data, and
+        // with that which data sources the provider of the chat may use at all. This component
+        // only ever selects for a chat:
+        //
+        var effectiveMode = await this.ToolRegistry.GetEffectiveRetrievalModeAsync(this.DataSourceOptions, this.LLMProvider, Tools.Components.CHAT);
+        var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, effectiveMode.Mode, this.GetDataSourcesFromConfiguredIds());
         if (generation != this.loadAndApplyFiltersGeneration)
             return;
 
+        this.effectiveRetrievalMode = effectiveMode;
         this.availableDataSources = sources.AllowedDataSources;
         this.dataSourcesAwaitingReindex = sources.DataSourcesAwaitingReindex;
         this.dataSourceIdsAwaitingReindex = sources.DataSourcesAwaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
@@ -326,6 +341,47 @@ public partial class DataSourceSelection : MSGComponentBase
         await this.OptionsChanged();
     }
     
+    private async Task SemanticSearchChanged(bool state)
+    {
+        this.retrievalMode = state ? DataSourceRetrievalMode.SEMANTIC_SEARCH : DataSourceRetrievalMode.EVERY_MESSAGE;
+        this.DataSourceOptions.RetrievalMode = this.retrievalMode;
+
+        // Which data sources the provider may use depends on it, see LoadAndApplyFilters:
+        await this.LoadAndApplyFilters();
+        await this.OptionsChanged();
+    }
+
+    private bool IsSemanticSearchPreferred => this.retrievalMode is DataSourceRetrievalMode.SEMANTIC_SEARCH;
+
+    /// <summary>
+    /// Whether the model of the chat searches the data sources itself.
+    /// </summary>
+    /// <remarks>
+    /// Only the selection mode can tell, since only it knows the provider of a chat. The effective
+    /// mode is found out while loading the data sources; asking the preference as well keeps a
+    /// switch to every message from showing the other way until that loading is done.
+    /// </remarks>
+    private bool IsSemanticSearchEffective => this.IsSemanticSearchPreferred && this.effectiveRetrievalMode.Mode is DataSourceRetrievalMode.SEMANTIC_SEARCH;
+
+    /// <summary>
+    /// Whether Semantic Search cannot be used in this chat, whatever the user prefers.
+    /// </summary>
+    /// <remarks>
+    /// Then there is nothing to choose: classic RAG searches the data sources. The choice is left
+    /// out rather than shown as one which changes nothing, and the preference stays as it is, so it
+    /// is back the moment the chat gets a model which can search itself.
+    /// </remarks>
+    private bool IsSemanticSearchUnavailable => this.effectiveRetrievalMode.SemanticSearchBlockReason is not ToolOfferBlockReason.NONE;
+
+    private string GetSemanticSearchUnavailableMessage() => this.effectiveRetrievalMode.SemanticSearchBlockReason switch
+    {
+        ToolOfferBlockReason.TOOLS_SWITCHED_OFF => T("Your organization has switched tools off, so AI Studio uses classic RAG instead and searches your data sources with every message."),
+        ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS => T("The selected model cannot use tools, so AI Studio uses classic RAG instead and searches your data sources with every message."),
+        ToolOfferBlockReason.TOOL_SWITCHED_OFF => T("Your organization has switched Semantic Search off, so AI Studio uses classic RAG instead and searches your data sources with every message."),
+        ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW => T("The selected provider is not trusted enough for Semantic Search, so AI Studio uses classic RAG instead and searches your data sources with every message."),
+        _ => T("Semantic Search cannot be used here, so AI Studio uses classic RAG instead and searches your data sources with every message."),
+    };
+
     private async Task ValidationModeChanged(bool state)
     {
         this.aiBasedValidation = state;
@@ -360,6 +416,14 @@ public partial class DataSourceSelection : MSGComponentBase
                && meta.IsLocked;
     }
 
+    private bool IsPreselectedDataSourcesRetrievalModeLocked()
+    {
+        return this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE
+               && this.ConfiguresChatDefaults
+               && ManagedConfiguration.TryGet(x => x.Chat, x => x.PreselectedDataSourcesRetrievalMode, out var meta)
+               && meta.IsLocked;
+    }
+
     private bool IsPreselectedDataSourcesAutomaticSelectionLocked()
     {
         return this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE