Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Components/Settings/SettingsPanelDataSources.razor

@@ -0,0 +1,10 @@
+@using AIStudio.Settings.DataModel
+@inherits SettingsPanelBase
+
+@if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
+{
+    <ExpansionPanel HeaderIcon="@AppIcons.DATABASE" HeaderText="@T("Configure Data Sources")">
+        <PreviewBeta ApplyInnerScrollingFix="true"/>
+        <DataSourceManagement/>
+    </ExpansionPanel>
+}

app/MindWork AI Studio/Components/Settings/SettingsPanelDataSources.razor.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Components.Settings;
+
+public partial class SettingsPanelDataSources : SettingsPanelBase;

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor

@@ -6,7 +6,7 @@
 @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
 {
     <ExpansionPanel HeaderIcon="@Icons.Material.Filled.IntegrationInstructions" HeaderText="@T("Configure Embedding Providers")">
-        <PreviewPrototype ApplyInnerScrollingFix="true"/>
+        <PreviewBeta ApplyInnerScrollingFix="true"/>
         <MudText Typo="Typo.h4" Class="mb-3">
             @T("Configured Embedding Providers")
         </MudText>

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs

@@ -2,6 +2,7 @@ using System.Globalization;
 using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
@@ -11,6 +12,9 @@ namespace AIStudio.Components.Settings;
 
 public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
 {
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     /// <summary>
     /// Groups the table by the used LLM provider. The embedding provider list is already sorted by
     /// that provider, so all instances of one LLM provider form a single, coherent group.
@@ -68,6 +72,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
         await this.UpdateEmbeddingProviders();
         
         await this.SettingsManager.StoreSettings();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
     
@@ -88,6 +93,9 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             { x => x.IsSelfHosted, embeddingProvider.IsSelfHosted },
             { x => x.IsEditing, true },
             { x => x.DataHost, embeddingProvider.Host },
+            { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
+            { x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit },
+            { x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize },
             { x => x.HFInferenceProviderId, embeddingProvider.HFInferenceProvider },
             { x => x.IsEnterpriseConfiguration, embeddingProvider.IsEnterpriseConfiguration },
         };
@@ -118,6 +126,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
         await this.UpdateEmbeddingProviders();
         
         await this.SettingsManager.StoreSettings();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
@@ -134,13 +143,36 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             return;
         
         var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.EMBEDDING_PROVIDER);
+
+        //
+        // Removing the tokenizer is best effort: it leaves an unused file behind when it fails,
+        // which is not worth bothering the user about while they are deleting the provider. The
+        // API key is different, though, because a leftover secret is a secret we promised to remove.
+        //
+        _ = await this.RustService.DeleteTokenizer(TokenizerModelId.ForEmbeddingProvider(provider));
         if(deleteSecretResponse.Success)
         {
             this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
             await this.SettingsManager.StoreSettings();
         }
+        else
+        {
+            var issueDialogParameters = new DialogParameters<ConfirmDialog>
+            {
+                { x => x.Message, string.Format(T("Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?"), provider.Name, deleteSecretResponse.Issue) },
+            };
+
+            var issueDialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Embedding Provider"), issueDialogParameters, DialogOptions.FULLSCREEN);
+            var issueDialogResult = await issueDialogReference.Result;
+            if (issueDialogResult is null || issueDialogResult.Canceled)
+                return;
+
+            this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
+            await this.SettingsManager.StoreSettings();
+        }
 
         await this.UpdateEmbeddingProviders();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
@@ -183,7 +215,21 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             return;
 
         var embeddingProvider = provider.CreateProvider();
-        var embeddings = await embeddingProvider.EmbedTextAsync(provider.Model, this.SettingsManager, default, new List<string> { inputText });
+        IReadOnlyList<IReadOnlyList<float>> embeddings;
+        try
+        {
+            embeddings = await embeddingProvider.EmbedTextAsync(provider.Model, this.SettingsManager, CancellationToken.None, inputText);
+        }
+        catch (ProviderRequestException exception)
+        {
+            //
+            // The provider named what went wrong and what to do about it. Showing that beats the
+            // sentence below, which used to be the same one for a missing API key, an unreachable
+            // provider and a provider which cannot embed anything at all:
+            //
+            await this.DialogService.ShowMessageBox(T("Embedding Result"), exception.UserMessage, T("Close"));
+            return;
+        }
 
         if (embeddings.Count == 0)
         {

app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs

@@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis;
 
 using AIStudio.Dialogs;
 using AIStudio.Settings;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
@@ -84,6 +85,7 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
             { x => x.DataHost, provider.Host },
             { x => x.HFInferenceProviderId, provider.HFInferenceProvider },
             { x => x.AdditionalJsonApiParameters, provider.AdditionalJsonApiParameters },
+            { x => x.DataTokenizerPath, provider.TokenizerPath },
             { x => x.DataCapabilityOverrides, provider.CapabilityOverrides },
             { x => x.IsEnterpriseConfiguration, provider.IsEnterpriseConfiguration },
         };
@@ -131,6 +133,13 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
             return;
         
         var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.LLM_PROVIDER);
+
+        //
+        // Removing the tokenizer is best effort: it leaves an unused file behind when it fails,
+        // which is not worth bothering the user about while they are deleting the provider. The
+        // API key is different, though, because a leftover secret is a secret we promised to remove.
+        //
+        _ = await this.RustService.DeleteTokenizer(TokenizerModelId.ForProvider(provider));
         if(deleteSecretResponse.Success)
         {
             this.SettingsManager.ConfigurationData.Providers.Remove(provider);