Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs

@@ -295,6 +295,17 @@ public static class ConfigurationSelectDataFactory
             }
         }
     }
+
+    public static IEnumerable<ConfigurationSelectData<ConfidenceLevel>> GetDataSourceConfidenceLevelsData()
+    {
+        foreach (var level in Enum.GetValues<ConfidenceLevel>())
+        {
+            if (level is ConfidenceLevel.NONE)
+                continue;
+
+            yield return new(level.GetName(), level);
+        }
+    }
     
     public static IEnumerable<ConfigurationSelectData<Themes>> GetThemesData()
     {

app/MindWork AI Studio/Settings/DataModel/DataApp.cs

@@ -179,6 +179,11 @@ public sealed class DataApp(Expression<Func<Data, DataApp>>? configSelection = n
     /// </summary>
     public bool ShowAdminSettings { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ShowAdminSettings, false);
 
+    /// <summary>
+    /// Settings for indexing local data sources.
+    /// </summary>
+    public DataDataSourceIndexing DataSourceIndexing { get; init; } = new();
+
     /// <summary>
     /// List of assistants that should be hidden from the UI.
     /// </summary>

app/MindWork AI Studio/Settings/DataModel/DataDataSourceIndexing.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Settings.DataModel;
+
+public sealed class DataDataSourceIndexing
+{
+    /// <summary>
+    /// Whether local data source embeddings should refresh automatically when files change.
+    /// </summary>
+    public bool AutomaticRefresh { get; set; } = true;
+}

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -7,6 +7,7 @@ using AIStudio.Tools.ERIClient.DataModel;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.RAG;
 using AIStudio.Tools.Services;
+using AIStudio.Tools.Validation;
 
 using SharedTools;
 
@@ -164,9 +165,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             return false;
         }
 
-        if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name))
+        if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || !DataSourceValidation.IsNameValid(name))
         {
-            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name. (Plugin ID: {configPluginId})");
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most {DataSourceValidation.MAX_NAME_LENGTH} characters without control characters. (Plugin ID: {configPluginId})");
             return false;
         }
 
@@ -390,4 +391,4 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
         var cleanedHostname = hostname.Trim();
         return cleanedHostname.EndsWith('/') ? cleanedHostname[..^1] : cleanedHostname;
     }
-}
+}

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -1,5 +1,7 @@
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Settings.DataModel;
 
@@ -32,9 +34,15 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     
     /// <inheritdoc />
     public string EmbeddingId { get; init; } = Guid.Empty.ToString();
+
+    /// <inheritdoc />
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <inheritdoc />
+    public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     
     /// <inheritdoc />
-    public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+    public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
 
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
@@ -46,11 +54,8 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     public ushort MaxMatches { get; init; } = 10;
     
     /// <inheritdoc />
-    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default)
-    {
-        IReadOnlyList<IRetrievalContext> retrievalContext = new List<IRetrievalContext>();
-        return Task.FromResult(retrievalContext);
-    }
+    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
     /// <summary>
     /// The path to the directory.

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -1,5 +1,7 @@
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Settings.DataModel;
 
@@ -32,9 +34,15 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     
     /// <inheritdoc />
     public string EmbeddingId { get; init; } = Guid.Empty.ToString();
+
+    /// <inheritdoc />
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <inheritdoc />
+    public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     
     /// <inheritdoc />
-    public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+    public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
 
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
@@ -46,11 +54,8 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     public ushort MaxMatches { get; init; } = 10;
     
     /// <inheritdoc />
-    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default)
-    {
-        IReadOnlyList<IRetrievalContext> retrievalContext = new List<IRetrievalContext>();
-        return Task.FromResult(retrievalContext);
-    }
+    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
     /// <summary>
     /// The path to the file.

app/MindWork AI Studio/Settings/DataModel/PreviewVisibilityExtensions.cs

@@ -13,6 +13,7 @@ public static class PreviewVisibilityExtensions
         {
             features.Add(PreviewFeatures.PRE_DOCUMENT_ANALYSIS_2025);
             features.Add(PreviewFeatures.PRE_META_ASSISTANT_V1);
+            features.Add(PreviewFeatures.PRE_RAG_2024);
         }
         
         if (visibility >= PreviewVisibility.ALPHA)
@@ -21,7 +22,6 @@ public static class PreviewVisibilityExtensions
         
         if (visibility >= PreviewVisibility.PROTOTYPE)
         {
-            features.Add(PreviewFeatures.PRE_RAG_2024);
             features.Add(PreviewFeatures.PRE_VISUAL_BRIEFING_ASSISTANT_2026);
         }
         

app/MindWork AI Studio/Settings/DataSourceSecurityTrustExtensions.cs

@@ -1,4 +1,5 @@
 using AIStudio.Provider;
+using AIStudio.Settings.DataModel;
 
 namespace AIStudio.Settings;
 
@@ -36,6 +37,88 @@ public static class DataSourceSecurityTrustExtensions
         return provider.Provider is LLMProviders.SELF_HOSTED || IsTrustedProviderId(provider.ConfiguredProviderId, settingsManager);
     }
 
+    public static ConfidenceLevel GetConfidenceLevel(this Provider provider, SettingsManager settingsManager)
+    {
+        if (provider == Provider.NONE)
+            return ConfidenceLevel.NONE;
+
+        return provider.UsedLLMProvider.GetConfidence(settingsManager).Level;
+    }
+
+    public static ConfidenceLevel GetConfidenceLevel(this EmbeddingProvider provider, SettingsManager settingsManager)
+    {
+        if (provider == EmbeddingProvider.NONE)
+            return ConfidenceLevel.NONE;
+
+        return provider.UsedLLMProvider.GetConfidence(settingsManager).Level;
+    }
+
+    public static ConfidenceLevel GetConfidenceLevel(this IProvider provider, SettingsManager settingsManager)
+    {
+        if (provider is NoProvider)
+            return ConfidenceLevel.NONE;
+
+        return provider.Provider.GetConfidence(settingsManager).Level;
+    }
+
+    public static bool AllowsDataSourceAccess(this Provider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel requiredConfidenceLevel)
+    {
+        return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager)
+            && provider.GetConfidenceLevel(settingsManager).AllowsDataSourceConfidenceLevel(requiredConfidenceLevel);
+    }
+
+    public static bool AllowsDataSourceAccess(this IProvider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel requiredConfidenceLevel)
+    {
+        return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager)
+            && provider.GetConfidenceLevel(settingsManager).AllowsDataSourceConfidenceLevel(requiredConfidenceLevel);
+    }
+
+    public static bool AllowsDataSourceSecurity(this Provider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager)
+        => provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity);
+
+    public static bool AllowsDataSourceSecurity(this IProvider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager)
+        => provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity);
+
+    public static bool AllowsDataSourceSecurity(this bool usingTrustedProvider, DataSourceSecurity dataSourceSecurity) => dataSourceSecurity switch
+    {
+        DataSourceSecurity.ALLOW_ANY => true,
+        DataSourceSecurity.SELF_HOSTED => usingTrustedProvider,
+        _ => false,
+    };
+
+    public static bool AllowsDataSourceConfidenceLevel(this ConfidenceLevel providerConfidenceLevel, ConfidenceLevel requiredConfidenceLevel)
+    {
+        if (requiredConfidenceLevel is ConfidenceLevel.NONE)
+            return true;
+
+        return providerConfidenceLevel >= requiredConfidenceLevel;
+    }
+
+    public static ConfidenceLevel GetRequiredConfidenceLevel(this IEnumerable<IDataSource> dataSources)
+    {
+        var requiredConfidenceLevel = ConfidenceLevel.NONE;
+        foreach (var dataSource in dataSources.OfType<IInternalDataSource>())
+            if (dataSource.ConfidenceLevel > requiredConfidenceLevel)
+                requiredConfidenceLevel = dataSource.ConfidenceLevel;
+
+        return requiredConfidenceLevel;
+    }
+
+    public static DataSourceSecurity GetRequiredSecurityPolicy(this IEnumerable<IDataSource> dataSources)
+    {
+        var requiredSecurityPolicy = DataSourceSecurity.ALLOW_ANY;
+        foreach (var dataSource in dataSources.OfType<IExternalDataSource>())
+        {
+            if (dataSource.SecurityPolicy is DataSourceSecurity.NOT_SPECIFIED)
+                return DataSourceSecurity.NOT_SPECIFIED;
+
+            if (dataSource.SecurityPolicy is DataSourceSecurity.SELF_HOSTED)
+                requiredSecurityPolicy = DataSourceSecurity.SELF_HOSTED;
+        }
+
+        return requiredSecurityPolicy;
+    }
+
     public static bool IsTrustedByConfiguration(this Provider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager);
 
     public static bool IsTrustedByConfiguration(this EmbeddingProvider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager);

app/MindWork AI Studio/Settings/EmbeddingProvider.cs

@@ -22,23 +22,21 @@ public sealed record EmbeddingProvider(
     Guid EnterpriseConfigurationPluginId = default,
     string Hostname = "http://localhost:1234",
     Host Host = Host.NONE,
+    string TokenizerPath = "",
+    int EmbeddingBatchSize = 0,
+    int TokenLimit = 0,
     bool AllowUserProvidedAPIKey = false,
     string CustomIconDataUrl = "",
     HFInferenceProvider HFInferenceProvider = HFInferenceProvider.NONE) : ConfigurationBaseObject, ISecretId, IUserProvidedAPIKey
 {
+    public const int DEFAULT_TOKEN_LIMIT = 8192;
+    public const int DEFAULT_EMBEDDING_BATCH_SIZE = 1;
+
     private static readonly ILogger<EmbeddingProvider> LOGGER = Program.LOGGER_FACTORY.CreateLogger<EmbeddingProvider>();
 
     public static readonly EmbeddingProvider NONE = new();
 
-    public EmbeddingProvider() : this(
-        0,
-        Guid.Empty.ToString(),
-        string.Empty,
-        LLMProviders.NONE,
-        default,
-        false,
-        false,
-        Guid.Empty)
+    public EmbeddingProvider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty)
     {
     }
 
@@ -54,6 +52,12 @@ public sealed record EmbeddingProvider(
     [JsonIgnore]
     public string SecretName => this.Name;
 
+    [JsonIgnore]
+    public int EffectiveTokenLimit => this.TokenLimit > 0 ? this.TokenLimit : DEFAULT_TOKEN_LIMIT;
+
+    [JsonIgnore]
+    public int EffectiveEmbeddingBatchSize => this.EmbeddingBatchSize > 0 ? this.EmbeddingBatchSize : DEFAULT_EMBEDDING_BATCH_SIZE;
+
     #endregion
 
     public static bool TryParseEmbeddingProviderTable(int idx, LuaTable table, Guid configPluginId, string pluginPath, out ConfigurationBaseObject provider)
@@ -101,6 +105,27 @@ public sealed record EmbeddingProvider(
             return false;
         }
 
+        var tokenizerPath = string.Empty;
+        if (table.TryGetValue("TokenizerPath", out var tokenizerPathValue) && !tokenizerPathValue.TryRead<string>(out tokenizerPath))
+        {
+            LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid tokenizer path. (Plugin ID: {configPluginId})");
+            tokenizerPath = string.Empty;
+        }
+
+        var tokenLimit = DEFAULT_TOKEN_LIMIT;
+        if (table.TryGetValue("TokenLimit", out var tokenLimitValue) && (!tokenLimitValue.TryRead<int>(out tokenLimit) || tokenLimit < 1))
+        {
+            LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid token limit. Falling back to {DEFAULT_TOKEN_LIMIT}. (Plugin ID: {configPluginId})");
+            tokenLimit = DEFAULT_TOKEN_LIMIT;
+        }
+
+        var embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
+        if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead<int>(out embeddingBatchSize) || embeddingBatchSize < 1))
+        {
+            LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid embedding batch size. Falling back to {DEFAULT_EMBEDDING_BATCH_SIZE}. (Plugin ID: {configPluginId})");
+            embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
+        }
+
         var allowUserProvidedApiKey = false;
         if (table.TryGetValue("AllowUserProvidedAPIKey", out var allowUserProvidedApiKeyValue) && allowUserProvidedApiKeyValue.TryRead<bool>(out var allowUserProvidedApiKeyBool))
             allowUserProvidedApiKey = allowUserProvidedApiKeyBool;
@@ -136,6 +161,9 @@ public sealed record EmbeddingProvider(
             EnterpriseConfigurationPluginId = configPluginId,
             Hostname = hostname,
             Host = host,
+            TokenizerPath = tokenizerPath,
+            EmbeddingBatchSize = embeddingBatchSize,
+            TokenLimit = tokenLimit,
             AllowUserProvidedAPIKey = allowUserProvidedApiKey,
             CustomIconDataUrl = customIconDataUrl,
             HFInferenceProvider = hfInferenceProvider,
@@ -227,6 +255,10 @@ public sealed record EmbeddingProvider(
                     ["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
 
+                    ["TokenizerPath"] = "{{this.TokenizerPath}}",
+                    ["TokenLimit"] = {{this.EffectiveTokenLimit}},
+                    ["EmbeddingBatchSize"] = {{this.EffectiveEmbeddingBatchSize}},
+
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",
                     {{hfInferenceProviderLine}}

app/MindWork AI Studio/Settings/IDataSource.cs

@@ -21,11 +21,6 @@ public interface IDataSource : IConfigurationObject
     /// </summary>
     public DataSourceType Type { get; init; }
 
-    /// <summary>
-    /// Which data security policy is applied to this data source?
-    /// </summary>
-    public DataSourceSecurity SecurityPolicy { get; init; }
-    
     /// <summary>
     /// The maximum number of matches to return when retrieving data from the ERI server.
     /// </summary>

app/MindWork AI Studio/Settings/IExternalDataSource.cs

@@ -1,9 +1,16 @@
 using System.Text.Json.Serialization;
 
+using AIStudio.Settings.DataModel;
+
 namespace AIStudio.Settings;
 
 public interface IExternalDataSource : IDataSource, ISecretId
 {
+    /// <summary>
+    /// Which data security policy is applied to this external data source?
+    /// </summary>
+    public DataSourceSecurity SecurityPolicy { get; init; }
+
     #region Implementation of ISecretId
 
     [JsonIgnore]

app/MindWork AI Studio/Settings/IInternalDataSource.cs

@@ -1,9 +1,27 @@
+using AIStudio.Provider;
+
 namespace AIStudio.Settings;
 
 public interface IInternalDataSource : IDataSource
 {
+    /// <summary>
+    /// Which provider confidence level is required by this internal data source?
+    /// </summary>
+    public ConfidenceLevel ConfidenceLevel { get; init; }
+
     /// <summary>
     /// The unique identifier of the embedding method used by this internal data source.
     /// </summary>
     public string EmbeddingId { get; init; }
+
+    /// <summary>
+    /// Optional maximum number of tokens per embedding chunk for this data source.
+    /// A value of 0 means the embedding provider's setting is used.
+    /// </summary>
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <summary>
+    /// Optional number of tokens to overlap between consecutive chunks.
+    /// </summary>
+    public int ChunkOverlapTokenLength { get; init; }
 }

app/MindWork AI Studio/Settings/Provider.cs

@@ -36,6 +36,7 @@ public sealed record Provider(
     Host Host = Host.NONE,
     HFInferenceProvider HFInferenceProvider = HFInferenceProvider.NONE,
     string AdditionalJsonApiParameters = "",
+    string TokenizerPath = "",
     ProviderCapabilityOverrides? CapabilityOverrides = null,
     bool AllowUserProvidedAPIKey = false,
     string CustomIconDataUrl = "") : ConfigurationBaseObject, ISecretId, IUserProvidedAPIKey
@@ -44,15 +45,7 @@ public sealed record Provider(
     
     public static readonly Provider NONE = new();
 
-    public Provider() : this(
-        0,
-        Guid.Empty.ToString(),
-        string.Empty,
-        LLMProviders.NONE,
-        default, 
-        false,
-        false,
-        Guid.Empty)
+    public Provider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty)
     {
     }
     
@@ -157,6 +150,12 @@ public sealed record Provider(
             additionalJsonApiParameters = string.Empty;
         }
 
+        var tokenizerPath = string.Empty;
+        if (table.TryGetValue("TokenizerPath", out var tokenizerPathValue) && !tokenizerPathValue.TryRead<string>(out tokenizerPath))
+        {
+            LOGGER.LogWarning($"The configured provider {idx} does not contain a valid tokenizer path. (Plugin ID: {configPluginId})");
+            tokenizerPath = string.Empty;
+        }
         var capabilityOverrides = ProviderCapabilityOverrides.TryParseFromLuaTable(idx, table, configPluginId, LOGGER);
 
         var allowUserProvidedApiKey = false;
@@ -186,6 +185,7 @@ public sealed record Provider(
             Host = host,
             HFInferenceProvider = hfInferenceProvider,
             AdditionalJsonApiParameters = additionalJsonApiParameters,
+            TokenizerPath = tokenizerPath,
             CapabilityOverrides = capabilityOverrides,
             AllowUserProvidedAPIKey = allowUserProvidedApiKey,
             CustomIconDataUrl = customIconDataUrl,
@@ -278,6 +278,8 @@ public sealed record Provider(
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
                     ["InstanceName"] = "{{LuaTools.EscapeLuaString(this.InstanceName)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
+                    
+                    ["TokenizerPath"] = "{{this.TokenizerPath}}",
 
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",