Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Settings/DataModel/DataApp.cs

@@ -179,6 +179,11 @@ public sealed class DataApp(Expression<Func<Data, DataApp>>? configSelection = n
     /// </summary>
     public bool ShowAdminSettings { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ShowAdminSettings, false);
 
+    /// <summary>
+    /// Settings for indexing local data sources.
+    /// </summary>
+    public DataDataSourceIndexing DataSourceIndexing { get; init; } = new();
+
     /// <summary>
     /// List of assistants that should be hidden from the UI.
     /// </summary>

app/MindWork AI Studio/Settings/DataModel/DataDataSourceIndexing.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Settings.DataModel;
+
+public sealed class DataDataSourceIndexing
+{
+    /// <summary>
+    /// Whether local data source embeddings should refresh automatically when files change.
+    /// </summary>
+    public bool AutomaticRefresh { get; set; } = true;
+}

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -7,6 +7,7 @@ using AIStudio.Tools.ERIClient.DataModel;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.RAG;
 using AIStudio.Tools.Services;
+using AIStudio.Tools.Validation;
 
 using SharedTools;
 
@@ -164,9 +165,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             return false;
         }
 
-        if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name))
+        if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || !DataSourceValidation.IsNameValid(name))
         {
-            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name. (Plugin ID: {configPluginId})");
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most {DataSourceValidation.MAX_NAME_LENGTH} characters without control characters. (Plugin ID: {configPluginId})");
             return false;
         }
 
@@ -390,4 +391,4 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
         var cleanedHostname = hostname.Trim();
         return cleanedHostname.EndsWith('/') ? cleanedHostname[..^1] : cleanedHostname;
     }
-}
+}

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -1,5 +1,7 @@
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Settings.DataModel;
 
@@ -32,9 +34,15 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     
     /// <inheritdoc />
     public string EmbeddingId { get; init; } = Guid.Empty.ToString();
+
+    /// <inheritdoc />
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <inheritdoc />
+    public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     
     /// <inheritdoc />
-    public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+    public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
 
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
@@ -46,11 +54,8 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     public ushort MaxMatches { get; init; } = 10;
     
     /// <inheritdoc />
-    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default)
-    {
-        IReadOnlyList<IRetrievalContext> retrievalContext = new List<IRetrievalContext>();
-        return Task.FromResult(retrievalContext);
-    }
+    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
     /// <summary>
     /// The path to the directory.

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -1,5 +1,7 @@
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Settings.DataModel;
 
@@ -32,9 +34,15 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     
     /// <inheritdoc />
     public string EmbeddingId { get; init; } = Guid.Empty.ToString();
+
+    /// <inheritdoc />
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <inheritdoc />
+    public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     
     /// <inheritdoc />
-    public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+    public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
 
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
@@ -46,11 +54,8 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     public ushort MaxMatches { get; init; } = 10;
     
     /// <inheritdoc />
-    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default)
-    {
-        IReadOnlyList<IRetrievalContext> retrievalContext = new List<IRetrievalContext>();
-        return Task.FromResult(retrievalContext);
-    }
+    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
     /// <summary>
     /// The path to the file.

app/MindWork AI Studio/Settings/DataModel/PreviewVisibilityExtensions.cs

@@ -13,6 +13,7 @@ public static class PreviewVisibilityExtensions
         {
             features.Add(PreviewFeatures.PRE_DOCUMENT_ANALYSIS_2025);
             features.Add(PreviewFeatures.PRE_META_ASSISTANT_V1);
+            features.Add(PreviewFeatures.PRE_RAG_2024);
         }
         
         if (visibility >= PreviewVisibility.ALPHA)
@@ -21,7 +22,6 @@ public static class PreviewVisibilityExtensions
         
         if (visibility >= PreviewVisibility.PROTOTYPE)
         {
-            features.Add(PreviewFeatures.PRE_RAG_2024);
             features.Add(PreviewFeatures.PRE_VISUAL_BRIEFING_ASSISTANT_2026);
         }