added metadata for bm25 and also confidence levels

app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs

@@ -294,6 +294,17 @@ public static class ConfigurationSelectDataFactory
             }
         }
     }
+
+    public static IEnumerable<ConfigurationSelectData<ConfidenceLevel>> GetDataSourceComplianceLevelsData()
+    {
+        foreach (var level in Enum.GetValues<ConfidenceLevel>())
+        {
+            if (level is ConfidenceLevel.NONE)
+                continue;
+
+            yield return new(level.GetName(), level);
+        }
+    }
     
     public static IEnumerable<ConfigurationSelectData<Themes>> GetThemesData()
     {

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -2,6 +2,7 @@
 
 using AIStudio.Assistants.ERI;
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.ERIClient;
 using AIStudio.Tools.ERIClient.DataModel;
 using AIStudio.Tools.PluginSystem;
@@ -57,6 +58,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
 
+    /// <inheritdoc />
+    public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
+
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
 
@@ -206,6 +210,16 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             return false;
         }
 
+        var complianceLevel = ConfidenceLevel.UNKNOWN;
+        if (table.TryGetValue("ComplianceLevel", out var complianceLevelValue) &&
+            (!complianceLevelValue.TryRead<string>(out var complianceLevelText) ||
+             !Enum.TryParse(complianceLevelText, true, out complianceLevel) ||
+             complianceLevel is ConfidenceLevel.NONE))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} contains an invalid compliance level. Falling back to UNKNOWN. (Plugin ID: {configPluginId})");
+            complianceLevel = ConfidenceLevel.UNKNOWN;
+        }
+
         if (!table.TryGetValue("SelectedRetrievalId", out var selectedRetrievalIdValue) || !selectedRetrievalIdValue.TryRead<string>(out var selectedRetrievalId) || string.IsNullOrWhiteSpace(selectedRetrievalId))
         {
             LOGGER.LogWarning($"The configured data source {idx} must specify a selected retrieval ID. (Plugin ID: {configPluginId})");
@@ -263,6 +277,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             Username = username,
             UsernamePasswordMode = usernamePasswordMode,
             SecurityPolicy = securityPolicy,
+            ComplianceLevel = complianceLevel,
             Version = ERIVersion.V1,
             SelectedRetrievalId = selectedRetrievalId,
             MaxMatches = (ushort)maxMatches,
@@ -323,6 +338,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                     {{usernameLine}}
                     {{secretLine}}
                     ["SecurityPolicy"] = "{{this.SecurityPolicy}}",
+                    ["ComplianceLevel"] = "{{this.ComplianceLevel}}",
                     ["SelectedRetrievalId"] = "{{LuaTools.EscapeLuaString(this.SelectedRetrievalId)}}",
                     ["MaxMatches"] = {{this.MaxMatches}},
                 }

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -1,4 +1,5 @@
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.RAG;
 
 namespace AIStudio.Settings.DataModel;
@@ -42,6 +43,9 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
 
+    /// <inheritdoc />
+    public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
+
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
 

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -1,4 +1,5 @@
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.RAG;
 
 namespace AIStudio.Settings.DataModel;
@@ -42,6 +43,9 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
 
+    /// <inheritdoc />
+    public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
+
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
 

app/MindWork AI Studio/Settings/IDataSource.cs

@@ -1,6 +1,7 @@
 using System.Text.Json.Serialization;
 
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.RAG;
@@ -25,6 +26,11 @@ public interface IDataSource : IConfigurationObject
     /// Which data security policy is applied to this data source?
     /// </summary>
     public DataSourceSecurity SecurityPolicy { get; init; }
+
+    /// <summary>
+    /// Which compliance level is assigned to this data source?
+    /// </summary>
+    public ConfidenceLevel ComplianceLevel { get; init; }
     
     /// <summary>
     /// The maximum number of matches to return when retrieving data from the ERI server.