Added an option to enforce a minimum confidence level throughout the entire app

This commit is contained in:
Thorsten Sommer 2024-09-13 23:27:51 +02:00
parent 838d3e2d4d
commit 9bbc289d93
Signed by: tsommer
GPG Key ID: 371BBA77A02C0108
9 changed files with 83 additions and 6 deletions

View File

@ -1 +1 @@
<ConfigurationSelect OptionDescription="Preselected provider" Disabled="@this.Disabled" OptionHelp="@this.HelpText()" Data="@this.Data" SelectedValue="@this.SelectedValue" SelectionUpdate="@this.SelectionUpdate"/>
<ConfigurationSelect OptionDescription="Preselected provider" Disabled="@this.Disabled" OptionHelp="@this.HelpText()" Data="@this.FilteredData()" SelectedValue="@this.SelectedValue" SelectionUpdate="@this.SelectionUpdate"/>

View File

@ -1,3 +1,4 @@
using AIStudio.Provider;
using AIStudio.Settings;
using Microsoft.AspNetCore.Components;
@ -43,6 +44,25 @@ public partial class ConfigurationProviderSelection : ComponentBase, IMessageBus
#endregion
private IEnumerable<ConfigurationSelectData<string>> FilteredData()
{
if (this.SettingsManager.ConfigurationData.LLMProviders is { EnforceGlobalMinimumConfidence: true, GlobalMinimumConfidence: not ConfidenceLevel.NONE and not ConfidenceLevel.UNKNOWN })
{
var minimumLevel = this.SettingsManager.ConfigurationData.LLMProviders.GlobalMinimumConfidence;
foreach (var providerId in this.Data)
{
var provider = this.SettingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == providerId.Value);
if (provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level >= minimumLevel)
yield return providerId;
}
}
else
{
foreach (var provider in this.Data)
yield return provider;
}
}
#region Implementation of IMessageBusReceiver
public async Task ProcessMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data)

View File

@ -1,7 +1,7 @@
@using AIStudio.Settings
<MudSelect T="Provider" Value="@this.ProviderSettings" ValueChanged="@this.SelectionChanged" Validation="@this.ValidateProvider" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Apps" Margin="Margin.Dense" Label="Provider" Class="mb-3 rounded-lg" Variant="Variant.Outlined">
@foreach (var provider in this.SettingsManager.ConfigurationData.Providers)
@foreach (var provider in this.GetAvailableProviders())
{
<MudSelectItem Value="@provider"/>
}

View File

@ -1,3 +1,4 @@
using AIStudio.Provider;
using AIStudio.Settings;
using Microsoft.AspNetCore.Components;
@ -16,11 +17,27 @@ public partial class ProviderSelection : ComponentBase
public Func<Settings.Provider, string?> ValidateProvider { get; set; } = _ => null;
[Inject]
protected SettingsManager SettingsManager { get; set; } = null!;
private SettingsManager SettingsManager { get; init; } = null!;
private async Task SelectionChanged(Settings.Provider provider)
{
this.ProviderSettings = provider;
await this.ProviderSettingsChanged.InvokeAsync(provider);
}
private IEnumerable<Settings.Provider> GetAvailableProviders()
{
if (this.SettingsManager.ConfigurationData.LLMProviders is { EnforceGlobalMinimumConfidence: true, GlobalMinimumConfidence: not ConfidenceLevel.NONE and not ConfidenceLevel.UNKNOWN })
{
var minimumLevel = this.SettingsManager.ConfigurationData.LLMProviders.GlobalMinimumConfidence;
foreach (var provider in this.SettingsManager.ConfigurationData.Providers)
if (provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level >= minimumLevel)
yield return provider;
}
else
{
foreach (var provider in this.SettingsManager.ConfigurationData.Providers)
yield return provider;
}
}
}

View File

@ -80,6 +80,12 @@
Either you choose a common schema, or you configure the trust levels for each LLM provider yourself.
</MudJustifiedText>
<ConfigurationOption OptionDescription="Do you want to enforce an app-wide minimum confidence level?" LabelOn="Yes, enforce a minimum confidence level" LabelOff="No, do not enforce a minimum confidence level" State="@(() => this.SettingsManager.ConfigurationData.LLMProviders.EnforceGlobalMinimumConfidence)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.LLMProviders.EnforceGlobalMinimumConfidence = updatedState)" OptionHelp="When enabled, you can enforce a minimum confidence level for all LLM providers. This way, you can ensure that only trustworthy providers are used."/>
@if(this.SettingsManager.ConfigurationData.LLMProviders.EnforceGlobalMinimumConfidence)
{
<ConfigurationSelect OptionDescription="Select a minimum confidence level" SelectedValue="@(() => this.SettingsManager.ConfigurationData.LLMProviders.GlobalMinimumConfidence)" Data="@ConfigurationSelectDataFactory.GetConfidenceLevelsData()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.LLMProviders.GlobalMinimumConfidence = selectedValue)" OptionHelp="Choose the minimum confidence level that all LLM providers must meet. This way, you can ensure that only trustworthy providers are used. You cannot use any provider that falls below this level."/>
}
<ConfigurationOption OptionDescription="Show provider's confidence level?" LabelOn="Yes, show me the confidence level" LabelOff="No, please hide the confidence level" State="@(() => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence = updatedState)" OptionHelp="When enabled, we show you the confidence level for the selected provider in the app. This helps you assess where you are sending your data at any time. Example: are you currently working with sensitive data? Then choose a particularly trustworthy provider, etc."/>
@if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence)
{

View File

@ -4,6 +4,7 @@ using AIStudio.Assistants.IconFinder;
using AIStudio.Assistants.RewriteImprove;
using AIStudio.Assistants.TextSummarizer;
using AIStudio.Assistants.EMail;
using AIStudio.Provider;
using AIStudio.Settings.DataModel;
using WritingStylesRewrite = AIStudio.Assistants.RewriteImprove.WritingStyles;
@ -142,4 +143,24 @@ public static class ConfigurationSelectDataFactory
foreach (var scheme in Enum.GetValues<ConfidenceSchemes>())
yield return new(scheme.GetListDescription(), scheme);
}
public static IEnumerable<ConfigurationSelectData<ConfidenceLevel>> GetConfidenceLevelsData()
{
foreach (var level in Enum.GetValues<ConfidenceLevel>())
{
switch (level)
{
case ConfidenceLevel.UNKNOWN:
continue;
case ConfidenceLevel.NONE:
yield return new("No minimum confidence level chosen", level);
break;
default:
yield return new(level.GetName(), level);
break;
}
}
}
}

View File

@ -4,6 +4,16 @@ namespace AIStudio.Settings.DataModel;
public sealed class DataLLMProviders
{
/// <summary>
/// Should we enforce a global minimum confidence level?
/// </summary>
public bool EnforceGlobalMinimumConfidence { get; set; }
/// <summary>
/// The global minimum confidence level to enforce.
/// </summary>
public ConfidenceLevel GlobalMinimumConfidence { get; set; } = ConfidenceLevel.NONE;
/// <summary>
/// Should we show the provider confidence level?
/// </summary>

View File

@ -114,7 +114,9 @@ public sealed class SettingsManager(ILogger<SettingsManager> logger)
public Provider GetPreselectedProvider(Tools.Components component)
{
if(this.ConfigurationData.Providers.Count == 1)
var minimumLevel = this.ConfigurationData.LLMProviders.EnforceGlobalMinimumConfidence ? this.ConfigurationData.LLMProviders.GlobalMinimumConfidence : ConfidenceLevel.NONE;
if (this.ConfigurationData.Providers.Count == 1 && this.ConfigurationData.Providers[0].UsedLLMProvider.GetConfidence(this).Level >= minimumLevel)
return this.ConfigurationData.Providers[0];
var preselection = component switch
@ -135,10 +137,10 @@ public sealed class SettingsManager(ILogger<SettingsManager> logger)
_ => default,
};
if (preselection != default)
if(preselection != default && preselection.UsedLLMProvider.GetConfidence(this).Level >= minimumLevel)
return preselection;
return this.ConfigurationData.Providers.FirstOrDefault(x => x.Id == this.ConfigurationData.App.PreselectedProvider);
return this.ConfigurationData.Providers.FirstOrDefault(x => x.Id == this.ConfigurationData.App.PreselectedProvider && x.UsedLLMProvider.GetConfidence(this).Level >= minimumLevel);
}
public Profile GetPreselectedProfile(Tools.Components component)

View File

@ -1,3 +1,4 @@
# v0.9.11, build 186
- Added a tooltip to the confidence card button.
- Added an option to enforce a minimum confidence level throughout the entire app.
- Renamed the `Providers` enum to `LLMProviders` for better clarity. Renamed also all dependent variables and methods.