diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor b/app/MindWork AI Studio/Assistants/AssistantBase.razor index 398f89ef..dd6d5b1d 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor @@ -97,7 +97,12 @@ Reset - + + @if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence) + { + + } + @if (this.AllowProfiles && this.ShowProfileSelection) { diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index b87f788a..837d19f2 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -113,6 +113,8 @@ public abstract partial class AssistantBase : ComponentBase } #endregion + + private string SubmitButtonStyle => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence ? this.providerSettings.UsedProvider.GetConfidence(this.SettingsManager).StyleBorder() : string.Empty; protected string? ValidatingProvider(AIStudio.Settings.Provider provider) { diff --git a/app/MindWork AI Studio/Components/ConfidenceInfo.razor b/app/MindWork AI Studio/Components/ConfidenceInfo.razor new file mode 100644 index 00000000..ebb76e8c --- /dev/null +++ b/app/MindWork AI Studio/Components/ConfidenceInfo.razor @@ -0,0 +1,56 @@ +@using AIStudio.Provider +
+@if (this.Mode is ConfidenceInfoMode.ICON) +{ + +} +else +{ + + Confidence + +} + + + + + + Confidence Card + + + + Description + + + @if (this.currentConfidence.Sources.Count > 0) + { + Sources + + @foreach (var sourceTuple in this.GetConfidenceSources()) + { + + } + + } + + @if (!string.IsNullOrWhiteSpace(this.currentConfidence.Region)) + { + Region + + @this.currentConfidence.Region + + } + + Confidence Level + + @this.currentConfidence.Level.GetName() + + + + + Close + + + + +
\ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs b/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs new file mode 100644 index 00000000..d8b72c95 --- /dev/null +++ b/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs @@ -0,0 +1,57 @@ +using AIStudio.Provider; +using AIStudio.Settings; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +public partial class ConfidenceInfo : ComponentBase +{ + [Parameter] + public ConfidenceInfoMode Mode { get; set; } = ConfidenceInfoMode.BUTTON; + + [Parameter] + public Providers Provider { get; set; } + + [Inject] + private SettingsManager SettingsManager { get; init; } = null!; + + private Confidence currentConfidence; + private bool showConfidence; + + public ConfidenceInfo() + { + this.currentConfidence = Providers.NONE.GetConfidence(this.SettingsManager); + } + + #region Overrides of ComponentBase + + protected override async Task OnParametersSetAsync() + { + this.currentConfidence = this.Provider.GetConfidence(this.SettingsManager); + await base.OnParametersSetAsync(); + } + + #endregion + + private void ToggleConfidence() + { + this.showConfidence = !this.showConfidence; + } + + private void HideConfidence() + { + this.showConfidence = false; + } + + private IEnumerable<(string Index, string Source)> GetConfidenceSources() + { + var index = 0; + foreach (var source in this.currentConfidence.Sources) + yield return ($"Source {++index}", source); + } + + private string GetCurrentConfidenceColor() => $"color: {this.currentConfidence.Level.GetColor()};"; + + private string GetPopoverStyle() => $"border-color: {this.currentConfidence.Level.GetColor()}; max-width: calc(35vw);"; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ConfidenceInfoMode.cs b/app/MindWork AI Studio/Components/ConfidenceInfoMode.cs new file mode 100644 index 00000000..d7e63da6 --- /dev/null +++ b/app/MindWork AI Studio/Components/ConfidenceInfoMode.cs @@ -0,0 +1,7 @@ +namespace AIStudio.Components; + +public enum ConfidenceInfoMode +{ + BUTTON, + ICON, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Pages/Chat.razor b/app/MindWork AI Studio/Pages/Chat.razor index 3d5d3b73..806f0607 100644 --- a/app/MindWork AI Studio/Pages/Chat.razor +++ b/app/MindWork AI Studio/Pages/Chat.razor @@ -28,7 +28,7 @@ - + @if (this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is not WorkspaceStorageBehavior.DISABLE_WORKSPACES) @@ -70,6 +70,11 @@ } + @if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence) + { + + } + diff --git a/app/MindWork AI Studio/Pages/Chat.razor.cs b/app/MindWork AI Studio/Pages/Chat.razor.cs index 96dcc98e..8e06c03d 100644 --- a/app/MindWork AI Studio/Pages/Chat.razor.cs +++ b/app/MindWork AI Studio/Pages/Chat.razor.cs @@ -120,6 +120,10 @@ public partial class Chat : MSGComponentBase, IAsyncDisposable private string TooltipAddChatToWorkspace => $"Start new chat in workspace \"{this.currentWorkspaceName}\""; + private string UserInputStyle => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence ? this.providerSettings.UsedProvider.GetConfidence(this.SettingsManager).SetColorStyle() : string.Empty; + + private string UserInputClass => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence ? "confidence-border" : string.Empty; + private void ProfileWasChanged(Profile profile) { this.currentProfile = profile; diff --git a/app/MindWork AI Studio/Pages/Settings.razor b/app/MindWork AI Studio/Pages/Settings.razor index 5ba3b60f..2cfed9d3 100644 --- a/app/MindWork AI Studio/Pages/Settings.razor +++ b/app/MindWork AI Studio/Pages/Settings.razor @@ -17,7 +17,7 @@ task. As an LLM provider, you can also choose local providers. However, to use this app, you must configure at least one provider. - + @@ -72,6 +72,56 @@ Add Provider + + LLM Provider Confidence + + Do you want to always be able to recognize how trustworthy your LLM providers are? This way, + you keep control over which provider you send your data to. You have two options for this: + Either you choose a common schema, or you configure the trust levels for each LLM provider yourself. + + + + @if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence) + { + + @if (this.SettingsManager.ConfigurationData.LLMProviders.ConfidenceScheme is ConfidenceSchemes.CUSTOM) + { + + + + + + + + LLM Provider + Description + Confidence Level + + + + @context.ToName() + + + + + + + @foreach (var confidenceLevel in Enum.GetValues().OrderBy(n => n)) + { + if(confidenceLevel is ConfidenceLevel.NONE or ConfidenceLevel.UNKNOWN) + continue; + + + @confidenceLevel.GetName() + + } + + + + + } + } + @@ -88,7 +138,7 @@ code. In these profiles, you can record how much experience you have or which methods you like or dislike using. Later, you can choose when and where you want to use each profile. - + diff --git a/app/MindWork AI Studio/Pages/Settings.razor.cs b/app/MindWork AI Studio/Pages/Settings.razor.cs index 5c09f985..90f3bdb9 100644 --- a/app/MindWork AI Studio/Pages/Settings.razor.cs +++ b/app/MindWork AI Studio/Pages/Settings.razor.cs @@ -160,6 +160,28 @@ public partial class Settings : ComponentBase, IMessageBusReceiver, IDisposable this.availableProviders.Add(new (provider.InstanceName, provider.Id)); } + private string GetCurrentConfidenceLevelName(Providers provider) + { + if (this.SettingsManager.ConfigurationData.LLMProviders.CustomConfidenceScheme.TryGetValue(provider, out var level)) + return level.GetName(); + + return "Not yet configured"; + } + + private string SetCurrentConfidenceLevelColorStyle(Providers provider) + { + if (this.SettingsManager.ConfigurationData.LLMProviders.CustomConfidenceScheme.TryGetValue(provider, out var level)) + return $"background-color: {level.GetColor()};"; + + return $"background-color: {ConfidenceLevel.UNKNOWN.GetColor()};"; + } + + private async Task ChangeCustomConfidenceLevel(Providers provider, ConfidenceLevel level) + { + this.SettingsManager.ConfigurationData.LLMProviders.CustomConfidenceScheme[provider] = level; + await this.SettingsManager.StoreSettings(); + } + #endregion #region Profile related diff --git a/app/MindWork AI Studio/Provider/Confidence.cs b/app/MindWork AI Studio/Provider/Confidence.cs new file mode 100644 index 00000000..3cf08acd --- /dev/null +++ b/app/MindWork AI Studio/Provider/Confidence.cs @@ -0,0 +1,65 @@ +namespace AIStudio.Provider; + +public sealed record Confidence +{ + public ConfidenceLevel Level { get; private init; } = ConfidenceLevel.UNKNOWN; + + public string Region { get; private init; } = string.Empty; + + public string Description { get; private init; } = string.Empty; + + public List Sources { get; private init; } = []; + + private Confidence() + { + } + + public Confidence WithSources(params string[] sources) => this with { Sources = sources.ToList() }; + + public Confidence WithRegion(string region) => this with { Region = region }; + + public Confidence WithLevel(ConfidenceLevel level) => this with { Level = level }; + + public string StyleBorder() => $"border: 2px solid {this.Level.GetColor()}; border-radius: 6px;"; + + public string SetColorStyle() => $"--confidence-color: {this.Level.GetColor()};"; + + public static readonly Confidence NONE = new() + { + Level = ConfidenceLevel.NONE, + Description = + """ + No provider selected. Please select a provider to get see its confidence level. + """, + }; + + public static readonly Confidence USA_NOT_TRUSTED = new() + { + Level = ConfidenceLevel.UNTRUSTED, + Description = "The provider operates its service from the USA and is subject to **U.S. jurisdiction**. In case of suspicion, authorities in the USA can access your data. The provider's terms of service state that **all your data can be used by the provider at will.**", + }; + + public static readonly Confidence UNKNOWN = new() + { + Level = ConfidenceLevel.UNKNOWN, + Description = "The trust level of this provider **has not yet** been thoroughly **investigated and evaluated**. We do not know if your data is safe.", + }; + + public static readonly Confidence USA_NO_TRAINING = new() + { + Level = ConfidenceLevel.MODERATE, + Description = "The provider operates its service from the USA and is subject to **US jurisdiction**. In case of suspicion, authorities in the USA can access your data. However, **your data is not used for training** purposes.", + }; + + public static readonly Confidence GDPR_NO_TRAINING = new() + { + Level = ConfidenceLevel.MEDIUM, + Description = "The provider is located in the EU and is subject to the **GDPR** (General Data Protection Regulation). Additionally, the provider states that **your data is not used for training**.", + }; + + public static readonly Confidence SELF_HOSTED = new() + { + Level = ConfidenceLevel.HIGH, + Description = "You or your organization operate the LLM locally or within your trusted network. In terms of data processing and security, this is the best possible way.", + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ConfidenceLevel.cs b/app/MindWork AI Studio/Provider/ConfidenceLevel.cs new file mode 100644 index 00000000..9b1f4175 --- /dev/null +++ b/app/MindWork AI Studio/Provider/ConfidenceLevel.cs @@ -0,0 +1,18 @@ +namespace AIStudio.Provider; + +public enum ConfidenceLevel +{ + NONE = 0, + UNTRUSTED = 1, + + UNKNOWN = 10, + + VERY_LOW = 90, + LOW = 100, + + MODERATE = 150, + + MEDIUM = 200, + + HIGH = 300, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs b/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs new file mode 100644 index 00000000..2f8d451b --- /dev/null +++ b/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs @@ -0,0 +1,34 @@ +namespace AIStudio.Provider; + +public static class ConfidenceLevelExtensions +{ + public static string GetName(this ConfidenceLevel level) => level switch + { + ConfidenceLevel.NONE => "No provider selected", + + ConfidenceLevel.UNTRUSTED => "Untrusted", + ConfidenceLevel.VERY_LOW => "Very Low", + ConfidenceLevel.LOW => "Low", + ConfidenceLevel.MODERATE => "Moderate", + ConfidenceLevel.MEDIUM => "Medium", + ConfidenceLevel.HIGH => "High", + + _ => "Unknown confidence level", + }; + + public static string GetColor(this ConfidenceLevel level) => level switch + { + ConfidenceLevel.NONE => "#cccccc", + + ConfidenceLevel.UNTRUSTED => "#ff0000", + ConfidenceLevel.VERY_LOW => "#ff6600", + ConfidenceLevel.LOW => "#ffcc00", + ConfidenceLevel.MODERATE => "#99cc00", + ConfidenceLevel.MEDIUM => "#86b300", + ConfidenceLevel.HIGH => "#009933", + + _ => "#cc6600", + }; + + public static string SetColorStyle(this ConfidenceLevel level) => $"--confidence-color: {level.GetColor()};"; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ProvidersExtensions.cs b/app/MindWork AI Studio/Provider/ProvidersExtensions.cs index 806c3bb8..2ddc7f8e 100644 --- a/app/MindWork AI Studio/Provider/ProvidersExtensions.cs +++ b/app/MindWork AI Studio/Provider/ProvidersExtensions.cs @@ -3,6 +3,7 @@ using AIStudio.Provider.Fireworks; using AIStudio.Provider.Mistral; using AIStudio.Provider.OpenAI; using AIStudio.Provider.SelfHosted; +using AIStudio.Settings; namespace AIStudio.Provider; @@ -27,6 +28,33 @@ public static class ProvidersExtensions _ => "Unknown", }; + + /// + /// Get a provider's confidence. + /// + /// The provider. + /// The settings manager. + /// The confidence of the provider. + public static Confidence GetConfidence(this Providers provider, SettingsManager settingsManager) => provider switch + { + Providers.NONE => Confidence.NONE, + + Providers.FIREWORKS => Confidence.USA_NOT_TRUSTED.WithRegion("America, U.S.").WithSources("https://fireworks.ai/terms-of-service").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + Providers.OPEN_AI => Confidence.USA_NO_TRAINING.WithRegion("America, U.S.").WithSources( + "https://openai.com/policies/terms-of-use/", + "https://help.openai.com/en/articles/5722486-how-your-data-is-used-to-improve-model-performance", + "https://openai.com/enterprise-privacy/" + ).WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + Providers.ANTHROPIC => Confidence.USA_NO_TRAINING.WithRegion("America, U.S.").WithSources("https://www.anthropic.com/legal/commercial-terms").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + Providers.MISTRAL => Confidence.GDPR_NO_TRAINING.WithRegion("Europe, France").WithSources("https://mistral.ai/terms/#terms-of-service-la-plateforme").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + Providers.SELF_HOSTED => Confidence.SELF_HOSTED.WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + _ => Confidence.UNKNOWN.WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + }; /// /// Creates a new provider instance based on the provider value. diff --git a/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs b/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs index c8dfd39f..c48ea6df 100644 --- a/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs +++ b/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs @@ -136,4 +136,10 @@ public static class ConfigurationSelectDataFactory foreach (var profile in profiles.GetAllProfiles()) yield return new(profile.Name, profile.Id); } + + public static IEnumerable> GetConfidenceSchemesData() + { + foreach (var scheme in Enum.GetValues()) + yield return new(scheme.GetListDescription(), scheme); + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/Data.cs b/app/MindWork AI Studio/Settings/DataModel/Data.cs index da8abc37..1fe0a921 100644 --- a/app/MindWork AI Studio/Settings/DataModel/Data.cs +++ b/app/MindWork AI Studio/Settings/DataModel/Data.cs @@ -16,6 +16,11 @@ public sealed class Data /// public List Providers { get; init; } = []; + /// + /// Settings concerning the LLM providers. + /// + public DataLLMProviders LLMProviders { get; init; } = new(); + /// /// List of configured profiles. /// diff --git a/app/MindWork AI Studio/Settings/DataModel/DataLLMProviders.cs b/app/MindWork AI Studio/Settings/DataModel/DataLLMProviders.cs new file mode 100644 index 00000000..73b7825f --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataLLMProviders.cs @@ -0,0 +1,21 @@ +using AIStudio.Provider; + +namespace AIStudio.Settings.DataModel; + +public sealed class DataLLMProviders +{ + /// + /// Should we show the provider confidence level? + /// + public bool ShowProviderConfidence { get; set; } = true; + + /// + /// Which confidence scheme to use. + /// + public ConfidenceSchemes ConfidenceScheme { get; set; } = ConfidenceSchemes.TRUST_USA_EUROPE; + + /// + /// Provide custom confidence levels for each LLM provider. + /// + public Dictionary CustomConfidenceScheme { get; set; } = new(); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/SettingsManager.cs b/app/MindWork AI Studio/Settings/SettingsManager.cs index e7459218..2a660b25 100644 --- a/app/MindWork AI Studio/Settings/SettingsManager.cs +++ b/app/MindWork AI Studio/Settings/SettingsManager.cs @@ -1,6 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; +using AIStudio.Provider; using AIStudio.Settings.DataModel; // ReSharper disable NotAccessedPositionalProperty.Local @@ -160,4 +161,57 @@ public sealed class SettingsManager(ILogger logger) preselection = this.ConfigurationData.Profiles.FirstOrDefault(x => x.Id == this.ConfigurationData.App.PreselectedProfile); return preselection != default ? preselection : Profile.NO_PROFILE; } + + public ConfidenceLevel GetConfiguredConfidenceLevel(Providers provider) + { + if(provider is Providers.NONE) + return ConfidenceLevel.NONE; + + switch (this.ConfigurationData.LLMProviders.ConfidenceScheme) + { + case ConfidenceSchemes.TRUST_USA_EUROPE: + return provider switch + { + Providers.SELF_HOSTED => ConfidenceLevel.HIGH, + Providers.FIREWORKS => ConfidenceLevel.UNTRUSTED, + + _ => ConfidenceLevel.MEDIUM, + }; + + case ConfidenceSchemes.TRUST_USA: + return provider switch + { + Providers.SELF_HOSTED => ConfidenceLevel.HIGH, + Providers.FIREWORKS => ConfidenceLevel.UNTRUSTED, + Providers.MISTRAL => ConfidenceLevel.LOW, + + _ => ConfidenceLevel.MEDIUM, + }; + + case ConfidenceSchemes.TRUST_EUROPE: + return provider switch + { + Providers.SELF_HOSTED => ConfidenceLevel.HIGH, + Providers.FIREWORKS => ConfidenceLevel.UNTRUSTED, + Providers.MISTRAL => ConfidenceLevel.MEDIUM, + + _ => ConfidenceLevel.LOW, + }; + + case ConfidenceSchemes.LOCAL_TRUST_ONLY: + return provider switch + { + Providers.SELF_HOSTED => ConfidenceLevel.HIGH, + Providers.FIREWORKS => ConfidenceLevel.UNTRUSTED, + + _ => ConfidenceLevel.VERY_LOW, + }; + + case ConfidenceSchemes.CUSTOM: + return this.ConfigurationData.LLMProviders.CustomConfidenceScheme.GetValueOrDefault(provider, ConfidenceLevel.UNKNOWN); + + default: + return ConfidenceLevel.UNKNOWN; + } + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ConfidenceSchemes.cs b/app/MindWork AI Studio/Tools/ConfidenceSchemes.cs new file mode 100644 index 00000000..97d4f4d6 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ConfidenceSchemes.cs @@ -0,0 +1,12 @@ +namespace AIStudio.Tools; + +public enum ConfidenceSchemes +{ + TRUST_USA_EUROPE = 0, + TRUST_USA = 1, + TRUST_EUROPE = 2, + + LOCAL_TRUST_ONLY = 3, + + CUSTOM = 10_000, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ConfidenceSchemesExtensions.cs b/app/MindWork AI Studio/Tools/ConfidenceSchemesExtensions.cs new file mode 100644 index 00000000..d36993f1 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ConfidenceSchemesExtensions.cs @@ -0,0 +1,16 @@ +namespace AIStudio.Tools; + +public static class ConfidenceSchemesExtensions +{ + public static string GetListDescription(this ConfidenceSchemes scheme) => scheme switch + { + ConfidenceSchemes.TRUST_USA_EUROPE => "Trust LLM providers from the USA and Europe", + ConfidenceSchemes.TRUST_USA => "Trust LLM providers from the USA", + ConfidenceSchemes.TRUST_EUROPE => "Trust LLM providers from Europe", + ConfidenceSchemes.LOCAL_TRUST_ONLY => "Trust only local LLM providers", + + ConfidenceSchemes.CUSTOM => "Configure your own confidence scheme", + + _ => "Unknown confidence scheme" + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/app.css b/app/MindWork AI Studio/wwwroot/app.css index 4bb06c5c..bc7638da 100644 --- a/app/MindWork AI Studio/wwwroot/app.css +++ b/app/MindWork AI Studio/wwwroot/app.css @@ -33,4 +33,17 @@ .mud-text-list .mud-list-item-icon { margin-top: 4px; +} + +:root { + --confidence-color: #000000; +} + +.confidence-icon { + color: var(--confidence-color); +} + +.confidence-border > .mud-input-control-input-container > .mud-input > .mud-input-outlined-border { + border-width: 2px; + border-color: var(--confidence-color) !important; } \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.9.9.md b/app/MindWork AI Studio/wwwroot/changelog/v0.9.9.md new file mode 100644 index 00000000..f3d0f2e5 --- /dev/null +++ b/app/MindWork AI Studio/wwwroot/changelog/v0.9.9.md @@ -0,0 +1,7 @@ +# v0.9.9, build 184 (2024-09-12 xx:xx UTC) +- Added confidence information to LLM providers. +- Added the possibility to choose from predefined confidence schemes. +- Added the possibility to define your own confidence scheme. +- Added an option to show/hide confidence information in the app. +- Added a visual indicator for confidence in the chat and all assistants. +- Added a confidence card to the chat and all assistants. \ No newline at end of file