From 4722c1baa1868d16e390fab590d454b883165a27 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 11 Sep 2024 23:08:02 +0200 Subject: [PATCH] Add confidence levels (#140) --- app/MindWork AI Studio.sln.DotSettings | 1 + .../Assistants/Agenda/AssistantAgenda.razor | 6 +- .../Agenda/AssistantAgenda.razor.cs | 4 ++ .../Assistants/AssistantBase.razor | 15 ++++- .../Assistants/AssistantBase.razor.cs | 8 +++ .../Assistants/Coding/AssistantCoding.razor | 6 +- .../Coding/AssistantCoding.razor.cs | 4 ++ .../Assistants/EMail/AssistantEMail.razor | 6 +- .../Assistants/EMail/AssistantEMail.razor.cs | 4 ++ .../AssistantGrammarSpelling.razor | 6 +- .../AssistantGrammarSpelling.razor.cs | 4 ++ .../IconFinder/AssistantIconFinder.razor | 6 +- .../IconFinder/AssistantIconFinder.razor.cs | 4 ++ .../LegalCheck/AssistantLegalCheck.razor | 6 +- .../LegalCheck/AssistantLegalCheck.razor.cs | 6 ++ .../Assistants/MyTasks/AssistantMyTasks.razor | 6 +- .../MyTasks/AssistantMyTasks.razor.cs | 4 ++ .../AssistantRewriteImprove.razor | 6 +- .../AssistantRewriteImprove.razor.cs | 4 ++ .../Synonym/AssistantSynonyms.razor | 4 -- .../Synonym/AssistantSynonyms.razor.cs | 4 ++ .../AssistantTextSummarizer.razor | 6 +- .../AssistantTextSummarizer.razor.cs | 6 ++ .../Translation/AssistantTranslation.razor | 6 +- .../Translation/AssistantTranslation.razor.cs | 6 ++ .../Components/ConfidenceInfo.razor | 56 ++++++++++++++++ .../Components/ConfidenceInfo.razor.cs | 57 ++++++++++++++++ .../Components/ConfidenceInfoMode.cs | 7 ++ app/MindWork AI Studio/Pages/Chat.razor | 7 +- app/MindWork AI Studio/Pages/Chat.razor.cs | 4 ++ app/MindWork AI Studio/Pages/Settings.razor | 54 ++++++++++++++- .../Pages/Settings.razor.cs | 22 +++++++ app/MindWork AI Studio/Provider/Confidence.cs | 65 +++++++++++++++++++ .../Provider/ConfidenceLevel.cs | 18 +++++ .../Provider/ConfidenceLevelExtensions.cs | 34 ++++++++++ .../Provider/ProvidersExtensions.cs | 28 ++++++++ .../Settings/ConfigurationSelectData.cs | 6 ++ .../Settings/DataModel/Data.cs | 5 ++ .../Settings/DataModel/DataLLMProviders.cs | 21 ++++++ .../Settings/SettingsManager.cs | 54 +++++++++++++++ .../Tools/ConfidenceSchemes.cs | 12 ++++ .../Tools/ConfidenceSchemesExtensions.cs | 16 +++++ app/MindWork AI Studio/wwwroot/app.css | 13 ++++ .../wwwroot/changelog/v0.9.9.md | 7 ++ 44 files changed, 564 insertions(+), 60 deletions(-) create mode 100644 app/MindWork AI Studio/Components/ConfidenceInfo.razor create mode 100644 app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs create mode 100644 app/MindWork AI Studio/Components/ConfidenceInfoMode.cs create mode 100644 app/MindWork AI Studio/Provider/Confidence.cs create mode 100644 app/MindWork AI Studio/Provider/ConfidenceLevel.cs create mode 100644 app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs create mode 100644 app/MindWork AI Studio/Settings/DataModel/DataLLMProviders.cs create mode 100644 app/MindWork AI Studio/Tools/ConfidenceSchemes.cs create mode 100644 app/MindWork AI Studio/Tools/ConfidenceSchemesExtensions.cs create mode 100644 app/MindWork AI Studio/wwwroot/changelog/v0.9.9.md diff --git a/app/MindWork AI Studio.sln.DotSettings b/app/MindWork AI Studio.sln.DotSettings index 74de15b..fceb77c 100644 --- a/app/MindWork AI Studio.sln.DotSettings +++ b/app/MindWork AI Studio.sln.DotSettings @@ -1,5 +1,6 @@  AI + LLM LM MSG True diff --git a/app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor b/app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor index ca0aca8..018800c 100644 --- a/app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor +++ b/app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor @@ -48,8 +48,4 @@ } - - - - Create agenda - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor.cs b/app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor.cs index c104e41..310f070 100644 --- a/app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor.cs +++ b/app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor.cs @@ -97,6 +97,10 @@ public partial class AssistantAgenda : AssistantBaseCore protected override IReadOnlyList FooterButtons => []; + protected override string SubmitText => "Create Agenda"; + + protected override Func SubmitAction => this.CreateAgenda; + protected override ChatThread ConvertToChatThread => (this.chatThread ?? new()) with { SystemPrompt = SystemPrompts.DEFAULT, diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor b/app/MindWork AI Studio/Assistants/AssistantBase.razor index ac744a4..dd6d5b1 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor @@ -8,12 +8,16 @@ - @(this.Description) + @this.Description @if (this.Body is not null) { - @(this.Body) + @this.Body + + + @this.SubmitText + } @@ -93,7 +97,12 @@ Reset - + + @if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence) + { + + } + @if (this.AllowProfiles && this.ShowProfileSelection) { diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index 5bf8425..837d19f 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -52,6 +52,12 @@ public abstract partial class AssistantBase : ComponentBase protected abstract bool MightPreselectValues(); + protected abstract string SubmitText { get; } + + protected abstract Func SubmitAction { get; } + + protected virtual bool SubmitDisabled => false; + private protected virtual RenderFragment? Body => null; protected virtual bool ShowResult => true; @@ -107,6 +113,8 @@ public abstract partial class AssistantBase : ComponentBase } #endregion + + private string SubmitButtonStyle => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence ? this.providerSettings.UsedProvider.GetConfidence(this.SettingsManager).StyleBorder() : string.Empty; protected string? ValidatingProvider(AIStudio.Settings.Provider provider) { diff --git a/app/MindWork AI Studio/Assistants/Coding/AssistantCoding.razor b/app/MindWork AI Studio/Assistants/Coding/AssistantCoding.razor index 65fb656..6af0819 100644 --- a/app/MindWork AI Studio/Assistants/Coding/AssistantCoding.razor +++ b/app/MindWork AI Studio/Assistants/Coding/AssistantCoding.razor @@ -24,8 +24,4 @@ - - - - Get support - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/Coding/AssistantCoding.razor.cs b/app/MindWork AI Studio/Assistants/Coding/AssistantCoding.razor.cs index 3476425..b06a3d4 100644 --- a/app/MindWork AI Studio/Assistants/Coding/AssistantCoding.razor.cs +++ b/app/MindWork AI Studio/Assistants/Coding/AssistantCoding.razor.cs @@ -28,6 +28,10 @@ public partial class AssistantCoding : AssistantBaseCore protected override IReadOnlyList FooterButtons => []; + protected override string SubmitText => "Get Support"; + + protected override Func SubmitAction => this.GetSupport; + protected override void ResetFrom() { this.codingContexts.Clear(); diff --git a/app/MindWork AI Studio/Assistants/EMail/AssistantEMail.razor b/app/MindWork AI Studio/Assistants/EMail/AssistantEMail.razor index e8997aa..57c5787 100644 --- a/app/MindWork AI Studio/Assistants/EMail/AssistantEMail.razor +++ b/app/MindWork AI Studio/Assistants/EMail/AssistantEMail.razor @@ -20,8 +20,4 @@ - - - - Create e-mail - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/EMail/AssistantEMail.razor.cs b/app/MindWork AI Studio/Assistants/EMail/AssistantEMail.razor.cs index a25077d..2f9b6f3 100644 --- a/app/MindWork AI Studio/Assistants/EMail/AssistantEMail.razor.cs +++ b/app/MindWork AI Studio/Assistants/EMail/AssistantEMail.razor.cs @@ -24,6 +24,10 @@ public partial class AssistantEMail : AssistantBaseCore protected override IReadOnlyList FooterButtons => []; + protected override string SubmitText => "Create email"; + + protected override Func SubmitAction => this.CreateMail; + protected override ChatThread ConvertToChatThread => (this.chatThread ?? new()) with { SystemPrompt = SystemPrompts.DEFAULT, diff --git a/app/MindWork AI Studio/Assistants/GrammarSpelling/AssistantGrammarSpelling.razor b/app/MindWork AI Studio/Assistants/GrammarSpelling/AssistantGrammarSpelling.razor index d09355c..18a4280 100644 --- a/app/MindWork AI Studio/Assistants/GrammarSpelling/AssistantGrammarSpelling.razor +++ b/app/MindWork AI Studio/Assistants/GrammarSpelling/AssistantGrammarSpelling.razor @@ -3,8 +3,4 @@ - - - - Proofread - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/GrammarSpelling/AssistantGrammarSpelling.razor.cs b/app/MindWork AI Studio/Assistants/GrammarSpelling/AssistantGrammarSpelling.razor.cs index 21227df..f90af67 100644 --- a/app/MindWork AI Studio/Assistants/GrammarSpelling/AssistantGrammarSpelling.razor.cs +++ b/app/MindWork AI Studio/Assistants/GrammarSpelling/AssistantGrammarSpelling.razor.cs @@ -41,6 +41,10 @@ public partial class AssistantGrammarSpelling : AssistantBaseCore }, ]; + protected override string SubmitText => "Proofread"; + + protected override Func SubmitAction => this.ProofreadText; + protected override ChatThread ConvertToChatThread => (this.chatThread ?? new()) with { SystemPrompt = SystemPrompts.DEFAULT, diff --git a/app/MindWork AI Studio/Assistants/IconFinder/AssistantIconFinder.razor b/app/MindWork AI Studio/Assistants/IconFinder/AssistantIconFinder.razor index de8425e..af241b8 100644 --- a/app/MindWork AI Studio/Assistants/IconFinder/AssistantIconFinder.razor +++ b/app/MindWork AI Studio/Assistants/IconFinder/AssistantIconFinder.razor @@ -15,8 +15,4 @@ Open website } - - - - Find icon - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/IconFinder/AssistantIconFinder.razor.cs b/app/MindWork AI Studio/Assistants/IconFinder/AssistantIconFinder.razor.cs index e239c8f..5123b91 100644 --- a/app/MindWork AI Studio/Assistants/IconFinder/AssistantIconFinder.razor.cs +++ b/app/MindWork AI Studio/Assistants/IconFinder/AssistantIconFinder.razor.cs @@ -29,6 +29,10 @@ public partial class AssistantIconFinder : AssistantBaseCore protected override IReadOnlyList FooterButtons => []; + protected override string SubmitText => "Find Icon"; + + protected override Func SubmitAction => this.FindIcon; + protected override void ResetFrom() { this.inputContext = string.Empty; diff --git a/app/MindWork AI Studio/Assistants/LegalCheck/AssistantLegalCheck.razor b/app/MindWork AI Studio/Assistants/LegalCheck/AssistantLegalCheck.razor index 0f1328b..f8494ea 100644 --- a/app/MindWork AI Studio/Assistants/LegalCheck/AssistantLegalCheck.razor +++ b/app/MindWork AI Studio/Assistants/LegalCheck/AssistantLegalCheck.razor @@ -8,8 +8,4 @@ - - - - Ask your questions - + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/LegalCheck/AssistantLegalCheck.razor.cs b/app/MindWork AI Studio/Assistants/LegalCheck/AssistantLegalCheck.razor.cs index 0520194..b8dd0ef 100644 --- a/app/MindWork AI Studio/Assistants/LegalCheck/AssistantLegalCheck.razor.cs +++ b/app/MindWork AI Studio/Assistants/LegalCheck/AssistantLegalCheck.razor.cs @@ -24,6 +24,12 @@ public partial class AssistantLegalCheck : AssistantBaseCore protected override IReadOnlyList FooterButtons => []; + protected override string SubmitText => "Ask your questions"; + + protected override Func SubmitAction => this.AksQuestions; + + protected override bool SubmitDisabled => this.isAgentRunning; + protected override void ResetFrom() { this.inputLegalDocument = string.Empty; diff --git a/app/MindWork AI Studio/Assistants/MyTasks/AssistantMyTasks.razor b/app/MindWork AI Studio/Assistants/MyTasks/AssistantMyTasks.razor index 1b34dba..23154c8 100644 --- a/app/MindWork AI Studio/Assistants/MyTasks/AssistantMyTasks.razor +++ b/app/MindWork AI Studio/Assistants/MyTasks/AssistantMyTasks.razor @@ -4,8 +4,4 @@ - - - - Analyze text - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/MyTasks/AssistantMyTasks.razor.cs b/app/MindWork AI Studio/Assistants/MyTasks/AssistantMyTasks.razor.cs index 6eba246..1b3f5be 100644 --- a/app/MindWork AI Studio/Assistants/MyTasks/AssistantMyTasks.razor.cs +++ b/app/MindWork AI Studio/Assistants/MyTasks/AssistantMyTasks.razor.cs @@ -28,6 +28,10 @@ public partial class AssistantMyTasks : AssistantBaseCore """; protected override IReadOnlyList FooterButtons => []; + + protected override string SubmitText => "Analyze text"; + + protected override Func SubmitAction => this.AnalyzeText; protected override bool ShowProfileSelection => false; diff --git a/app/MindWork AI Studio/Assistants/RewriteImprove/AssistantRewriteImprove.razor b/app/MindWork AI Studio/Assistants/RewriteImprove/AssistantRewriteImprove.razor index c9f8be4..c599ad1 100644 --- a/app/MindWork AI Studio/Assistants/RewriteImprove/AssistantRewriteImprove.razor +++ b/app/MindWork AI Studio/Assistants/RewriteImprove/AssistantRewriteImprove.razor @@ -5,8 +5,4 @@ - - - - Improve - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/RewriteImprove/AssistantRewriteImprove.razor.cs b/app/MindWork AI Studio/Assistants/RewriteImprove/AssistantRewriteImprove.razor.cs index 6d8aef6..0274610 100644 --- a/app/MindWork AI Studio/Assistants/RewriteImprove/AssistantRewriteImprove.razor.cs +++ b/app/MindWork AI Studio/Assistants/RewriteImprove/AssistantRewriteImprove.razor.cs @@ -42,6 +42,10 @@ public partial class AssistantRewriteImprove : AssistantBaseCore }, ]; + protected override string SubmitText => "Improve"; + + protected override Func SubmitAction => this.RewriteText; + protected override ChatThread ConvertToChatThread => (this.chatThread ?? new()) with { SystemPrompt = SystemPrompts.DEFAULT, diff --git a/app/MindWork AI Studio/Assistants/Synonym/AssistantSynonyms.razor b/app/MindWork AI Studio/Assistants/Synonym/AssistantSynonyms.razor index 908972c..5ce13aa 100644 --- a/app/MindWork AI Studio/Assistants/Synonym/AssistantSynonyms.razor +++ b/app/MindWork AI Studio/Assistants/Synonym/AssistantSynonyms.razor @@ -6,7 +6,3 @@ - - - Get synonyms - diff --git a/app/MindWork AI Studio/Assistants/Synonym/AssistantSynonyms.razor.cs b/app/MindWork AI Studio/Assistants/Synonym/AssistantSynonyms.razor.cs index c238aac..444c348 100644 --- a/app/MindWork AI Studio/Assistants/Synonym/AssistantSynonyms.razor.cs +++ b/app/MindWork AI Studio/Assistants/Synonym/AssistantSynonyms.razor.cs @@ -51,6 +51,10 @@ public partial class AssistantSynonyms : AssistantBaseCore protected override IReadOnlyList FooterButtons => []; + protected override string SubmitText => "Find synonyms"; + + protected override Func SubmitAction => this.FindSynonyms; + protected override ChatThread ConvertToChatThread => (this.chatThread ?? new()) with { SystemPrompt = SystemPrompts.DEFAULT, diff --git a/app/MindWork AI Studio/Assistants/TextSummarizer/AssistantTextSummarizer.razor b/app/MindWork AI Studio/Assistants/TextSummarizer/AssistantTextSummarizer.razor index 69a7de9..5302a1a 100644 --- a/app/MindWork AI Studio/Assistants/TextSummarizer/AssistantTextSummarizer.razor +++ b/app/MindWork AI Studio/Assistants/TextSummarizer/AssistantTextSummarizer.razor @@ -9,8 +9,4 @@ - - - - Summarize - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/TextSummarizer/AssistantTextSummarizer.razor.cs b/app/MindWork AI Studio/Assistants/TextSummarizer/AssistantTextSummarizer.razor.cs index 20c0300..8e5b071 100644 --- a/app/MindWork AI Studio/Assistants/TextSummarizer/AssistantTextSummarizer.razor.cs +++ b/app/MindWork AI Studio/Assistants/TextSummarizer/AssistantTextSummarizer.razor.cs @@ -29,6 +29,12 @@ public partial class AssistantTextSummarizer : AssistantBaseCore protected override IReadOnlyList FooterButtons => []; + protected override string SubmitText => "Summarize"; + + protected override Func SubmitAction => this.SummarizeText; + + protected override bool SubmitDisabled => this.isAgentRunning; + protected override ChatThread ConvertToChatThread => (this.chatThread ?? new()) with { SystemPrompt = SystemPrompts.DEFAULT, diff --git a/app/MindWork AI Studio/Assistants/Translation/AssistantTranslation.razor b/app/MindWork AI Studio/Assistants/Translation/AssistantTranslation.razor index f05f662..fa15879 100644 --- a/app/MindWork AI Studio/Assistants/Translation/AssistantTranslation.razor +++ b/app/MindWork AI Studio/Assistants/Translation/AssistantTranslation.razor @@ -17,8 +17,4 @@ else } - - - - Translate - \ No newline at end of file + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/Translation/AssistantTranslation.razor.cs b/app/MindWork AI Studio/Assistants/Translation/AssistantTranslation.razor.cs index 29cc0ef..3f2c37f 100644 --- a/app/MindWork AI Studio/Assistants/Translation/AssistantTranslation.razor.cs +++ b/app/MindWork AI Studio/Assistants/Translation/AssistantTranslation.razor.cs @@ -25,6 +25,12 @@ public partial class AssistantTranslation : AssistantBaseCore protected override IReadOnlyList FooterButtons => []; + protected override string SubmitText => "Translate"; + + protected override Func SubmitAction => () => this.TranslateText(true); + + protected override bool SubmitDisabled => this.isAgentRunning; + protected override ChatThread ConvertToChatThread => (this.chatThread ?? new()) with { SystemPrompt = SystemPrompts.DEFAULT, diff --git a/app/MindWork AI Studio/Components/ConfidenceInfo.razor b/app/MindWork AI Studio/Components/ConfidenceInfo.razor new file mode 100644 index 0000000..ebb76e8 --- /dev/null +++ b/app/MindWork AI Studio/Components/ConfidenceInfo.razor @@ -0,0 +1,56 @@ +@using AIStudio.Provider +
+@if (this.Mode is ConfidenceInfoMode.ICON) +{ + +} +else +{ + + Confidence + +} + + + + + + Confidence Card + + + + Description + + + @if (this.currentConfidence.Sources.Count > 0) + { + Sources + + @foreach (var sourceTuple in this.GetConfidenceSources()) + { + + } + + } + + @if (!string.IsNullOrWhiteSpace(this.currentConfidence.Region)) + { + Region + + @this.currentConfidence.Region + + } + + Confidence Level + + @this.currentConfidence.Level.GetName() + + + + + Close + + + + +
\ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs b/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs new file mode 100644 index 0000000..d8b72c9 --- /dev/null +++ b/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs @@ -0,0 +1,57 @@ +using AIStudio.Provider; +using AIStudio.Settings; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +public partial class ConfidenceInfo : ComponentBase +{ + [Parameter] + public ConfidenceInfoMode Mode { get; set; } = ConfidenceInfoMode.BUTTON; + + [Parameter] + public Providers Provider { get; set; } + + [Inject] + private SettingsManager SettingsManager { get; init; } = null!; + + private Confidence currentConfidence; + private bool showConfidence; + + public ConfidenceInfo() + { + this.currentConfidence = Providers.NONE.GetConfidence(this.SettingsManager); + } + + #region Overrides of ComponentBase + + protected override async Task OnParametersSetAsync() + { + this.currentConfidence = this.Provider.GetConfidence(this.SettingsManager); + await base.OnParametersSetAsync(); + } + + #endregion + + private void ToggleConfidence() + { + this.showConfidence = !this.showConfidence; + } + + private void HideConfidence() + { + this.showConfidence = false; + } + + private IEnumerable<(string Index, string Source)> GetConfidenceSources() + { + var index = 0; + foreach (var source in this.currentConfidence.Sources) + yield return ($"Source {++index}", source); + } + + private string GetCurrentConfidenceColor() => $"color: {this.currentConfidence.Level.GetColor()};"; + + private string GetPopoverStyle() => $"border-color: {this.currentConfidence.Level.GetColor()}; max-width: calc(35vw);"; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ConfidenceInfoMode.cs b/app/MindWork AI Studio/Components/ConfidenceInfoMode.cs new file mode 100644 index 0000000..d7e63da --- /dev/null +++ b/app/MindWork AI Studio/Components/ConfidenceInfoMode.cs @@ -0,0 +1,7 @@ +namespace AIStudio.Components; + +public enum ConfidenceInfoMode +{ + BUTTON, + ICON, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Pages/Chat.razor b/app/MindWork AI Studio/Pages/Chat.razor index 3d5d3b7..806f060 100644 --- a/app/MindWork AI Studio/Pages/Chat.razor +++ b/app/MindWork AI Studio/Pages/Chat.razor @@ -28,7 +28,7 @@ - + @if (this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is not WorkspaceStorageBehavior.DISABLE_WORKSPACES) @@ -70,6 +70,11 @@ } + @if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence) + { + + } + diff --git a/app/MindWork AI Studio/Pages/Chat.razor.cs b/app/MindWork AI Studio/Pages/Chat.razor.cs index 96dcc98..8e06c03 100644 --- a/app/MindWork AI Studio/Pages/Chat.razor.cs +++ b/app/MindWork AI Studio/Pages/Chat.razor.cs @@ -120,6 +120,10 @@ public partial class Chat : MSGComponentBase, IAsyncDisposable private string TooltipAddChatToWorkspace => $"Start new chat in workspace \"{this.currentWorkspaceName}\""; + private string UserInputStyle => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence ? this.providerSettings.UsedProvider.GetConfidence(this.SettingsManager).SetColorStyle() : string.Empty; + + private string UserInputClass => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence ? "confidence-border" : string.Empty; + private void ProfileWasChanged(Profile profile) { this.currentProfile = profile; diff --git a/app/MindWork AI Studio/Pages/Settings.razor b/app/MindWork AI Studio/Pages/Settings.razor index 5ba3b60..2cfed9d 100644 --- a/app/MindWork AI Studio/Pages/Settings.razor +++ b/app/MindWork AI Studio/Pages/Settings.razor @@ -17,7 +17,7 @@ task. As an LLM provider, you can also choose local providers. However, to use this app, you must configure at least one provider. - + @@ -72,6 +72,56 @@ Add Provider + + LLM Provider Confidence + + Do you want to always be able to recognize how trustworthy your LLM providers are? This way, + you keep control over which provider you send your data to. You have two options for this: + Either you choose a common schema, or you configure the trust levels for each LLM provider yourself. + + + + @if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence) + { + + @if (this.SettingsManager.ConfigurationData.LLMProviders.ConfidenceScheme is ConfidenceSchemes.CUSTOM) + { + + + + + + + + LLM Provider + Description + Confidence Level + + + + @context.ToName() + + + + + + + @foreach (var confidenceLevel in Enum.GetValues().OrderBy(n => n)) + { + if(confidenceLevel is ConfidenceLevel.NONE or ConfidenceLevel.UNKNOWN) + continue; + + + @confidenceLevel.GetName() + + } + + + + + } + } + @@ -88,7 +138,7 @@ code. In these profiles, you can record how much experience you have or which methods you like or dislike using. Later, you can choose when and where you want to use each profile. - + diff --git a/app/MindWork AI Studio/Pages/Settings.razor.cs b/app/MindWork AI Studio/Pages/Settings.razor.cs index 5c09f98..90f3bdb 100644 --- a/app/MindWork AI Studio/Pages/Settings.razor.cs +++ b/app/MindWork AI Studio/Pages/Settings.razor.cs @@ -160,6 +160,28 @@ public partial class Settings : ComponentBase, IMessageBusReceiver, IDisposable this.availableProviders.Add(new (provider.InstanceName, provider.Id)); } + private string GetCurrentConfidenceLevelName(Providers provider) + { + if (this.SettingsManager.ConfigurationData.LLMProviders.CustomConfidenceScheme.TryGetValue(provider, out var level)) + return level.GetName(); + + return "Not yet configured"; + } + + private string SetCurrentConfidenceLevelColorStyle(Providers provider) + { + if (this.SettingsManager.ConfigurationData.LLMProviders.CustomConfidenceScheme.TryGetValue(provider, out var level)) + return $"background-color: {level.GetColor()};"; + + return $"background-color: {ConfidenceLevel.UNKNOWN.GetColor()};"; + } + + private async Task ChangeCustomConfidenceLevel(Providers provider, ConfidenceLevel level) + { + this.SettingsManager.ConfigurationData.LLMProviders.CustomConfidenceScheme[provider] = level; + await this.SettingsManager.StoreSettings(); + } + #endregion #region Profile related diff --git a/app/MindWork AI Studio/Provider/Confidence.cs b/app/MindWork AI Studio/Provider/Confidence.cs new file mode 100644 index 0000000..3cf08ac --- /dev/null +++ b/app/MindWork AI Studio/Provider/Confidence.cs @@ -0,0 +1,65 @@ +namespace AIStudio.Provider; + +public sealed record Confidence +{ + public ConfidenceLevel Level { get; private init; } = ConfidenceLevel.UNKNOWN; + + public string Region { get; private init; } = string.Empty; + + public string Description { get; private init; } = string.Empty; + + public List Sources { get; private init; } = []; + + private Confidence() + { + } + + public Confidence WithSources(params string[] sources) => this with { Sources = sources.ToList() }; + + public Confidence WithRegion(string region) => this with { Region = region }; + + public Confidence WithLevel(ConfidenceLevel level) => this with { Level = level }; + + public string StyleBorder() => $"border: 2px solid {this.Level.GetColor()}; border-radius: 6px;"; + + public string SetColorStyle() => $"--confidence-color: {this.Level.GetColor()};"; + + public static readonly Confidence NONE = new() + { + Level = ConfidenceLevel.NONE, + Description = + """ + No provider selected. Please select a provider to get see its confidence level. + """, + }; + + public static readonly Confidence USA_NOT_TRUSTED = new() + { + Level = ConfidenceLevel.UNTRUSTED, + Description = "The provider operates its service from the USA and is subject to **U.S. jurisdiction**. In case of suspicion, authorities in the USA can access your data. The provider's terms of service state that **all your data can be used by the provider at will.**", + }; + + public static readonly Confidence UNKNOWN = new() + { + Level = ConfidenceLevel.UNKNOWN, + Description = "The trust level of this provider **has not yet** been thoroughly **investigated and evaluated**. We do not know if your data is safe.", + }; + + public static readonly Confidence USA_NO_TRAINING = new() + { + Level = ConfidenceLevel.MODERATE, + Description = "The provider operates its service from the USA and is subject to **US jurisdiction**. In case of suspicion, authorities in the USA can access your data. However, **your data is not used for training** purposes.", + }; + + public static readonly Confidence GDPR_NO_TRAINING = new() + { + Level = ConfidenceLevel.MEDIUM, + Description = "The provider is located in the EU and is subject to the **GDPR** (General Data Protection Regulation). Additionally, the provider states that **your data is not used for training**.", + }; + + public static readonly Confidence SELF_HOSTED = new() + { + Level = ConfidenceLevel.HIGH, + Description = "You or your organization operate the LLM locally or within your trusted network. In terms of data processing and security, this is the best possible way.", + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ConfidenceLevel.cs b/app/MindWork AI Studio/Provider/ConfidenceLevel.cs new file mode 100644 index 0000000..9b1f417 --- /dev/null +++ b/app/MindWork AI Studio/Provider/ConfidenceLevel.cs @@ -0,0 +1,18 @@ +namespace AIStudio.Provider; + +public enum ConfidenceLevel +{ + NONE = 0, + UNTRUSTED = 1, + + UNKNOWN = 10, + + VERY_LOW = 90, + LOW = 100, + + MODERATE = 150, + + MEDIUM = 200, + + HIGH = 300, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs b/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs new file mode 100644 index 0000000..2f8d451 --- /dev/null +++ b/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs @@ -0,0 +1,34 @@ +namespace AIStudio.Provider; + +public static class ConfidenceLevelExtensions +{ + public static string GetName(this ConfidenceLevel level) => level switch + { + ConfidenceLevel.NONE => "No provider selected", + + ConfidenceLevel.UNTRUSTED => "Untrusted", + ConfidenceLevel.VERY_LOW => "Very Low", + ConfidenceLevel.LOW => "Low", + ConfidenceLevel.MODERATE => "Moderate", + ConfidenceLevel.MEDIUM => "Medium", + ConfidenceLevel.HIGH => "High", + + _ => "Unknown confidence level", + }; + + public static string GetColor(this ConfidenceLevel level) => level switch + { + ConfidenceLevel.NONE => "#cccccc", + + ConfidenceLevel.UNTRUSTED => "#ff0000", + ConfidenceLevel.VERY_LOW => "#ff6600", + ConfidenceLevel.LOW => "#ffcc00", + ConfidenceLevel.MODERATE => "#99cc00", + ConfidenceLevel.MEDIUM => "#86b300", + ConfidenceLevel.HIGH => "#009933", + + _ => "#cc6600", + }; + + public static string SetColorStyle(this ConfidenceLevel level) => $"--confidence-color: {level.GetColor()};"; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ProvidersExtensions.cs b/app/MindWork AI Studio/Provider/ProvidersExtensions.cs index 806c3bb..2ddc7f8 100644 --- a/app/MindWork AI Studio/Provider/ProvidersExtensions.cs +++ b/app/MindWork AI Studio/Provider/ProvidersExtensions.cs @@ -3,6 +3,7 @@ using AIStudio.Provider.Fireworks; using AIStudio.Provider.Mistral; using AIStudio.Provider.OpenAI; using AIStudio.Provider.SelfHosted; +using AIStudio.Settings; namespace AIStudio.Provider; @@ -27,6 +28,33 @@ public static class ProvidersExtensions _ => "Unknown", }; + + /// + /// Get a provider's confidence. + /// + /// The provider. + /// The settings manager. + /// The confidence of the provider. + public static Confidence GetConfidence(this Providers provider, SettingsManager settingsManager) => provider switch + { + Providers.NONE => Confidence.NONE, + + Providers.FIREWORKS => Confidence.USA_NOT_TRUSTED.WithRegion("America, U.S.").WithSources("https://fireworks.ai/terms-of-service").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + Providers.OPEN_AI => Confidence.USA_NO_TRAINING.WithRegion("America, U.S.").WithSources( + "https://openai.com/policies/terms-of-use/", + "https://help.openai.com/en/articles/5722486-how-your-data-is-used-to-improve-model-performance", + "https://openai.com/enterprise-privacy/" + ).WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + Providers.ANTHROPIC => Confidence.USA_NO_TRAINING.WithRegion("America, U.S.").WithSources("https://www.anthropic.com/legal/commercial-terms").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + Providers.MISTRAL => Confidence.GDPR_NO_TRAINING.WithRegion("Europe, France").WithSources("https://mistral.ai/terms/#terms-of-service-la-plateforme").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + Providers.SELF_HOSTED => Confidence.SELF_HOSTED.WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + + _ => Confidence.UNKNOWN.WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)), + }; /// /// Creates a new provider instance based on the provider value. diff --git a/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs b/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs index c8dfd39..c48ea6d 100644 --- a/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs +++ b/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs @@ -136,4 +136,10 @@ public static class ConfigurationSelectDataFactory foreach (var profile in profiles.GetAllProfiles()) yield return new(profile.Name, profile.Id); } + + public static IEnumerable> GetConfidenceSchemesData() + { + foreach (var scheme in Enum.GetValues()) + yield return new(scheme.GetListDescription(), scheme); + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/Data.cs b/app/MindWork AI Studio/Settings/DataModel/Data.cs index da8abc3..1fe0a92 100644 --- a/app/MindWork AI Studio/Settings/DataModel/Data.cs +++ b/app/MindWork AI Studio/Settings/DataModel/Data.cs @@ -16,6 +16,11 @@ public sealed class Data /// public List Providers { get; init; } = []; + /// + /// Settings concerning the LLM providers. + /// + public DataLLMProviders LLMProviders { get; init; } = new(); + /// /// List of configured profiles. /// diff --git a/app/MindWork AI Studio/Settings/DataModel/DataLLMProviders.cs b/app/MindWork AI Studio/Settings/DataModel/DataLLMProviders.cs new file mode 100644 index 0000000..73b7825 --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataLLMProviders.cs @@ -0,0 +1,21 @@ +using AIStudio.Provider; + +namespace AIStudio.Settings.DataModel; + +public sealed class DataLLMProviders +{ + /// + /// Should we show the provider confidence level? + /// + public bool ShowProviderConfidence { get; set; } = true; + + /// + /// Which confidence scheme to use. + /// + public ConfidenceSchemes ConfidenceScheme { get; set; } = ConfidenceSchemes.TRUST_USA_EUROPE; + + /// + /// Provide custom confidence levels for each LLM provider. + /// + public Dictionary CustomConfidenceScheme { get; set; } = new(); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/SettingsManager.cs b/app/MindWork AI Studio/Settings/SettingsManager.cs index e745921..2a660b2 100644 --- a/app/MindWork AI Studio/Settings/SettingsManager.cs +++ b/app/MindWork AI Studio/Settings/SettingsManager.cs @@ -1,6 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; +using AIStudio.Provider; using AIStudio.Settings.DataModel; // ReSharper disable NotAccessedPositionalProperty.Local @@ -160,4 +161,57 @@ public sealed class SettingsManager(ILogger logger) preselection = this.ConfigurationData.Profiles.FirstOrDefault(x => x.Id == this.ConfigurationData.App.PreselectedProfile); return preselection != default ? preselection : Profile.NO_PROFILE; } + + public ConfidenceLevel GetConfiguredConfidenceLevel(Providers provider) + { + if(provider is Providers.NONE) + return ConfidenceLevel.NONE; + + switch (this.ConfigurationData.LLMProviders.ConfidenceScheme) + { + case ConfidenceSchemes.TRUST_USA_EUROPE: + return provider switch + { + Providers.SELF_HOSTED => ConfidenceLevel.HIGH, + Providers.FIREWORKS => ConfidenceLevel.UNTRUSTED, + + _ => ConfidenceLevel.MEDIUM, + }; + + case ConfidenceSchemes.TRUST_USA: + return provider switch + { + Providers.SELF_HOSTED => ConfidenceLevel.HIGH, + Providers.FIREWORKS => ConfidenceLevel.UNTRUSTED, + Providers.MISTRAL => ConfidenceLevel.LOW, + + _ => ConfidenceLevel.MEDIUM, + }; + + case ConfidenceSchemes.TRUST_EUROPE: + return provider switch + { + Providers.SELF_HOSTED => ConfidenceLevel.HIGH, + Providers.FIREWORKS => ConfidenceLevel.UNTRUSTED, + Providers.MISTRAL => ConfidenceLevel.MEDIUM, + + _ => ConfidenceLevel.LOW, + }; + + case ConfidenceSchemes.LOCAL_TRUST_ONLY: + return provider switch + { + Providers.SELF_HOSTED => ConfidenceLevel.HIGH, + Providers.FIREWORKS => ConfidenceLevel.UNTRUSTED, + + _ => ConfidenceLevel.VERY_LOW, + }; + + case ConfidenceSchemes.CUSTOM: + return this.ConfigurationData.LLMProviders.CustomConfidenceScheme.GetValueOrDefault(provider, ConfidenceLevel.UNKNOWN); + + default: + return ConfidenceLevel.UNKNOWN; + } + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ConfidenceSchemes.cs b/app/MindWork AI Studio/Tools/ConfidenceSchemes.cs new file mode 100644 index 0000000..97d4f4d --- /dev/null +++ b/app/MindWork AI Studio/Tools/ConfidenceSchemes.cs @@ -0,0 +1,12 @@ +namespace AIStudio.Tools; + +public enum ConfidenceSchemes +{ + TRUST_USA_EUROPE = 0, + TRUST_USA = 1, + TRUST_EUROPE = 2, + + LOCAL_TRUST_ONLY = 3, + + CUSTOM = 10_000, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ConfidenceSchemesExtensions.cs b/app/MindWork AI Studio/Tools/ConfidenceSchemesExtensions.cs new file mode 100644 index 0000000..d36993f --- /dev/null +++ b/app/MindWork AI Studio/Tools/ConfidenceSchemesExtensions.cs @@ -0,0 +1,16 @@ +namespace AIStudio.Tools; + +public static class ConfidenceSchemesExtensions +{ + public static string GetListDescription(this ConfidenceSchemes scheme) => scheme switch + { + ConfidenceSchemes.TRUST_USA_EUROPE => "Trust LLM providers from the USA and Europe", + ConfidenceSchemes.TRUST_USA => "Trust LLM providers from the USA", + ConfidenceSchemes.TRUST_EUROPE => "Trust LLM providers from Europe", + ConfidenceSchemes.LOCAL_TRUST_ONLY => "Trust only local LLM providers", + + ConfidenceSchemes.CUSTOM => "Configure your own confidence scheme", + + _ => "Unknown confidence scheme" + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/app.css b/app/MindWork AI Studio/wwwroot/app.css index 4bb06c5..bc7638d 100644 --- a/app/MindWork AI Studio/wwwroot/app.css +++ b/app/MindWork AI Studio/wwwroot/app.css @@ -33,4 +33,17 @@ .mud-text-list .mud-list-item-icon { margin-top: 4px; +} + +:root { + --confidence-color: #000000; +} + +.confidence-icon { + color: var(--confidence-color); +} + +.confidence-border > .mud-input-control-input-container > .mud-input > .mud-input-outlined-border { + border-width: 2px; + border-color: var(--confidence-color) !important; } \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.9.9.md b/app/MindWork AI Studio/wwwroot/changelog/v0.9.9.md new file mode 100644 index 0000000..f3d0f2e --- /dev/null +++ b/app/MindWork AI Studio/wwwroot/changelog/v0.9.9.md @@ -0,0 +1,7 @@ +# v0.9.9, build 184 (2024-09-12 xx:xx UTC) +- Added confidence information to LLM providers. +- Added the possibility to choose from predefined confidence schemes. +- Added the possibility to define your own confidence scheme. +- Added an option to show/hide confidence information in the app. +- Added a visual indicator for confidence in the chat and all assistants. +- Added a confidence card to the chat and all assistants. \ No newline at end of file