Add confidence levels (#140)

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2024-09-11 23:08:02 +02:00
1 parent 222c1e929f
commit 4722c1baa1
44 files changed
+564 -60

No files matched your search

@@ -0,0 +1,65 @@
namespace AIStudio.Provider;
public sealed record Confidence
{
public ConfidenceLevel Level { get; private init; } = ConfidenceLevel.UNKNOWN;
public string Region { get; private init; } = string.Empty;
public string Description { get; private init; } = string.Empty;
public List<string> Sources { get; private init; } = [];
private Confidence()
{
}
public Confidence WithSources(params string[] sources) => this with { Sources = sources.ToList() };
public Confidence WithRegion(string region) => this with { Region = region };
public Confidence WithLevel(ConfidenceLevel level) => this with { Level = level };
public string StyleBorder() => $"border: 2px solid {this.Level.GetColor()}; border-radius: 6px;";
public string SetColorStyle() => $"--confidence-color: {this.Level.GetColor()};";
public static readonly Confidence NONE = new()
{
Level = ConfidenceLevel.NONE,
Description =
"""
No provider selected. Please select a provider to get see its confidence level.
""",
};
public static readonly Confidence USA_NOT_TRUSTED = new()
{
Level = ConfidenceLevel.UNTRUSTED,
Description = "The provider operates its service from the USA and is subject to **U.S. jurisdiction**. In case of suspicion, authorities in the USA can access your data. The provider's terms of service state that **all your data can be used by the provider at will.**",
};
public static readonly Confidence UNKNOWN = new()
{
Level = ConfidenceLevel.UNKNOWN,
Description = "The trust level of this provider **has not yet** been thoroughly **investigated and evaluated**. We do not know if your data is safe.",
};
public static readonly Confidence USA_NO_TRAINING = new()
{
Level = ConfidenceLevel.MODERATE,
Description = "The provider operates its service from the USA and is subject to **US jurisdiction**. In case of suspicion, authorities in the USA can access your data. However, **your data is not used for training** purposes.",
};
public static readonly Confidence GDPR_NO_TRAINING = new()
{
Level = ConfidenceLevel.MEDIUM,
Description = "The provider is located in the EU and is subject to the **GDPR** (General Data Protection Regulation). Additionally, the provider states that **your data is not used for training**.",
};
public static readonly Confidence SELF_HOSTED = new()
{
Level = ConfidenceLevel.HIGH,
Description = "You or your organization operate the LLM locally or within your trusted network. In terms of data processing and security, this is the best possible way.",
};
}
@@ -0,0 +1,18 @@
namespace AIStudio.Provider;
public enum ConfidenceLevel
{
NONE = 0,
UNTRUSTED = 1,
UNKNOWN = 10,
VERY_LOW = 90,
LOW = 100,
MODERATE = 150,
MEDIUM = 200,
HIGH = 300,
}
@@ -0,0 +1,34 @@
namespace AIStudio.Provider;
public static class ConfidenceLevelExtensions
{
public static string GetName(this ConfidenceLevel level) => level switch
{
ConfidenceLevel.NONE => "No provider selected",
ConfidenceLevel.UNTRUSTED => "Untrusted",
ConfidenceLevel.VERY_LOW => "Very Low",
ConfidenceLevel.LOW => "Low",
ConfidenceLevel.MODERATE => "Moderate",
ConfidenceLevel.MEDIUM => "Medium",
ConfidenceLevel.HIGH => "High",
_ => "Unknown confidence level",
};
public static string GetColor(this ConfidenceLevel level) => level switch
{
ConfidenceLevel.NONE => "#cccccc",
ConfidenceLevel.UNTRUSTED => "#ff0000",
ConfidenceLevel.VERY_LOW => "#ff6600",
ConfidenceLevel.LOW => "#ffcc00",
ConfidenceLevel.MODERATE => "#99cc00",
ConfidenceLevel.MEDIUM => "#86b300",
ConfidenceLevel.HIGH => "#009933",
_ => "#cc6600",
};
public static string SetColorStyle(this ConfidenceLevel level) => $"--confidence-color: {level.GetColor()};";
}
@@ -3,6 +3,7 @@ using AIStudio.Provider.Fireworks;
using AIStudio.Provider.Mistral;
using AIStudio.Provider.OpenAI;
using AIStudio.Provider.SelfHosted;
using AIStudio.Settings;
namespace AIStudio.Provider;
@@ -27,6 +28,33 @@ public static class ProvidersExtensions
_ => "Unknown",
};
/// <summary>
/// Get a provider's confidence.
/// </summary>
/// <param name="provider">The provider.</param>
/// <param name="settingsManager">The settings manager.</param>
/// <returns>The confidence of the provider.</returns>
public static Confidence GetConfidence(this Providers provider, SettingsManager settingsManager) => provider switch
{
Providers.NONE => Confidence.NONE,
Providers.FIREWORKS => Confidence.USA_NOT_TRUSTED.WithRegion("America, U.S.").WithSources("https://fireworks.ai/terms-of-service").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)),
Providers.OPEN_AI => Confidence.USA_NO_TRAINING.WithRegion("America, U.S.").WithSources(
"https://openai.com/policies/terms-of-use/",
"https://help.openai.com/en/articles/5722486-how-your-data-is-used-to-improve-model-performance",
"https://openai.com/enterprise-privacy/"
).WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)),
Providers.ANTHROPIC => Confidence.USA_NO_TRAINING.WithRegion("America, U.S.").WithSources("https://www.anthropic.com/legal/commercial-terms").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)),
Providers.MISTRAL => Confidence.GDPR_NO_TRAINING.WithRegion("Europe, France").WithSources("https://mistral.ai/terms/#terms-of-service-la-plateforme").WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)),
Providers.SELF_HOSTED => Confidence.SELF_HOSTED.WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)),
_ => Confidence.UNKNOWN.WithLevel(settingsManager.GetConfiguredConfidenceLevel(provider)),
};
/// <summary>
/// Creates a new provider instance based on the provider value.