diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index a3a3e68a..a287efb0 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -569,7 +569,7 @@ CONFIG["SETTINGS"] = {} -- Configure a custom confidence scheme. -- This is used when DataConfidence.ConfidenceScheme is set to CUSTOM. -- Allowed provider keys are: OPEN_AI, ANTHROPIC, MISTRAL, GOOGLE, X, DEEP_SEEK, ALIBABA_CLOUD, --- PERPLEXITY, OPEN_ROUTER, FIREWORKS, GROQ, HUGGINGFACE, SELF_HOSTED, HELMHOLTZ, GWDG +-- PERPLEXITY, OPEN_ROUTER, LITE_LLM, FIREWORKS, GROQ, HUGGINGFACE, SELF_HOSTED, HELMHOLTZ, GWDG -- Allowed confidence values are: UNTRUSTED, VERY_LOW, LOW, MODERATE, MEDIUM, HIGH -- -- Replaces, does not merge: a configuration with a higher priority replaces the whole @@ -586,6 +586,7 @@ CONFIG["SETTINGS"] = {} -- ["ALIBABA_CLOUD"] = "LOW", -- ["PERPLEXITY"] = "MODERATE", -- ["OPEN_ROUTER"] = "MODERATE", +-- ["LITE_LLM"] = "HIGH", -- ["FIREWORKS"] = "MODERATE", -- ["GROQ"] = "MODERATE", -- ["HUGGINGFACE"] = "MODERATE", diff --git a/app/MindWork AI Studio/Provider/LLMProviders.cs b/app/MindWork AI Studio/Provider/LLMProviders.cs index 6a560036..fafd45e4 100644 --- a/app/MindWork AI Studio/Provider/LLMProviders.cs +++ b/app/MindWork AI Studio/Provider/LLMProviders.cs @@ -16,6 +16,7 @@ public enum LLMProviders ALIBABA_CLOUD = 12, PERPLEXITY = 14, OPEN_ROUTER = 15, + LITE_LLM = 16, FIREWORKS = 5, GROQ = 6, diff --git a/app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs b/app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs index 92a7860d..c2306085 100644 --- a/app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs +++ b/app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs @@ -7,6 +7,7 @@ using AIStudio.Provider.Groq; using AIStudio.Provider.GWDG; using AIStudio.Provider.Helmholtz; using AIStudio.Provider.HuggingFace; +using AIStudio.Provider.LiteLLM; using AIStudio.Provider.Mistral; using AIStudio.Provider.OpenAI; using AIStudio.Provider.OpenRouter; @@ -56,11 +57,12 @@ public static class LLMProvidersExtensions LLMProviders.ALIBABA_CLOUD => "Alibaba Cloud", LLMProviders.PERPLEXITY => "Perplexity", LLMProviders.OPEN_ROUTER => "OpenRouter", + LLMProviders.LITE_LLM => "LiteLLM", LLMProviders.GROQ => "Groq", LLMProviders.FIREWORKS => "Fireworks.ai", LLMProviders.HUGGINGFACE => "Hugging Face", - + LLMProviders.SELF_HOSTED => translate ? TB("Self-hosted") : "Self-hosted", LLMProviders.HELMHOLTZ => "Helmholtz Blablador", @@ -91,6 +93,7 @@ public static class LLMProvidersExtensions LLMProviders.ALIBABA_CLOUD => "Alibaba Cloud", LLMProviders.PERPLEXITY => "Perplexity", LLMProviders.OPEN_ROUTER => "OpenRouter", + LLMProviders.LITE_LLM => "LiteLLM", LLMProviders.GROQ => "Groq", LLMProviders.FIREWORKS => "Fireworks.ai", @@ -144,6 +147,11 @@ public static class LLMProvidersExtensions LLMProviders.OPEN_ROUTER => Confidence.USA_HUB.WithRegion("America, U.S.").WithSources("https://openrouter.ai/privacy", "https://openrouter.ai/terms").WithLevel(settingsManager.GetConfiguredConfidenceLevel(llmProvider)), + // LiteLLM is a self-operated gateway: the user runs the proxy and decides which downstream + // providers it routes to, so the data destination cannot be known in advance. It is treated + // like a self-hosted endpoint, and the user assigns the trust level themselves. + LLMProviders.LITE_LLM => Confidence.SELF_HOSTED.WithLevel(settingsManager.GetConfiguredConfidenceLevel(llmProvider)), + LLMProviders.SELF_HOSTED => Confidence.SELF_HOSTED.WithLevel(settingsManager.GetConfiguredConfidenceLevel(llmProvider)), LLMProviders.HELMHOLTZ => Confidence.GDPR_NO_TRAINING.WithRegion("Europe, Germany").WithSources("https://helmholtz.cloud/services/?serviceID=d7d5c597-a2f6-4bd1-b71e-4d6499d98570").WithLevel(settingsManager.GetConfiguredConfidenceLevel(llmProvider)), @@ -180,15 +188,16 @@ public static class LLMProvidersExtensions LLMProviders.HUGGINGFACE => false, LLMProviders.PERPLEXITY => false, LLMProviders.OPEN_ROUTER => true, + LLMProviders.LITE_LLM => false, // // Self-hosted providers are treated as a special case anyway. // LLMProviders.SELF_HOSTED => true, - + _ => false, }; - + public static bool ProvideTranscriptionAPI(this LLMProviders llmProvider) => llmProvider switch { // @@ -215,7 +224,8 @@ public static class LLMProvidersExtensions LLMProviders.DEEP_SEEK => false, LLMProviders.HUGGINGFACE => false, LLMProviders.PERPLEXITY => false, - + LLMProviders.LITE_LLM => false, + LLMProviders.HELMHOLTZ => false, // @@ -271,6 +281,7 @@ public static class LLMProvidersExtensions LLMProviders.ALIBABA_CLOUD => new ProviderAlibabaCloud { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration }, LLMProviders.PERPLEXITY => new ProviderPerplexity { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration }, LLMProviders.OPEN_ROUTER => new ProviderOpenRouter { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration }, + LLMProviders.LITE_LLM => new ProviderLiteLLM(hostname) { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration }, LLMProviders.GROQ => new ProviderGroq { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration }, LLMProviders.FIREWORKS => new ProviderFireworks { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration }, @@ -408,6 +419,7 @@ public static class LLMProvidersExtensions public static bool IsHostnameNeeded(this LLMProviders provider) => provider switch { LLMProviders.SELF_HOSTED => true, + LLMProviders.LITE_LLM => true, _ => false, }; @@ -422,15 +434,16 @@ public static class LLMProvidersExtensions LLMProviders.ALIBABA_CLOUD => true, LLMProviders.PERPLEXITY => true, LLMProviders.OPEN_ROUTER => true, + LLMProviders.LITE_LLM => true, LLMProviders.GROQ => true, LLMProviders.FIREWORKS => true, LLMProviders.HELMHOLTZ => true, LLMProviders.GWDG => true, LLMProviders.HUGGINGFACE => true, - + LLMProviders.SELF_HOSTED => host is (Host.OLLAMA or Host.VLLM), - + _ => false, }; diff --git a/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs b/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs new file mode 100644 index 00000000..a7a0a9ef --- /dev/null +++ b/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs @@ -0,0 +1,118 @@ +using System.Runtime.CompilerServices; + +using AIStudio.Chat; +using AIStudio.Provider.OpenAI; +using AIStudio.Settings; + +namespace AIStudio.Provider.LiteLLM; + +public sealed class ProviderLiteLLM(string hostname) : BaseProvider(LLMProviders.LITE_LLM, BuildBaseUri(hostname), ExternalHttpTrustPolicy.ALLOW_CUSTOM_ROOTS_WHEN_HOST_WHITELISTED, LOGGER) +{ + private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(); + + #region Implementation of IProvider + + /// + public override string Id => LLMProviders.LITE_LLM.ToSecretId(); + + /// + public override string InstanceName { get; set; } = "LiteLLM"; + + /// + public override bool HasModelLoadingCapability => true; + + /// + public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) + { + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + "LiteLLM", + chatModel, + chatThread, + settingsManager, + async (systemPrompt, apiParameters) => + { + // Build the list of messages: + var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); + + return new ChatCompletionAPIRequest + { + Model = chatModel.Id, + + // Build the messages: + // - First of all the system prompt + // - Then none-empty user and AI messages + Messages = [systemPrompt, ..messages], + + Stream = true, + AdditionalApiParameters = apiParameters + }; + }, + token: token)) + yield return content; + } + + #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + /// + public override async IAsyncEnumerable StreamImageCompletion(Model imageModel, string promptPositive, string promptNegative = FilterOperator.String.Empty, ImageURL referenceImageURL = default, [EnumeratorCancellation] CancellationToken token = default) + { + yield break; + } + #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously + + /// + public override Task TranscribeAudioAsync(Model transcriptionModel, string audioFilePath, SettingsManager settingsManager, CancellationToken token = default) + { + return Task.FromResult(TranscriptionResult.Failure()); + } + + /// + public override Task>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List texts) + { + return Task.FromResult>>([]); + } + + /// + public override Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) + { + return this.LoadModels(SecretStoreType.LLM_PROVIDER, token, apiKeyProvisional); + } + + /// + public override Task GetImageModels(string? apiKeyProvisional = null, CancellationToken token = default) + { + return Task.FromResult(ModelLoadResult.FromModels([])); + } + + /// + public override Task GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default) + { + return Task.FromResult(ModelLoadResult.FromModels([])); + } + + /// + public override Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) + { + return Task.FromResult(ModelLoadResult.FromModels([])); + } + + #endregion + + private static Uri BuildBaseUri(string hostname) + { + // LiteLLM exposes an OpenAI-compatible API under the "/v1/" path. Users configure the + // base URL of their LiteLLM proxy (e.g. http://localhost:4000); we normalize any trailing + // slash and append the OpenAI-compatible path. + var normalizedHostname = (hostname ?? string.Empty).TrimEnd('/'); + return new Uri($"{normalizedHostname}/v1/"); + } + + private Task LoadModels(SecretStoreType storeType, CancellationToken token, string? apiKeyProvisional = null) + { + return this.LoadModelsResponse( + storeType, + "models", + modelResponse => modelResponse.Data, + token, + apiKeyProvisional); + } +} diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs index ec95ee9b..4e6e2c13 100644 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs +++ b/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs @@ -90,6 +90,7 @@ public static partial class ProviderExtensions GetQwenReasoningState(parameters)), LLMProviders.OPEN_ROUTER or + LLMProviders.LITE_LLM or LLMProviders.X or LLMProviders.DEEP_SEEK or LLMProviders.GROQ or diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.cs index 3d18e586..c98f18aa 100644 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.cs +++ b/app/MindWork AI Studio/Settings/ProviderExtensions.cs @@ -54,6 +54,7 @@ public static partial class ProviderExtensions LLMProviders.ALIBABA_CLOUD => GetModelCapabilitiesAlibaba(model), LLMProviders.PERPLEXITY => GetModelCapabilitiesPerplexity(model), LLMProviders.OPEN_ROUTER => GetModelCapabilitiesOpenRouter(model), + LLMProviders.LITE_LLM => GetModelCapabilitiesOpenSource(model), LLMProviders.GROQ => GetModelCapabilitiesOpenSource(model), LLMProviders.FIREWORKS => GetModelCapabilitiesOpenSource(model), diff --git a/app/MindWork AI Studio/Settings/SettingsManager.cs b/app/MindWork AI Studio/Settings/SettingsManager.cs index 43d86255..29cdddbd 100644 --- a/app/MindWork AI Studio/Settings/SettingsManager.cs +++ b/app/MindWork AI Studio/Settings/SettingsManager.cs @@ -590,6 +590,7 @@ public sealed class SettingsManager return llmProvider switch { LLMProviders.SELF_HOSTED => ConfidenceLevel.HIGH, + LLMProviders.LITE_LLM => ConfidenceLevel.HIGH, _ => ConfidenceLevel.MEDIUM, }; @@ -598,6 +599,7 @@ public sealed class SettingsManager return llmProvider switch { LLMProviders.SELF_HOSTED => ConfidenceLevel.HIGH, + LLMProviders.LITE_LLM => ConfidenceLevel.HIGH, LLMProviders.DEEP_SEEK => ConfidenceLevel.LOW, _ => ConfidenceLevel.MEDIUM, @@ -607,6 +609,7 @@ public sealed class SettingsManager return llmProvider switch { LLMProviders.SELF_HOSTED => ConfidenceLevel.HIGH, + LLMProviders.LITE_LLM => ConfidenceLevel.HIGH, LLMProviders.MISTRAL => ConfidenceLevel.LOW, LLMProviders.HELMHOLTZ => ConfidenceLevel.LOW, LLMProviders.GWDG => ConfidenceLevel.LOW, @@ -619,6 +622,7 @@ public sealed class SettingsManager return llmProvider switch { LLMProviders.SELF_HOSTED => ConfidenceLevel.HIGH, + LLMProviders.LITE_LLM => ConfidenceLevel.HIGH, LLMProviders.MISTRAL => ConfidenceLevel.MEDIUM, LLMProviders.HELMHOLTZ => ConfidenceLevel.MEDIUM, LLMProviders.GWDG => ConfidenceLevel.MEDIUM, @@ -630,6 +634,7 @@ public sealed class SettingsManager return llmProvider switch { LLMProviders.SELF_HOSTED => ConfidenceLevel.HIGH, + LLMProviders.LITE_LLM => ConfidenceLevel.HIGH, LLMProviders.DEEP_SEEK => ConfidenceLevel.MEDIUM, _ => ConfidenceLevel.LOW, @@ -639,6 +644,7 @@ public sealed class SettingsManager return llmProvider switch { LLMProviders.SELF_HOSTED => ConfidenceLevel.HIGH, + LLMProviders.LITE_LLM => ConfidenceLevel.HIGH, _ => ConfidenceLevel.VERY_LOW, }; diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md index 35c346ce..c759a762 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md @@ -7,6 +7,7 @@ - Added options for organizations to disable importing, sharing, and exporting plugins, with a separate option for configuration plugins. Organizations can now let people import assistants while keeping configurations to their IT department. - Added a priority for configuration plugins. Organizations that deploy several configurations can now decide which one wins: a configuration with a higher priority overrides the settings and providers of a lower one. This allows a company-wide base configuration that each department refines for itself. - Added a way for IT departments to try out a configuration before rolling it out. A configuration placed in the new `.config-tests` directory below the plugins directory acts like one your organization deployed, including the approval of assistant plugins, so a test shows exactly what colleagues will see later. No configuration server is needed for this. AI Studio empties that directory every time it starts, so a test configuration is valid for one session, and the information page reports it while it is active. The Enterprise IT documentation describes the whole procedure. +- Added LiteLLM as a new LLM provider. LiteLLM is an AI gateway you run yourself that gives you a single, OpenAI-compatible endpoint in front of 100+ models from providers such as OpenAI, Anthropic, Google, Azure, AWS Bedrock, and many more. Configure the address of your LiteLLM proxy and your key, and AI Studio loads the available models automatically and streams chat responses through it. - Improved how your organization's configuration behaves when a configuration plugin is present but cannot be loaded, e.g. because of an error in the plugin. Such a plugin still manages your app, so its settings, providers, data sources, profiles, and chat templates now stay in place instead of being removed. - Changed how approvals for assistant plugins combine when your organization deploys several configurations. They now add up, so a department can approve additional assistant plugins without repeating the approvals of the company-wide configuration. Previously, the last configuration replaced all earlier approvals, which silently required a new security check for those assistants. - Fixed reset buttons in assistants. As you may have noticed in the Document Analysis Assistant, resetting it could leave content from the previous analysis visible. Reset buttons now clear previous results completely.