MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2025-07-04 03:42:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add model capabilities (#450)

Browse Source
This commit is contained in:
Thorsten Sommer 2025-05-11 12:51:35 +02:00 committed by GitHub
parent 07b858fd72
commit d6af5a3afd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 587 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs
View File

@ -140,6 +140,77 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
return this.LoadModels(["text-embedding-"], token, apiKeyProvisional).ContinueWith(t => t.Result.Concat(additionalModels).OrderBy(x => x.Id).AsEnumerable(), token);
}
/// <inheritdoc />
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model)
{
var modelName = model.Id.ToLowerInvariant().AsSpan();
// Qwen models:
if (modelName.StartsWith("qwen"))
{
// Check for omni models:
if (modelName.IndexOf("omni") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.AUDIO_INPUT, Capability.SPEECH_INPUT,
Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT, Capability.SPEECH_OUTPUT
];
// Check for Qwen 3:
if(modelName.StartsWith("qwen3"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING
];
if(modelName.IndexOf("-vl-") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
];
}
// QwQ models:
if (modelName.StartsWith("qwq"))
{
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING
];
}
// QVQ models:
if (modelName.StartsWith("qvq"))
{
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING
];
}
// Default to text input and output:
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
];
}
#endregion
private async Task<IEnumerable<Model>> LoadModels(string[] prefixes, CancellationToken token, string? apiKeyProvisional = null)

27
app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs
View File

@ -110,6 +110,33 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap
return Task.FromResult(Enumerable.Empty<Model>());
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model)
{
var modelName = model.Id.ToLowerInvariant().AsSpan();
// Claude 3.7 is able to do reasoning:
if(modelName.StartsWith("claude-3-7"))
return [
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING];
// All other 3.x models are able to process text and images as input:
if(modelName.StartsWith("claude-3-"))
return [
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING];
// Any other model is able to process text only:
return [
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING];
}
#endregion
private async Task<IEnumerable<Model>> LoadModels(CancellationToken token, string? apiKeyProvisional = null)

3
app/MindWork AI Studio/Provider/BaseProvider.cs
View File

@ -77,6 +77,9 @@ public abstract class BaseProvider : IProvider, ISecretId
/// <inheritdoc />
public abstract Task<IEnumerable<Model>> GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default);
/// <inheritdoc />
public abstract IReadOnlyCollection<Capability> GetModelCapabilities(Model model);
#endregion
#region Implementation of ISecretId

150
app/MindWork AI Studio/Provider/CapabilitiesOpenSource.cs Normal file
View File

@ -0,0 +1,150 @@
namespace AIStudio.Provider;
public static class CapabilitiesOpenSource
{
public static IReadOnlyCollection<Capability> GetCapabilities(Model model)
{
var modelName = model.Id.ToLowerInvariant().AsSpan();
//
// Checking for names in the case of open source models is a hard task.
// Let's assume we want to check for the llama 3.1 405b model.
//
// Here is a not complete list of how providers name this model:
// - Fireworks: accounts/fireworks/models/llama-v3p1-405b-instruct
// - Hugging Face -> Nebius AI Studio: meta-llama/Meta-Llama-3.1-405B-Instruct
// - Groq: llama-3.1-405b-instruct
// - LM Studio: llama-3.1-405b-instruct
// - Helmholtz Blablador: 1 - Llama3 405 the best general model
// - GWDG: Llama 3.1 405B Instruct
//
//
// Meta llama models:
//
if (modelName.IndexOf("llama") is not -1)
{
if (modelName.IndexOf("llama4") is not -1 ||
modelName.IndexOf("llama 4") is not -1 ||
modelName.IndexOf("llama-4") is not -1 ||
modelName.IndexOf("llama-v4") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
// The old vision models cannot do function calling:
if (modelName.IndexOf("vision") is not -1)
return [Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.TEXT_OUTPUT];
//
// All models >= 3.1 are able to do function calling:
//
if (modelName.IndexOf("llama3.") is not -1 ||
modelName.IndexOf("llama 3.") is not -1 ||
modelName.IndexOf("llama-3.") is not -1 ||
modelName.IndexOf("llama-v3p") is not -1)
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
// All other llama models can only do text input and output:
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
}
//
// DeepSeek models:
//
if (modelName.IndexOf("deepseek") is not -1)
{
if(modelName.IndexOf("deepseek-r1") is not -1 ||
modelName.IndexOf("deepseek r1") is not -1)
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, Capability.ALWAYS_REASONING];
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
}
//
// Qwen models:
//
if (modelName.IndexOf("qwen") is not -1 || modelName.IndexOf("qwq") is not -1)
{
if (modelName.IndexOf("qwq") is not -1)
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, Capability.ALWAYS_REASONING];
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
}
//
// Mistral models:
//
if (modelName.IndexOf("mistral") is not -1 ||
modelName.IndexOf("pixtral") is not -1)
{
if(modelName.IndexOf("pixtral") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
];
if (modelName.IndexOf("3.1") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
];
// Default:
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
];
}
//
// Grok models:
//
if (modelName.IndexOf("grok") is not -1)
{
if(modelName.IndexOf("-vision-") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
];
if(modelName.StartsWith("grok-3-mini"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING,
];
if(modelName.StartsWith("grok-3"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
}
// Default:
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
}
}

97
app/MindWork AI Studio/Provider/Capability.cs Normal file
View File

@ -0,0 +1,97 @@
namespace AIStudio.Provider;
/// <summary>
/// Represents the capabilities of an AI model.
/// </summary>
public enum Capability
{
/// <summary>
/// No capabilities specified.
/// </summary>
NONE,
/// <summary>
/// We don't know what the AI model can do.
/// </summary>
UNKNOWN,
/// <summary>
/// The AI model can perform text input.
/// </summary>
TEXT_INPUT,
/// <summary>
/// The AI model can perform audio input, such as music or sound.
/// </summary>
AUDIO_INPUT,
/// <summary>
/// The AI model can perform one image input, such as one photo or drawing.
/// </summary>
SINGLE_IMAGE_INPUT,
/// <summary>
/// The AI model can perform multiple images as input, such as multiple photos or drawings.
/// </summary>
MULTIPLE_IMAGE_INPUT,
/// <summary>
/// The AI model can perform speech input.
/// </summary>
SPEECH_INPUT,
/// <summary>
/// The AI model can perform video input, such as video files or streams.
/// </summary>
VIDEO_INPUT,
/// <summary>
/// The AI model can generate text output.
/// </summary>
TEXT_OUTPUT,
/// <summary>
/// The AI model can generate audio output, such as music or sound.
/// </summary>
AUDIO_OUTPUT,
/// <summary>
/// The AI model can generate image output, such as photos or drawings.
/// </summary>
IMAGE_OUTPUT,
/// <summary>
/// The AI model can generate speech output.
/// </summary>
SPEECH_OUTPUT,
/// <summary>
/// The AI model can generate video output.
/// </summary>
VIDEO_OUTPUT,
/// <summary>
/// The AI model can perform reasoning tasks.
/// </summary>
OPTIONAL_REASONING,
/// <summary>
/// The AI model always performs reasoning.
/// </summary>
ALWAYS_REASONING,
/// <summary>
/// The AI model can embed information or data.
/// </summary>
EMBEDDING,
/// <summary>
/// The AI model can perform in real-time.
/// </summary>
REALTIME,
/// <summary>
/// The AI model can perform function calling, such as invoking APIs or executing functions.
/// </summary>
FUNCTION_CALLING,
}

21
app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs
View File

@ -106,6 +106,27 @@ public sealed class ProviderDeepSeek(ILogger logger) : BaseProvider("https://api
return Task.FromResult(Enumerable.Empty<Model>());
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model)
{
var modelName = model.Id.ToLowerInvariant().AsSpan();
if(modelName.IndexOf("reasoner") is not -1)
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING,
];
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
];
}
#endregion
private async Task<IEnumerable<Model>> LoadModels(CancellationToken token, string? apiKeyProvisional = null)

2
app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs
View File

@ -107,5 +107,7 @@ public class ProviderFireworks(ILogger logger) : BaseProvider("https://api.firew
return Task.FromResult(Enumerable.Empty<Model>());
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model) => CapabilitiesOpenSource.GetCapabilities(model);
#endregion
}

2
app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs
View File

@ -108,6 +108,8 @@ public sealed class ProviderGWDG(ILogger logger) : BaseProvider("https://chat-ai
return models.Where(model => model.Id.StartsWith("e5-", StringComparison.InvariantCultureIgnoreCase));
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model) => CapabilitiesOpenSource.GetCapabilities(model);
#endregion
private async Task<IEnumerable<Model>> LoadModels(CancellationToken token, string? apiKeyProvisional = null)

82
app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs
View File

@ -119,6 +119,88 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
.Select(n => new Provider.Model(n.Name.Replace("models/", string.Empty), n.DisplayName));
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Provider.Model model)
{
var modelName = model.Id.ToLowerInvariant().AsSpan();
if (modelName.IndexOf("gemini-") is not -1)
{
// Reasoning models:
if (modelName.IndexOf("gemini-2.5") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.AUDIO_INPUT,
Capability.SPEECH_INPUT, Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING,
];
// Image generation:
if(modelName.IndexOf("-2.0-flash-preview-image-") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.AUDIO_INPUT,
Capability.SPEECH_INPUT, Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT, Capability.IMAGE_OUTPUT,
];
// Realtime model:
if(modelName.IndexOf("-2.0-flash-live-") is not -1)
return
[
Capability.TEXT_INPUT, Capability.AUDIO_INPUT, Capability.SPEECH_INPUT,
Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT, Capability.SPEECH_OUTPUT,
Capability.FUNCTION_CALLING,
];
// The 2.0 flash models cannot call functions:
if(modelName.IndexOf("-2.0-flash-") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.AUDIO_INPUT,
Capability.SPEECH_INPUT, Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT,
];
// The old 1.0 pro vision model:
if(modelName.IndexOf("pro-vision") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
];
// Default to all other Gemini models:
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.AUDIO_INPUT,
Capability.SPEECH_INPUT, Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
}
// Default for all other models:
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
}
#endregion
private async Task<ModelsResponse> LoadModels(CancellationToken token, string? apiKeyProvisional = null)

2
app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs
View File

@ -110,6 +110,8 @@ public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/o
return Task.FromResult(Enumerable.Empty<Model>());
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model) => CapabilitiesOpenSource.GetCapabilities(model);
#endregion
private async Task<IEnumerable<Model>> LoadModels(CancellationToken token, string? apiKeyProvisional = null)

2
app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs
View File

@ -112,6 +112,8 @@ public sealed class ProviderHelmholtz(ILogger logger) : BaseProvider("https://ap
model.Id.Contains("gritlm", StringComparison.InvariantCultureIgnoreCase));
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model) => CapabilitiesOpenSource.GetCapabilities(model);
#endregion
private async Task<IEnumerable<Model>> LoadModels(CancellationToken token, string? apiKeyProvisional = null)

2
app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs
View File

@ -111,5 +111,7 @@ public sealed class ProviderHuggingFace : BaseProvider
return Task.FromResult(Enumerable.Empty<Model>());
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model) => CapabilitiesOpenSource.GetCapabilities(model);
#endregion
}

7
app/MindWork AI Studio/Provider/IProvider.cs
View File

@ -63,4 +63,11 @@ public interface IProvider
/// <param name="token">The cancellation token.</param>
/// <returns>The list of embedding models.</returns>
public Task<IEnumerable<Model>> GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default);
/// <summary>
/// Get the capabilities of a model.
/// </summary>
/// <param name="model">The model to get the capabilities for.</param>
/// <returns>The capabilities of the model.</returns>
public IReadOnlyCollection<Capability> GetModelCapabilities(Model model);
}

46
app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs
View File

@ -122,6 +122,52 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
return Task.FromResult(Enumerable.Empty<Provider.Model>());
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Provider.Model model)
{
var modelName = model.Id.ToLowerInvariant().AsSpan();
// Pixtral models are able to do process images:
if (modelName.IndexOf("pixtral") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
// Mistral medium:
if (modelName.IndexOf("mistral-medium-") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
// Mistral small:
if (modelName.IndexOf("mistral-small-") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
// Mistral saba:
if (modelName.IndexOf("mistral-saba-") is not -1)
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
];
// Default:
return CapabilitiesOpenSource.GetCapabilities(model);
}
#endregion
private async Task<ModelsResponse> LoadModelList(string? apiKeyProvisional, CancellationToken token)

2
app/MindWork AI Studio/Provider/NoProvider.cs
View File

@ -31,5 +31,7 @@ public class NoProvider : IProvider
yield break;
}
public IReadOnlyCollection<Capability> GetModelCapabilities(Model model) => [ Capability.NONE ];
#endregion
}

63
app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs
View File

@ -143,6 +143,69 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
return this.LoadModels(["text-embedding-"], token, apiKeyProvisional);
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model)
{
var modelName = model.Id.ToLowerInvariant().AsSpan();
if (modelName.StartsWith("o1-mini"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING,
];
if (modelName.StartsWith("o3-mini"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING
];
if (modelName.StartsWith("o4-mini") || modelName.StartsWith("o1") || modelName.StartsWith("o3"))
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING
];
if(modelName.StartsWith("gpt-3.5"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
];
if(modelName.StartsWith("gpt-4-turbo"))
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
];
if(modelName is "gpt-4" || modelName.StartsWith("gpt-4-"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
];
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
];
}
#endregion
private async Task<IEnumerable<Model>> LoadModels(string[] prefixes, CancellationToken token, string? apiKeyProvisional = null)

3
app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs
View File

@ -88,7 +88,6 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam
}
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
public override async Task<IEnumerable<Provider.Model>> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
{
try
@ -140,6 +139,8 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam
}
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Provider.Model model) => CapabilitiesOpenSource.GetCapabilities(model);
#endregion
private async Task<IEnumerable<Provider.Model>> LoadModels(string[] ignorePhrases, string[] filterPhrases, CancellationToken token, string? apiKeyProvisional = null)

2
app/MindWork AI Studio/Provider/X/ProviderX.cs
View File

@ -111,6 +111,8 @@ public sealed class ProviderX(ILogger logger) : BaseProvider("https://api.x.ai/v
return Task.FromResult<IEnumerable<Model>>([]);
}
public override IReadOnlyCollection<Capability> GetModelCapabilities(Model model) => CapabilitiesOpenSource.GetCapabilities(model);
#endregion
private async Task<IEnumerable<Model>> LoadModels(string[] prefixes, CancellationToken token, string? apiKeyProvisional = null)