mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-01 20:09:15 +00:00
Added transcription for Helmholtz and Groq, embeddings for GWDG, and fixed Mistral model capabilities (#942)
This commit is contained in:
parent
53da222562
commit
2965102c53
@ -9202,6 +9202,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T1999987800"] = "We tried to
|
||||
-- We tried to communicate with the LLM provider '{0}' (type={1}). You might not be able to use this provider from your location. The provider message is: '{2}'
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2107463087"] = "We tried to communicate with the LLM provider '{0}' (type={1}). You might not be able to use this provider from your location. The provider message is: '{2}'"
|
||||
|
||||
-- The provider '{0}' was not able to read the audio file. It probably does not support the WebM/Opus format which AI Studio sends. Please contact the provider about it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2304106455"] = "The provider '{0}' was not able to read the audio file. It probably does not support the WebM/Opus format which AI Studio sends. Please contact the provider about it."
|
||||
|
||||
-- We tried to communicate with the LLM provider '{0}' (type={1}). Something was not found. The provider message is: '{2}'
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3014737766"] = "We tried to communicate with the LLM provider '{0}' (type={1}). Something was not found. The provider message is: '{2}'"
|
||||
|
||||
|
||||
@ -9204,6 +9204,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T1999987800"] = "Wir haben ve
|
||||
-- We tried to communicate with the LLM provider '{0}' (type={1}). You might not be able to use this provider from your location. The provider message is: '{2}'
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2107463087"] = "Wir haben versucht, mit dem LLM-Anbieter „{0}“ (Typ={1}) zu kommunizieren. Möglicherweise können Sie diesen Anbieter von Ihrem Standort aus nicht nutzen. Die Nachricht des Anbieters lautet: „{2}“."
|
||||
|
||||
-- The provider '{0}' was not able to read the audio file. It probably does not support the WebM/Opus format which AI Studio sends. Please contact the provider about it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2304106455"] = "Der Anbieter „{0}“ konnte die Audiodatei nicht lesen. Er unterstützt vermutlich das WebM/Opus-Format nicht, das AI Studio sendet. Bitte kontaktieren Sie den Anbieter dazu."
|
||||
|
||||
-- We tried to communicate with the LLM provider '{0}' (type={1}). Something was not found. The provider message is: '{2}'
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3014737766"] = "Wir haben versucht, mit dem LLM-Anbieter „{0}“ (Typ={1}) zu kommunizieren. Etwas wurde nicht gefunden. Die Nachricht des Anbieters lautet: „{2}“"
|
||||
|
||||
|
||||
@ -9204,6 +9204,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T1999987800"] = "We tried to
|
||||
-- We tried to communicate with the LLM provider '{0}' (type={1}). You might not be able to use this provider from your location. The provider message is: '{2}'
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2107463087"] = "We tried to communicate with the LLM provider '{0}' (type={1}). You might not be able to use this provider from your location. The provider message is: '{2}'"
|
||||
|
||||
-- The provider '{0}' was not able to read the audio file. It probably does not support the WebM/Opus format which AI Studio sends. Please contact the provider about it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2304106455"] = "The provider '{0}' was not able to read the audio file. It probably does not support the WebM/Opus format which AI Studio sends. Please contact the provider about it."
|
||||
|
||||
-- We tried to communicate with the LLM provider '{0}' (type={1}). Something was not found. The provider message is: '{2}'
|
||||
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3014737766"] = "We tried to communicate with the LLM provider '{0}' (type={1}). Something was not found. The provider message is: '{2}'"
|
||||
|
||||
|
||||
@ -375,7 +375,36 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
|
||||
errorCode = TryGetString(root, "code");
|
||||
errorType = TryGetString(root, "type");
|
||||
errorMessage = TryGetString(root, "message");
|
||||
|
||||
// Services built on FastAPI, such as Helmholtz Blablador, word their errors as "detail":
|
||||
errorMessage = TryGetString(root, "message") ?? TryGetString(root, "detail");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the error message a provider sent in the body of a failed response.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Providers word their errors differently, but they all put a sentence somewhere into the
|
||||
/// body. Passing that sentence on is what lets a user act on the problem instead of only
|
||||
/// learning that something went wrong.
|
||||
/// </remarks>
|
||||
/// <param name="responseBody">The body of the failed response.</param>
|
||||
/// <returns>The message, or an empty string when the body carries none.</returns>
|
||||
private static string ReadProviderErrorMessage(string responseBody)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(responseBody))
|
||||
return string.Empty;
|
||||
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(responseBody);
|
||||
TryGetProviderStreamError(document.RootElement, out _, out _, out var errorMessage);
|
||||
return errorMessage ?? string.Empty;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryGetErrorElement(JsonElement root, out JsonElement errorElement)
|
||||
@ -1013,6 +1042,35 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
yield return content;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the message a user gets to see when a transcription request failed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// AI Studio always sends WebM/Opus. Some providers run their speech recognition behind a
|
||||
/// decoder which reads WAV only, and they answer with a bad request whose body says that it
|
||||
/// could not decode the file. Nobody can do anything about that inside AI Studio, so we name
|
||||
/// the likely cause and point at the provider instead of showing the raw message.
|
||||
/// </remarks>
|
||||
/// <param name="statusCode">The status code the provider answered with.</param>
|
||||
/// <param name="responseBody">The body the provider answered with.</param>
|
||||
/// <returns>The message to show, or an empty string when we have nothing to say.</returns>
|
||||
private string GetTranscriptionFailureUserMessage(HttpStatusCode statusCode, string responseBody)
|
||||
{
|
||||
var failureReason = this.ClassifyProviderRequestFailure(statusCode, responseBody);
|
||||
var classifiedMessage = this.GetProviderRequestFailureUserMessage(failureReason);
|
||||
if (!string.IsNullOrWhiteSpace(classifiedMessage))
|
||||
return classifiedMessage;
|
||||
|
||||
if (statusCode is HttpStatusCode.BadRequest && responseBody.Contains("not decode", StringComparison.OrdinalIgnoreCase))
|
||||
return string.Format(TB("The provider '{0}' was not able to read the audio file. It probably does not support the WebM/Opus format which AI Studio sends. Please contact the provider about it."), this.InstanceName);
|
||||
|
||||
var providerMessage = ReadProviderErrorMessage(responseBody);
|
||||
if (!string.IsNullOrWhiteSpace(providerMessage))
|
||||
return string.Format(TB("The provider '{0}' reported an error: {1}"), this.InstanceName, providerMessage);
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
protected async Task<TranscriptionResult> PerformStandardTranscriptionRequest(RequestedSecret requestedSecret, Model transcriptionModel, string audioFilePath, Host host = Host.NONE, CancellationToken token = default)
|
||||
{
|
||||
try
|
||||
@ -1082,8 +1140,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
this.logger.LogError("Transcription request failed with status code {ResponseStatusCode} and body: '{ResponseBody}'.", response.StatusCode, responseBody);
|
||||
var providerRequestFailure = this.ClassifyProviderRequestFailure(response.StatusCode, responseBody);
|
||||
return TranscriptionResult.Failure(this.GetProviderRequestFailureUserMessage(providerRequestFailure));
|
||||
return TranscriptionResult.Failure(this.GetTranscriptionFailureUserMessage(response.StatusCode, responseBody));
|
||||
}
|
||||
|
||||
var transcriptionResponse = JsonSerializer.Deserialize<TranscriptionResponse>(responseBody, JSON_SERIALIZER_OPTIONS);
|
||||
@ -1162,6 +1219,15 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
this.logger.LogError("Embedding request failed with status code {ResponseStatusCode} and body: '{ResponseBody}'.", response.StatusCode, responseBody);
|
||||
var providerRequestFailure = this.ClassifyProviderRequestFailure(response.StatusCode, responseBody);
|
||||
var userMessage = this.GetProviderRequestFailureUserMessage(providerRequestFailure);
|
||||
|
||||
// We know nothing about this failure, so we pass on what the provider said about it:
|
||||
if (string.IsNullOrWhiteSpace(userMessage))
|
||||
{
|
||||
var providerMessage = ReadProviderErrorMessage(responseBody);
|
||||
if (!string.IsNullOrWhiteSpace(providerMessage))
|
||||
userMessage = string.Format(TB("The provider '{0}' reported an error: {1}"), this.InstanceName, providerMessage);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(userMessage))
|
||||
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, userMessage));
|
||||
|
||||
@ -1214,7 +1280,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
|
||||
protected static bool TryPopIntParameter(IDictionary<string, object> parameters, string key, out int value)
|
||||
{
|
||||
value = default;
|
||||
value = 0;
|
||||
if (!TryPopParameter(parameters, key, out var raw) || raw is null)
|
||||
return false;
|
||||
|
||||
@ -1224,15 +1290,15 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
value = i;
|
||||
return true;
|
||||
|
||||
case long l when l is >= int.MinValue and <= int.MaxValue:
|
||||
case long l and >= int.MinValue and <= int.MaxValue:
|
||||
value = (int)l;
|
||||
return true;
|
||||
|
||||
case double d when d is >= int.MinValue and <= int.MaxValue:
|
||||
case double d and >= int.MinValue and <= int.MaxValue:
|
||||
value = (int)d;
|
||||
return true;
|
||||
|
||||
case decimal m when m is >= int.MinValue and <= int.MaxValue:
|
||||
case decimal m and >= int.MinValue and <= int.MaxValue:
|
||||
value = (int)m;
|
||||
return true;
|
||||
}
|
||||
@ -1242,7 +1308,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
|
||||
protected static bool TryPopBoolParameter(IDictionary<string, object> parameters, string key, out bool value)
|
||||
{
|
||||
value = default;
|
||||
value = false;
|
||||
if (!TryPopParameter(parameters, key, out var raw) || raw is null)
|
||||
return false;
|
||||
|
||||
|
||||
@ -10,6 +10,14 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht
|
||||
{
|
||||
private static readonly ILogger<ProviderGWDG> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGWDG>();
|
||||
|
||||
// Source: https://docs.hpc.gwdg.de/services/saia/index.html#embeddings
|
||||
private static readonly Model[] KNOWN_EMBEDDING_MODELS =
|
||||
[
|
||||
new("e5-mistral-7b-instruct", "E5 Mistral 7B Instruct"),
|
||||
new("multilingual-e5-large-instruct", "Multilingual E5 Large Instruct"),
|
||||
new("qwen3-embedding-4b", "Qwen3 Embedding 4B"),
|
||||
];
|
||||
|
||||
#region Implementation of IProvider
|
||||
|
||||
/// <inheritdoc />
|
||||
@ -67,9 +75,10 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht
|
||||
}
|
||||
|
||||
/// <inhertidoc />
|
||||
public override Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
|
||||
public override async Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<IReadOnlyList<float>>>([]);
|
||||
var requestedSecret = await Program.RUST_SERVICE.GetAPIKey(this, SecretStoreType.EMBEDDING_PROVIDER);
|
||||
return await this.PerformStandardTextEmbeddingRequest(requestedSecret, embeddingModel, token: token, texts: texts);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@ -89,12 +98,26 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// SAIA answers the models endpoint with its chat models only, so asking it for the embedding
|
||||
/// models comes back empty. We therefore fall back to the models the documentation names. The
|
||||
/// endpoint is still asked first: should SAIA start reporting them one day, its answer wins
|
||||
/// over our list. A failed request is passed on unchanged, so a wrong API key stays visible
|
||||
/// as such instead of being covered up by the fallback.
|
||||
/// </remarks>
|
||||
public override async Task<ModelLoadResult> GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
{
|
||||
var result = await this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, token, apiKeyProvisional);
|
||||
if (!result.Success)
|
||||
return result;
|
||||
|
||||
var embeddingModels = result.Models.Where(model => model.IsEmbeddingModel()).ToList();
|
||||
if (embeddingModels.Count is 0)
|
||||
return ModelLoadResult.FromModels(KNOWN_EMBEDDING_MODELS);
|
||||
|
||||
return result with
|
||||
{
|
||||
Models = [..result.Models.Where(model => model.IsEmbeddingModel())]
|
||||
Models = [..embeddingModels]
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -64,9 +64,10 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a
|
||||
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<TranscriptionResult> TranscribeAudioAsync(Model transcriptionModel, string audioFilePath, SettingsManager settingsManager, CancellationToken token = default)
|
||||
public override async Task<TranscriptionResult> TranscribeAudioAsync(Model transcriptionModel, string audioFilePath, SettingsManager settingsManager, CancellationToken token = default)
|
||||
{
|
||||
return Task.FromResult(TranscriptionResult.Failure());
|
||||
var requestedSecret = await Program.RUST_SERVICE.GetAPIKey(this, SecretStoreType.TRANSCRIPTION_PROVIDER);
|
||||
return await this.PerformStandardTranscriptionRequest(requestedSecret, transcriptionModel, audioFilePath, token: token);
|
||||
}
|
||||
|
||||
/// <inhertidoc />
|
||||
@ -76,9 +77,13 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<ModelLoadResult> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
public override async Task<ModelLoadResult> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
{
|
||||
return this.LoadModels(SecretStoreType.LLM_PROVIDER, token, apiKeyProvisional);
|
||||
var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, token, apiKeyProvisional);
|
||||
return result with
|
||||
{
|
||||
Models = [..result.Models.Where(model => model.IsChatModel())]
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@ -94,9 +99,13 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<ModelLoadResult> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
public override async Task<ModelLoadResult> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
{
|
||||
return Task.FromResult(ModelLoadResult.FromModels([]));
|
||||
var result = await this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, token, apiKeyProvisional);
|
||||
return result with
|
||||
{
|
||||
Models = [..result.Models.Where(model => model.IsTranscriptionModel())]
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
@ -106,10 +115,7 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a
|
||||
return this.LoadModelsResponse<ModelsResponse>(
|
||||
storeType,
|
||||
"models",
|
||||
modelResponse => modelResponse.Data.Where(n =>
|
||||
!n.Id.StartsWith("whisper-", StringComparison.OrdinalIgnoreCase) &&
|
||||
!n.Id.StartsWith("distil-", StringComparison.OrdinalIgnoreCase) &&
|
||||
!n.Id.Contains("-tts", StringComparison.OrdinalIgnoreCase)),
|
||||
modelResponse => modelResponse.Data,
|
||||
token,
|
||||
apiKeyProvisional);
|
||||
}
|
||||
|
||||
@ -62,9 +62,10 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, n
|
||||
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<TranscriptionResult> TranscribeAudioAsync(Model transcriptionModel, string audioFilePath, SettingsManager settingsManager, CancellationToken token = default)
|
||||
public override async Task<TranscriptionResult> TranscribeAudioAsync(Model transcriptionModel, string audioFilePath, SettingsManager settingsManager, CancellationToken token = default)
|
||||
{
|
||||
return Task.FromResult(TranscriptionResult.Failure());
|
||||
var requestedSecret = await Program.RUST_SERVICE.GetAPIKey(this, SecretStoreType.TRANSCRIPTION_PROVIDER);
|
||||
return await this.PerformStandardTranscriptionRequest(requestedSecret, transcriptionModel, audioFilePath, token: token);
|
||||
}
|
||||
|
||||
/// <inhertidoc />
|
||||
@ -107,9 +108,16 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, n
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<ModelLoadResult> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
public override async Task<ModelLoadResult> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
{
|
||||
return Task.FromResult(ModelLoadResult.FromModels([]));
|
||||
var result = await this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, token, apiKeyProvisional);
|
||||
return result with
|
||||
{
|
||||
Models =
|
||||
[
|
||||
..result.Models.Where(model => model.IsTranscriptionModel())
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
@ -185,7 +185,9 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.HELMHOLTZ => true,
|
||||
LLMProviders.ALIBABA_CLOUD => true,
|
||||
LLMProviders.IONOS => true,
|
||||
|
||||
LLMProviders.GWDG => true,
|
||||
LLMProviders.OPEN_ROUTER => true,
|
||||
|
||||
//
|
||||
// Providers that do not support embeddings:
|
||||
//
|
||||
@ -193,11 +195,9 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.ANTHROPIC => false,
|
||||
LLMProviders.FIREWORKS => false,
|
||||
LLMProviders.X => false,
|
||||
LLMProviders.GWDG => false,
|
||||
LLMProviders.DEEP_SEEK => false,
|
||||
LLMProviders.HUGGINGFACE => false,
|
||||
LLMProviders.PERPLEXITY => false,
|
||||
LLMProviders.OPEN_ROUTER => true,
|
||||
LLMProviders.HETZNER => false,
|
||||
|
||||
//
|
||||
@ -217,7 +217,9 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.MISTRAL => true,
|
||||
LLMProviders.FIREWORKS => true,
|
||||
LLMProviders.GWDG => true,
|
||||
|
||||
LLMProviders.HELMHOLTZ => true,
|
||||
LLMProviders.GROQ => true,
|
||||
|
||||
//
|
||||
// Providers that support transcription but provide no OpenAI-compatible API yet:
|
||||
//
|
||||
@ -230,20 +232,17 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.OPEN_ROUTER => false,
|
||||
LLMProviders.HETZNER => false,
|
||||
LLMProviders.IONOS => false,
|
||||
LLMProviders.GROQ => false,
|
||||
LLMProviders.ANTHROPIC => false,
|
||||
LLMProviders.X => false,
|
||||
LLMProviders.DEEP_SEEK => false,
|
||||
LLMProviders.HUGGINGFACE => false,
|
||||
LLMProviders.PERPLEXITY => false,
|
||||
|
||||
LLMProviders.HELMHOLTZ => false,
|
||||
|
||||
//
|
||||
// Self-hosted providers are treated as a special case anyway.
|
||||
//
|
||||
LLMProviders.SELF_HOSTED => true,
|
||||
|
||||
|
||||
_ => false,
|
||||
};
|
||||
|
||||
|
||||
@ -105,6 +105,13 @@ public static class ModelKindExtensions
|
||||
/// <returns>True, when the model is an embedding model.</returns>
|
||||
public static bool IsEmbeddingModel(this Model model) => model.DetermineKind() is ModelKind.EMBEDDING;
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether this model transcribes audio.
|
||||
/// </summary>
|
||||
/// <param name="model">The model to check.</param>
|
||||
/// <returns>True, when the model is a transcription model.</returns>
|
||||
public static bool IsTranscriptionModel(this Model model) => model.DetermineKind() is ModelKind.TRANSCRIPTION;
|
||||
|
||||
private static bool HasAnyMarker(string modelId, string[] markers)
|
||||
{
|
||||
foreach (var marker in markers)
|
||||
|
||||
@ -4,100 +4,79 @@ namespace AIStudio.Settings;
|
||||
|
||||
public static partial class ProviderExtensions
|
||||
{
|
||||
//
|
||||
// Mistral names its models after the month they were released: mistral-large-2512 is
|
||||
// Mistral Large 3 from December 2025. The version number lives in the marketing name only,
|
||||
// so matching on it misses nearly every model the API actually serves. The constants below
|
||||
// read as YYMM and say from which release on a family gained a capability.
|
||||
//
|
||||
private const int MISTRAL_LARGE_VISION_SINCE = 2512; // Mistral Large 3
|
||||
private const int MISTRAL_LARGE_REASONING_SINCE = 2512; // Mistral Large 3
|
||||
private const int MISTRAL_MEDIUM_VISION_SINCE = 2505; // Mistral Medium 3
|
||||
private const int MISTRAL_MEDIUM_REASONING_SINCE = 2604; // Mistral Medium 3.5
|
||||
private const int MISTRAL_SMALL_VISION_SINCE = 2503; // Mistral Small 3.1
|
||||
private const int MISTRAL_SMALL_REASONING_SINCE = 2603; // Mistral Small 4
|
||||
private const int MINISTRAL_VISION_SINCE = 2512; // Ministral 3
|
||||
|
||||
/// <summary>
|
||||
/// Used for families which have no reasoning at all. No release date can ever reach it.
|
||||
/// </summary>
|
||||
private const int MISTRAL_REASONING_NEVER = int.MaxValue;
|
||||
|
||||
//
|
||||
// Where the "latest" aliases point to. Mistral moves them on with every release, so they
|
||||
// have to behave like the release they resolve to instead of carrying their own rules.
|
||||
//
|
||||
private const int MISTRAL_LARGE_LATEST = 2512;
|
||||
private const int MISTRAL_MEDIUM_LATEST = 2604;
|
||||
private const int MISTRAL_SMALL_LATEST = 2603;
|
||||
private const int MINISTRAL_LATEST = 2512;
|
||||
|
||||
/// <summary>
|
||||
/// Mistral released its first date-named model in 2023. Anything below that is not a release
|
||||
/// date but a parameter count or a context size which happens to have four digits.
|
||||
/// </summary>
|
||||
private const int MISTRAL_FIRST_RELEASE_YEAR = 23;
|
||||
|
||||
//
|
||||
// Mistral serves some models under their marketing version as well, and it writes the version
|
||||
// separator both ways: mistral-medium-3.5 and mistral-medium-3-5 are the same model. Those
|
||||
// names carry no release date, so we map them onto the release they stand for. The order
|
||||
// matters: the more specific version has to come first, otherwise "3" would swallow "3.5".
|
||||
//
|
||||
private static readonly (string VersionName, int ReleaseDate)[] MISTRAL_VERSION_NAMES =
|
||||
[
|
||||
("mistral-large-3", 2512),
|
||||
|
||||
("mistral-medium-3.5", 2604),
|
||||
("mistral-medium-3-5", 2604),
|
||||
("mistral-medium-3.1", 2508),
|
||||
("mistral-medium-3-1", 2508),
|
||||
("mistral-medium-3", 2505),
|
||||
|
||||
("mistral-small-4", 2603),
|
||||
("mistral-small-3.2", 2506),
|
||||
("mistral-small-3-2", 2506),
|
||||
("mistral-small-3.1", 2503),
|
||||
("mistral-small-3-1", 2503),
|
||||
("mistral-small-3", 2501),
|
||||
];
|
||||
|
||||
private static List<Capability> GetModelCapabilitiesMistral(Model model)
|
||||
{
|
||||
var modelName = model.Id.ToLowerInvariant().AsSpan();
|
||||
|
||||
|
||||
// Pixtral models are able to do process images:
|
||||
if (modelName.IndexOf("pixtral") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
|
||||
Capability.TEXT_OUTPUT,
|
||||
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
// Mistral large latest:
|
||||
if (modelName.IndexOf("mistral-large-latest") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT,
|
||||
Capability.MULTIPLE_IMAGE_INPUT,
|
||||
Capability.TEXT_OUTPUT,
|
||||
|
||||
Capability.OPTIONAL_REASONING,
|
||||
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
// Mistral large:
|
||||
if (modelName.IndexOf("mistral-large-") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT,
|
||||
Capability.TEXT_OUTPUT,
|
||||
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
// Mistral medium latest:
|
||||
if (modelName.IndexOf("mistral-medium-latest") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT,
|
||||
Capability.MULTIPLE_IMAGE_INPUT,
|
||||
Capability.TEXT_OUTPUT,
|
||||
|
||||
Capability.OPTIONAL_REASONING,
|
||||
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
// Mistral medium:
|
||||
if (modelName.IndexOf("mistral-medium-") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT,
|
||||
Capability.TEXT_OUTPUT,
|
||||
|
||||
Capability.OPTIONAL_REASONING,
|
||||
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
// Mistral small latest:
|
||||
if (modelName.IndexOf("mistral-small-latest") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT,
|
||||
Capability.MULTIPLE_IMAGE_INPUT,
|
||||
Capability.TEXT_OUTPUT,
|
||||
|
||||
Capability.OPTIONAL_REASONING,
|
||||
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
// Mistral small:
|
||||
if (modelName.IndexOf("mistral-small-") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT,
|
||||
Capability.TEXT_OUTPUT,
|
||||
|
||||
Capability.OPTIONAL_REASONING,
|
||||
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
// Mistral saba:
|
||||
if (modelName.IndexOf("mistral-saba-") is not -1)
|
||||
return
|
||||
@ -106,8 +85,107 @@ public static partial class ProviderExtensions
|
||||
Capability.TEXT_OUTPUT,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
|
||||
//
|
||||
// The four families Mistral versions by release date. Ministral has to be matched before
|
||||
// the others, although its name does not contain "mistral" as a substring: keeping the
|
||||
// families together makes the block easier to read.
|
||||
//
|
||||
if (modelName.IndexOf("ministral") is not -1)
|
||||
return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MINISTRAL_LATEST), MINISTRAL_VISION_SINCE, MISTRAL_REASONING_NEVER);
|
||||
|
||||
if (modelName.IndexOf("mistral-large") is not -1)
|
||||
return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MISTRAL_LARGE_LATEST), MISTRAL_LARGE_VISION_SINCE, MISTRAL_LARGE_REASONING_SINCE);
|
||||
|
||||
if (modelName.IndexOf("mistral-medium") is not -1)
|
||||
return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MISTRAL_MEDIUM_LATEST), MISTRAL_MEDIUM_VISION_SINCE, MISTRAL_MEDIUM_REASONING_SINCE);
|
||||
|
||||
if (modelName.IndexOf("mistral-small") is not -1)
|
||||
return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MISTRAL_SMALL_LATEST), MISTRAL_SMALL_VISION_SINCE, MISTRAL_SMALL_REASONING_SINCE);
|
||||
|
||||
// Default:
|
||||
return GetModelCapabilitiesOpenSource(model);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the release date a Mistral model belongs to.
|
||||
/// </summary>
|
||||
/// <param name="modelName">The lowercase model name to inspect.</param>
|
||||
/// <param name="latestReleaseDate">The release the family's "latest" alias points to.</param>
|
||||
/// <returns>The release date as YYMM, or 0 when the name carries none.</returns>
|
||||
private static int GetMistralReleaseDate(ReadOnlySpan<char> modelName, int latestReleaseDate)
|
||||
{
|
||||
// The "latest" alias always points to the newest release of its family:
|
||||
if (modelName.IndexOf("-latest") is not -1)
|
||||
return latestReleaseDate;
|
||||
|
||||
foreach (var (versionName, releaseDate) in MISTRAL_VERSION_NAMES)
|
||||
if (modelName.IndexOf(versionName) is not -1)
|
||||
return releaseDate;
|
||||
|
||||
return ReadMistralReleaseDate(modelName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the four-digit release date out of a Mistral model name.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The block has to be exactly four digits long and has to read as a plausible year and month.
|
||||
/// Without that, the size of a model would be mistaken for its release: ministral-14b-2512
|
||||
/// must resolve to 2512 and not to anything the "14b" part could be read as.
|
||||
/// </remarks>
|
||||
/// <param name="modelName">The lowercase model name to inspect.</param>
|
||||
/// <returns>The release date as YYMM, or 0 when the name carries none.</returns>
|
||||
private static int ReadMistralReleaseDate(ReadOnlySpan<char> modelName)
|
||||
{
|
||||
for (var index = 0; index + 4 <= modelName.Length; index++)
|
||||
{
|
||||
// A digit next to the block means the block is longer than four digits:
|
||||
if (index > 0 && char.IsAsciiDigit(modelName[index - 1]))
|
||||
continue;
|
||||
|
||||
if (index + 4 < modelName.Length && char.IsAsciiDigit(modelName[index + 4]))
|
||||
continue;
|
||||
|
||||
var candidate = modelName.Slice(index, 4);
|
||||
if (!char.IsAsciiDigit(candidate[0]) || !char.IsAsciiDigit(candidate[1]) ||
|
||||
!char.IsAsciiDigit(candidate[2]) || !char.IsAsciiDigit(candidate[3]))
|
||||
continue;
|
||||
|
||||
var releaseDate = int.Parse(candidate);
|
||||
var year = releaseDate / 100;
|
||||
var month = releaseDate % 100;
|
||||
if (year < MISTRAL_FIRST_RELEASE_YEAR || month is < 1 or > 12)
|
||||
continue;
|
||||
|
||||
return releaseDate;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the capabilities of a Mistral model from its release date.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A model whose release date we cannot read gets neither image input nor reasoning. That is
|
||||
/// the safe direction: offering an ability the model does not have would fail the request,
|
||||
/// whereas a missing one can be added by hand through the capability overrides.
|
||||
/// </remarks>
|
||||
/// <param name="releaseDate">The release date of the model as YYMM, or 0 when unknown.</param>
|
||||
/// <param name="visionSince">The release from which this family accepts images.</param>
|
||||
/// <param name="reasoningSince">The release from which this family can reason.</param>
|
||||
/// <returns>The capabilities of the model.</returns>
|
||||
private static List<Capability> BuildMistralCapabilities(int releaseDate, int visionSince, int reasoningSince)
|
||||
{
|
||||
List<Capability> capabilities = [Capability.TEXT_INPUT, Capability.FUNCTION_CALLING, Capability.CHAT_COMPLETION_API, Capability.TEXT_OUTPUT];
|
||||
|
||||
if (releaseDate >= visionSince)
|
||||
capabilities.Add(Capability.MULTIPLE_IMAGE_INPUT);
|
||||
|
||||
if (releaseDate >= reasoningSince)
|
||||
capabilities.Add(Capability.OPTIONAL_REASONING);
|
||||
|
||||
return capabilities;
|
||||
}
|
||||
}
|
||||
@ -238,6 +238,14 @@ public static partial class ProviderExtensions
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
//
|
||||
// Ministral models. They need their own block because their names do not contain
|
||||
// "mistral" as a substring, so the block below never sees them. Only Ministral 3 accepts
|
||||
// images, the 2024 models are text only, which is why the release date decides here too:
|
||||
//
|
||||
if (modelName.IndexOf("ministral") is not -1)
|
||||
return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MINISTRAL_LATEST), MINISTRAL_VISION_SINCE, MISTRAL_REASONING_NEVER);
|
||||
|
||||
//
|
||||
// Mistral models:
|
||||
//
|
||||
@ -286,19 +294,6 @@ public static partial class ProviderExtensions
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
if (modelName.IndexOf("mistral-small-4") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT,
|
||||
Capability.MULTIPLE_IMAGE_INPUT,
|
||||
Capability.TEXT_OUTPUT,
|
||||
|
||||
Capability.OPTIONAL_REASONING,
|
||||
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
if (modelName.IndexOf("mistral-small-3") is not -1 ||
|
||||
modelName.IndexOf("mistral-small-4") is not -1)
|
||||
return
|
||||
@ -472,11 +467,18 @@ public static partial class ProviderExtensions
|
||||
//
|
||||
if (modelName.IndexOf("glm") is not -1)
|
||||
{
|
||||
//
|
||||
// Both version checks below accept a hyphen as the version separator as well:
|
||||
// Mistral serves these models as glm-5-2 and zai-glm-5-2, while everybody else
|
||||
// writes the version with a dot.
|
||||
//
|
||||
|
||||
// GLM 5.3 uses forced thinking: the reasoning effort can be lowered, but
|
||||
// reasoning cannot be turned off. This check must stay in front of the
|
||||
// vision check below, because quantized builds such as GLM-5.3-Flash-NVFP4
|
||||
// contain a "v" and would be misread as a vision model:
|
||||
if (modelName.IndexOf("glm-5.3") is not -1)
|
||||
if (modelName.IndexOf("glm-5.3") is not -1 ||
|
||||
modelName.IndexOf("glm-5-3") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
|
||||
@ -486,7 +488,8 @@ public static partial class ProviderExtensions
|
||||
Capability.CHAT_COMPLETION_API,
|
||||
];
|
||||
|
||||
if (modelName.IndexOf("glm-5.2") is not -1)
|
||||
if (modelName.IndexOf("glm-5.2") is not -1 ||
|
||||
modelName.IndexOf("glm-5-2") is not -1)
|
||||
return
|
||||
[
|
||||
Capability.TEXT_INPUT,
|
||||
|
||||
@ -531,7 +531,15 @@ public sealed class MediaTranscriptionService(
|
||||
fileName,
|
||||
operation.Id,
|
||||
providerResult.ErrorMessage);
|
||||
return MediaTranscriptionResult.Failed(TB("The transcription provider could not transcribe the media file."));
|
||||
|
||||
//
|
||||
// When the provider told us why it failed, the user gets to read it. Only that
|
||||
// message says whether to wait, to check the API key, or to ask the provider for
|
||||
// a format it can read:
|
||||
//
|
||||
return MediaTranscriptionResult.Failed(string.IsNullOrWhiteSpace(providerResult.ErrorMessage)
|
||||
? TB("The transcription provider could not transcribe the media file.")
|
||||
: providerResult.ErrorMessage);
|
||||
}
|
||||
|
||||
return MediaTranscriptionResult.Succeeded(providerResult.Text.Trim());
|
||||
|
||||
@ -5,11 +5,13 @@
|
||||
- Added the option for IT departments to enable assistant plugins they rolled out. Approving an assistant only stated that it is safe, so everybody still had to switch it on themselves. An approval can now also enable the assistant, either as a default which you may switch off again, or in a way your IT department keeps in place. The plugin page and the security card of the assistant tell you which of the two applies.
|
||||
- Added knowledge about the latest AI models. AI Studio now recognizes Qwen 3.8 Flash, GLM-5.3 Flash, Meta's Muse Glimmer, NVIDIA's Nemotron 3.5, Tencent's Hunyuan Hy3, Grok 4, Claude Opus 5 and Sonnet 5, and Gemini 3.6 and 3.7. It knows what each of them is capable of, so images, videos, tool usage, and reasoning are available right away instead of staying hidden.
|
||||
- Added the IONOS AI Model Hub as a provider for chats and embeddings. It runs open-source models in Germany, is subject to the GDPR, and IONOS states that your data is not used for training.
|
||||
- Added speech-to-text for Helmholtz Blablador and GroqCloud, and embeddings for GWDG SAIA. These providers offer these services now, so you can select them when you dictate a message or when you set up a data source.
|
||||
- Improved the safety of plugin symbols: AI Studio now shows the symbol of a plugin in isolation, so nothing inside a symbol can reach the rest of the app.
|
||||
- Improved how much memory AI Studio needs. Working with large documents used to grow the app to several gigabytes, and on macOS that memory was never handed back. AI Studio now stays at a fraction of that and returns memory to your system. This matters most on devices with little memory, such as a Raspberry Pi.
|
||||
- Improved the preview for large documents. It now shows you the beginning of your document instead of loading all of it, so the dialog opens right away. Your complete document still goes to the AI.
|
||||
- Improved how AI Studio deals with rare internal hiccups. When the app window reloads, or when it briefly loses the connection to its own user interface, work which was still running in the background is now ended properly instead of leaving errors behind.
|
||||
- Improved how IT departments roll plugins out. A configuration server can deliver any kind of plugin, not only configurations: one archive may carry assistant plugins and further types alongside a configuration, each in its own folder. The folder for staging a test behaves the same way, so a test can mirror the later rollout exactly. The Enterprise IT documentation describes the whole procedure.
|
||||
- Changed the model list of GroqCloud. Models which cannot be used for chatting, such as the speech and the safety models, no longer show up among the chat models. The speech models now appear where they belong, in the settings for speech-to-text.
|
||||
- Changed how plugins your organization rolled out are protected. They can no longer be deleted or edited in AI Studio, which already applied to sharing and replacing them. This also covers plugins staged for a test: such a test now ends by restarting AI Studio or by removing the staged files, instead of through the plugin page.
|
||||
- Fixed assistants created by the Assistant Builder being named after an internal placeholder, such as "Model decides", when you left the display name empty. The model now picks a fitting name instead.
|
||||
- Fixed AI Studio reading a document to the end even after you closed its preview. Closing the dialog now stops that work immediately.
|
||||
@ -20,4 +22,6 @@
|
||||
- Fixed the code editor being offered for assistant plugins your organization manages. Editing one would have withdrawn the approval of your IT department and demanded a fresh security audit.
|
||||
- Fixed assistant plugins your organization deployed alongside a configuration not being recognized as centrally managed unless they declared it themselves.
|
||||
- Fixed a misleading warning in the log when an organization deployed an archive that carries no configuration of its own.
|
||||
- Fixed AI Studio underrating what many models can do. Newer Claude, Gemini, Grok, DeepSeek, and Qwen models were missing abilities they actually have, such as image input or tool usage. This was most noticeable with OpenRouter, where nearly every model was affected. AI Studio now derives these abilities from the same source for all providers, so a model offers the same capabilities no matter which way you reach it.
|
||||
- Fixed AI Studio underrating what many models can do. Newer Claude, Gemini, Grok, DeepSeek, and Qwen models were missing abilities they actually have, such as image input or tool usage. This was most noticeable with OpenRouter, where nearly every model was affected. AI Studio now derives these abilities from the same source for all providers, so a model offers the same capabilities no matter which way you reach it.
|
||||
- Fixed the abilities shown for the models offered by Mistral. Mistral names its models after their release date, so almost every one of them was missing image input or reasoning: picking Mistral Large from the list gave you a different set of abilities than picking the very same model by its full name. AI Studio now goes by the release date and gets all of them right, including the Ministral models and the open-source models Mistral hosts, such as GLM.
|
||||
- Fixed AI Studio staying silent about why dictating or an embedding failed. When a provider explains what went wrong, you now get to read it instead of a general note that something did not work. If a provider cannot handle the audio format AI Studio sends, it says so and suggests contacting that provider.
|
||||
Loading…
Reference in New Issue
Block a user