mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 18:38:00 +00:00
Read the context window from four more model lists
This commit is contained in:
16 files changed
+415
-28
No files matched your search
@@ -5,4 +5,30 @@ namespace AIStudio.Provider.HuggingFace;
|
||||
/// </summary>
|
||||
/// <param name="Id">The ID of the model, written as "org/model".</param>
|
||||
/// <param name="Providers">The inference providers serving this model.</param>
|
||||
public readonly record struct HFModel(string Id, IList<HFModelProvider>? Providers);
|
||||
public readonly record struct HFModel(string Id, IList<HFModelProvider>? Providers)
|
||||
{
|
||||
/// <summary>
|
||||
/// The window this model has when it is reached the way this user set things up.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A window belongs to an inference provider here, not to the model: the same weights run
|
||||
/// behind several of them, each configured by somebody else. Where the user named one, its
|
||||
/// number is the answer. Where they let the router choose, the smallest window among the
|
||||
/// providers currently serving the model is -- nobody knows which one the router will take, and
|
||||
/// a number promising more than the chosen provider delivers would walk a conversation into an
|
||||
/// error the user could not see coming.
|
||||
/// </remarks>
|
||||
/// <param name="providerSlug">The inference provider the user chose, or empty when the router chooses.</param>
|
||||
/// <returns>The window in tokens, or null where nobody stated one.</returns>
|
||||
public int? ContextWindowTokens(string providerSlug)
|
||||
{
|
||||
if (this.Providers is null)
|
||||
return null;
|
||||
|
||||
var serving = this.Providers.Where(provider => provider.IsLive);
|
||||
if (!string.IsNullOrEmpty(providerSlug))
|
||||
serving = serving.Where(provider => string.Equals(provider.Provider, providerSlug, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
return serving.Where(provider => provider.ContextWindowTokens is > 0).Min(provider => provider.ContextWindowTokens);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace AIStudio.Provider.HuggingFace;
|
||||
|
||||
/// <summary>
|
||||
@@ -5,4 +7,13 @@ namespace AIStudio.Provider.HuggingFace;
|
||||
/// </summary>
|
||||
/// <param name="Provider">The slug of the inference provider, e.g. "novita".</param>
|
||||
/// <param name="Status">Whether the provider currently serves the model. Known value: "live".</param>
|
||||
public readonly record struct HFModelProvider(string Provider, string Status);
|
||||
/// <param name="ContextWindowTokens">How much this provider reads and writes in one conversation, in tokens.</param>
|
||||
public readonly record struct HFModelProvider(string Provider, string Status, [property: JsonPropertyName("context_length")] int? ContextWindowTokens)
|
||||
{
|
||||
private const string LIVE = "live";
|
||||
|
||||
/// <summary>
|
||||
/// Whether this provider serves the model right now.
|
||||
/// </summary>
|
||||
public bool IsLive => string.Equals(this.Status, LIVE, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Models.Live;
|
||||
using AIStudio.Provider.OpenAI;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
@@ -221,7 +222,23 @@ public sealed class ProviderHuggingFace : BaseProvider
|
||||
/// <inheritdoc />
|
||||
public override Task<ModelLoadResult> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
{
|
||||
return this.LoadModelsResponse<ModelsResponse>(SecretStoreType.LLM_PROVIDER, "models", this.SelectChatModels, apiKeyProvisional, token: token);
|
||||
return this.LoadModelsResponse<ModelsResponse>(SecretStoreType.LLM_PROVIDER, "models", this.SelectChatModels, apiKeyProvisional, listingFactory: this.ListingsOf, token: token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the router stated about the models it knows.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every model the router reports, not only the ones offered for chatting below: which models
|
||||
/// are offered depends on the chosen inference provider, while a window belongs to whoever is
|
||||
/// configured here, and both questions are asked of the same list.
|
||||
/// </remarks>
|
||||
/// <param name="response">The response of the model endpoint.</param>
|
||||
/// <returns>One listing per model, which says nothing for the models nobody stated a window for.</returns>
|
||||
private IEnumerable<ModelListing> ListingsOf(ModelsResponse response)
|
||||
{
|
||||
var providerSlug = this.hfProvider.EndpointsId();
|
||||
return response.Data.Select(hfModel => ModelListing.For(hfModel.Id, hfModel.ContextWindowTokens(providerSlug)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -253,8 +270,8 @@ public sealed class ProviderHuggingFace : BaseProvider
|
||||
return false;
|
||||
|
||||
return hfModel.Providers.Any(provider =>
|
||||
string.Equals(provider.Provider, providerSlug, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(provider.Status, "live", StringComparison.OrdinalIgnoreCase));
|
||||
provider.IsLive &&
|
||||
string.Equals(provider.Provider, providerSlug, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
Reference in new issue
Block a user