Add support for ollama (#34)

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2024-07-16 10:28:13 +02:00
1 parent 179f350af5
commit 063fd0fcfa
19 files changed
+271 -75

No files matched your search

@@ -0,0 +1,42 @@
namespace AIStudio.Provider.SelfHosted;
public enum Host
{
NONE,
LM_STUDIO,
LLAMACPP,
OLLAMA,
}
public static class HostExtensions
{
public static string Name(this Host host) => host switch
{
Host.NONE => "None",
Host.LM_STUDIO => "LM Studio",
Host.LLAMACPP => "llama.cpp",
Host.OLLAMA => "ollama",
_ => "Unknown",
};
public static string BaseURL(this Host host) => host switch
{
Host.LM_STUDIO => "/v1/",
Host.LLAMACPP => "/v1/",
Host.OLLAMA => "/v1/",
_ => "/v1/",
};
public static string ChatURL(this Host host) => host switch
{
Host.LM_STUDIO => "chat/completions",
Host.LLAMACPP => "chat/completions",
Host.OLLAMA => "chat/completions",
_ => "chat/completions",
};
}
@@ -8,7 +8,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.SelfHosted;
public sealed class ProviderSelfHosted(string hostname) : BaseProvider($"{hostname}/v1/"), IProvider
public sealed class ProviderSelfHosted(Settings.Provider provider) : BaseProvider($"{provider.Hostname}{provider.Host.BaseURL()}"), IProvider
{
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
{
@@ -33,7 +33,7 @@ public sealed class ProviderSelfHosted(string hostname) : BaseProvider($"{hostna
// Prepare the OpenAI HTTP chat request:
var providerChatRequest = JsonSerializer.Serialize(new ChatRequest
{
Model = (await this.GetTextModels(jsRuntime, settings, token: token)).First().Id,
Model = chatModel.Id,
// Build the messages:
// - First of all the system prompt
@@ -62,7 +62,7 @@ public sealed class ProviderSelfHosted(string hostname) : BaseProvider($"{hostna
}, JSON_SERIALIZER_OPTIONS);
// Build the HTTP post request:
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
var request = new HttpRequestMessage(HttpMethod.Post, provider.Host.ChatURL());
// Set the content:
request.Content = new StringContent(providerChatRequest, Encoding.UTF8, "application/json");
@@ -137,17 +137,33 @@ public sealed class ProviderSelfHosted(string hostname) : BaseProvider($"{hostna
public async Task<IEnumerable<Provider.Model>> GetTextModels(IJSRuntime jsRuntime, SettingsManager settings, string? apiKeyProvisional = null, CancellationToken token = default)
{
var request = new HttpRequestMessage(HttpMethod.Get, "models");
var response = await this.httpClient.SendAsync(request, token);
if(!response.IsSuccessStatusCode)
return [];
try
{
switch (provider.Host)
{
case Host.LLAMACPP:
// Right now, llama.cpp only supports one model.
// There is no API to list the model(s).
return [ new Provider.Model("as configured by llama.cpp") ];
case Host.LM_STUDIO:
case Host.OLLAMA:
var lmStudioRequest = new HttpRequestMessage(HttpMethod.Get, "models");
var lmStudioResponse = await this.httpClient.SendAsync(lmStudioRequest, token);
if(!lmStudioResponse.IsSuccessStatusCode)
return [];
var modelResponse = await response.Content.ReadFromJsonAsync<ModelsResponse>(token);
if (modelResponse.Data.Length > 1)
Console.WriteLine("Warning: multiple models found; using the first one.");
var firstModel = modelResponse.Data.First();
return [ new Provider.Model(firstModel.Id) ];
var lmStudioModelResponse = await lmStudioResponse.Content.ReadFromJsonAsync<ModelsResponse>(token);
return lmStudioModelResponse.Data.Select(n => new Provider.Model(n.Id));
}
return [];
}
catch(Exception e)
{
Console.WriteLine($"Failed to load text models from self-hosted provider: {e.Message}");
return [];
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously