From ac6748e9eb56265aac8061b5d8f084cce5621727 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 3 Jul 2024 20:24:39 +0200 Subject: [PATCH] Implemented self-hosted and local provider --- .../Provider/SelfHosted/ChatRequest.cs | 16 ++ .../Provider/SelfHosted/Message.cs | 8 + .../Provider/SelfHosted/ModelsResponse.cs | 5 + .../Provider/SelfHosted/ProviderSelfHosted.cs | 162 ++++++++++++++++++ 4 files changed, 191 insertions(+) create mode 100644 app/MindWork AI Studio/Provider/SelfHosted/ChatRequest.cs create mode 100644 app/MindWork AI Studio/Provider/SelfHosted/Message.cs create mode 100644 app/MindWork AI Studio/Provider/SelfHosted/ModelsResponse.cs create mode 100644 app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ChatRequest.cs b/app/MindWork AI Studio/Provider/SelfHosted/ChatRequest.cs new file mode 100644 index 00000000..74b3f089 --- /dev/null +++ b/app/MindWork AI Studio/Provider/SelfHosted/ChatRequest.cs @@ -0,0 +1,16 @@ +namespace AIStudio.Provider.SelfHosted; + +/// +/// The chat request model. +/// +/// Which model to use for chat completion. +/// The chat messages. +/// Whether to stream the chat completion. +/// The maximum number of tokens to generate. +public readonly record struct ChatRequest( + string Model, + IList Messages, + bool Stream, + + int MaxTokens +); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/SelfHosted/Message.cs b/app/MindWork AI Studio/Provider/SelfHosted/Message.cs new file mode 100644 index 00000000..e4ecc70a --- /dev/null +++ b/app/MindWork AI Studio/Provider/SelfHosted/Message.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Provider.SelfHosted; + +/// +/// Chat message model. +/// +/// The text content of the message. +/// The role of the message. +public readonly record struct Message(string Content, string Role); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ModelsResponse.cs b/app/MindWork AI Studio/Provider/SelfHosted/ModelsResponse.cs new file mode 100644 index 00000000..8ea8fb57 --- /dev/null +++ b/app/MindWork AI Studio/Provider/SelfHosted/ModelsResponse.cs @@ -0,0 +1,5 @@ +namespace AIStudio.Provider.SelfHosted; + +public readonly record struct ModelsResponse(string Object, Model[] Data); + +public readonly record struct Model(string Id, string Object, string OwnedBy); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs new file mode 100644 index 00000000..82a458e3 --- /dev/null +++ b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs @@ -0,0 +1,162 @@ +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Provider.OpenAI; +using AIStudio.Settings; + +namespace AIStudio.Provider.SelfHosted; + +public sealed class ProviderSelfHosted(string hostname) : BaseProvider($"{hostname}/v1/"), IProvider +{ + private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + }; + + #region Implementation of IProvider + + public string Id => "Self-hosted"; + + public string InstanceName { get; set; } = "Self-hosted"; + + public async IAsyncEnumerable StreamChatCompletion(IJSRuntime jsRuntime, SettingsManager settings, Provider.Model chatModel, ChatThread chatThread, [EnumeratorCancellation] CancellationToken token = default) + { + // Prepare the system prompt: + var systemPrompt = new Message + { + Role = "system", + Content = chatThread.SystemPrompt, + }; + + // Prepare the OpenAI HTTP chat request: + var providerChatRequest = JsonSerializer.Serialize(new ChatRequest + { + Model = (await this.GetTextModels(jsRuntime, settings, token: token)).First().Id, + + // Build the messages: + // - First of all the system prompt + // - Then none-empty user and AI messages + Messages = [systemPrompt, ..chatThread.Blocks.Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text)).Select(n => new Message + { + Role = n.Role switch + { + ChatRole.USER => "user", + ChatRole.AI => "assistant", + ChatRole.SYSTEM => "system", + + _ => "user", + }, + + Content = n.Content switch + { + ContentText text => text.Text, + _ => string.Empty, + } + }).ToList()], + + // Right now, we only support streaming completions: + Stream = true, + MaxTokens = -1, + }, JSON_SERIALIZER_OPTIONS); + + // Build the HTTP post request: + var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); + + // Set the content: + request.Content = new StringContent(providerChatRequest, Encoding.UTF8, "application/json"); + + // Send the request with the ResponseHeadersRead option. + // This allows us to read the stream as soon as the headers are received. + // This is important because we want to stream the responses. + var response = await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); + + // Open the response stream: + var providerStream = await response.Content.ReadAsStreamAsync(token); + + // Add a stream reader to read the stream, line by line: + var streamReader = new StreamReader(providerStream); + + // Read the stream, line by line: + while(!streamReader.EndOfStream) + { + // Check if the token is cancelled: + if(token.IsCancellationRequested) + yield break; + + // Read the next line: + var line = await streamReader.ReadLineAsync(token); + + // Skip empty lines: + if(string.IsNullOrWhiteSpace(line)) + continue; + + // Skip lines that do not start with "data: ". Regard + // to the specification, we only want to read the data lines: + if(!line.StartsWith("data: ", StringComparison.InvariantCulture)) + continue; + + // Check if the line is the end of the stream: + if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture)) + yield break; + + ResponseStreamLine providerResponse; + try + { + // We know that the line starts with "data: ". Hence, we can + // skip the first 6 characters to get the JSON data after that. + var jsonData = line[6..]; + + // Deserialize the JSON data: + providerResponse = JsonSerializer.Deserialize(jsonData, JSON_SERIALIZER_OPTIONS); + } + catch + { + // Skip invalid JSON data: + continue; + } + + // Skip empty responses: + if(providerResponse == default || providerResponse.Choices.Count == 0) + continue; + + // Yield the response: + yield return providerResponse.Choices[0].Delta.Content; + } + } + + #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + /// + public async IAsyncEnumerable StreamImageCompletion(IJSRuntime jsRuntime, SettingsManager settings, Provider.Model imageModel, string promptPositive, string promptNegative = FilterOperator.String.Empty, ImageURL referenceImageURL = default, [EnumeratorCancellation] CancellationToken token = default) + { + yield break; + } + #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously + + + public async Task> GetTextModels(IJSRuntime jsRuntime, SettingsManager settings, string? apiKeyProvisional = null, CancellationToken token = default) + { + var request = new HttpRequestMessage(HttpMethod.Get, "models"); + var response = await this.httpClient.SendAsync(request, token); + if(!response.IsSuccessStatusCode) + return []; + + var modelResponse = await response.Content.ReadFromJsonAsync(token); + if (modelResponse.Data.Length > 1) + Console.WriteLine("Warning: multiple models found; using the first one."); + + var firstModel = modelResponse.Data.First(); + return [ new Provider.Model(firstModel.Id) ]; + } + + #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + /// + public Task> GetImageModels(IJSRuntime jsRuntime, SettingsManager settings, string? apiKeyProvisional = null, CancellationToken token = default) + { + return Task.FromResult(Enumerable.Empty()); + } + #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously + + #endregion +} \ No newline at end of file