From e21d734cdde2f3e81e9cac452c0ead070d1507f9 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 25 Jul 2024 15:17:26 +0200 Subject: [PATCH] Added Fireworks as provider --- .../Provider/Fireworks/ChatRequest.cs | 13 ++ .../Provider/Fireworks/Message.cs | 8 + .../Provider/Fireworks/ProviderFireworks.cs | 160 ++++++++++++++++++ .../Provider/Fireworks/ResponseStreamLine.cs | 24 +++ app/MindWork AI Studio/Provider/Providers.cs | 21 ++- .../wwwroot/changelog/v0.8.3.md | 5 +- 6 files changed, 222 insertions(+), 9 deletions(-) create mode 100644 app/MindWork AI Studio/Provider/Fireworks/ChatRequest.cs create mode 100644 app/MindWork AI Studio/Provider/Fireworks/Message.cs create mode 100644 app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs create mode 100644 app/MindWork AI Studio/Provider/Fireworks/ResponseStreamLine.cs diff --git a/app/MindWork AI Studio/Provider/Fireworks/ChatRequest.cs b/app/MindWork AI Studio/Provider/Fireworks/ChatRequest.cs new file mode 100644 index 00000000..a0e5a7ab --- /dev/null +++ b/app/MindWork AI Studio/Provider/Fireworks/ChatRequest.cs @@ -0,0 +1,13 @@ +namespace AIStudio.Provider.Fireworks; + +/// +/// The Fireworks chat request model. +/// +/// Which model to use for chat completion. +/// The chat messages. +/// Whether to stream the chat completion. +public readonly record struct ChatRequest( + string Model, + IList Messages, + bool Stream +); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Fireworks/Message.cs b/app/MindWork AI Studio/Provider/Fireworks/Message.cs new file mode 100644 index 00000000..2b0055bd --- /dev/null +++ b/app/MindWork AI Studio/Provider/Fireworks/Message.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Provider.Fireworks; + +/// +/// Chat message model. +/// +/// The text content of the message. +/// The role of the message. +public readonly record struct Message(string Content, string Role); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs new file mode 100644 index 00000000..2f6d1ea0 --- /dev/null +++ b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs @@ -0,0 +1,160 @@ +using System.Net.Http.Headers; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Settings; + +namespace AIStudio.Provider.Fireworks; + +public class ProviderFireworks() : BaseProvider("https://api.fireworks.ai/inference/v1/"), IProvider +{ + private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + }; + + #region Implementation of IProvider + + /// + public string Id => "Fireworks.ai"; + + /// + public string InstanceName { get; set; } = "Fireworks.ai"; + + /// + public async IAsyncEnumerable StreamChatCompletion(IJSRuntime jsRuntime, SettingsManager settings, Model chatModel, ChatThread chatThread, [EnumeratorCancellation] CancellationToken token = default) + { + // Get the API key: + var requestedSecret = await settings.GetAPIKey(jsRuntime, this); + if(!requestedSecret.Success) + yield break; + + // Prepare the system prompt: + var systemPrompt = new Message + { + Role = "system", + Content = chatThread.SystemPrompt, + }; + + // Prepare the Fireworks HTTP chat request: + var fireworksChatRequest = JsonSerializer.Serialize(new ChatRequest + { + Model = chatModel.Id, + + // Build the messages: + // - First of all the system prompt + // - Then none-empty user and AI messages + Messages = [systemPrompt, ..chatThread.Blocks.Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text)).Select(n => new Message + { + Role = n.Role switch + { + ChatRole.USER => "user", + ChatRole.AI => "assistant", + ChatRole.SYSTEM => "system", + + _ => "user", + }, + + Content = n.Content switch + { + ContentText text => text.Text, + _ => string.Empty, + } + }).ToList()], + + // Right now, we only support streaming completions: + Stream = true, + }, JSON_SERIALIZER_OPTIONS); + + // Build the HTTP post request: + var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); + + // Set the authorization header: + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", requestedSecret.Secret); + + // Set the content: + request.Content = new StringContent(fireworksChatRequest, Encoding.UTF8, "application/json"); + + // Send the request with the ResponseHeadersRead option. + // This allows us to read the stream as soon as the headers are received. + // This is important because we want to stream the responses. + var response = await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); + + // Open the response stream: + var fireworksStream = await response.Content.ReadAsStreamAsync(token); + + // Add a stream reader to read the stream, line by line: + var streamReader = new StreamReader(fireworksStream); + + // Read the stream, line by line: + while(!streamReader.EndOfStream) + { + // Check if the token is canceled: + if(token.IsCancellationRequested) + yield break; + + // Read the next line: + var line = await streamReader.ReadLineAsync(token); + + // Skip empty lines: + if(string.IsNullOrWhiteSpace(line)) + continue; + + // Skip lines that do not start with "data: ". Regard + // to the specification, we only want to read the data lines: + if(!line.StartsWith("data: ", StringComparison.InvariantCulture)) + continue; + + // Check if the line is the end of the stream: + if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture)) + yield break; + + ResponseStreamLine fireworksResponse; + try + { + // We know that the line starts with "data: ". Hence, we can + // skip the first 6 characters to get the JSON data after that. + var jsonData = line[6..]; + + // Deserialize the JSON data: + fireworksResponse = JsonSerializer.Deserialize(jsonData, JSON_SERIALIZER_OPTIONS); + } + catch + { + // Skip invalid JSON data: + continue; + } + + // Skip empty responses: + if(fireworksResponse == default || fireworksResponse.Choices.Count == 0) + continue; + + // Yield the response: + yield return fireworksResponse.Choices[0].Delta.Content; + } + } + + #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + /// + public async IAsyncEnumerable StreamImageCompletion(IJSRuntime jsRuntime, SettingsManager settings, Model imageModel, string promptPositive, string promptNegative = FilterOperator.String.Empty, ImageURL referenceImageURL = default, [EnumeratorCancellation] CancellationToken token = default) + { + yield break; + } + #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously + + /// + public Task> GetTextModels(IJSRuntime jsRuntime, SettingsManager settings, string? apiKeyProvisional = null, CancellationToken token = default) + { + return Task.FromResult(Enumerable.Empty()); + } + + /// + public Task> GetImageModels(IJSRuntime jsRuntime, SettingsManager settings, string? apiKeyProvisional = null, CancellationToken token = default) + { + return Task.FromResult(Enumerable.Empty()); + } + + #endregion +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Fireworks/ResponseStreamLine.cs b/app/MindWork AI Studio/Provider/Fireworks/ResponseStreamLine.cs new file mode 100644 index 00000000..c4d54e01 --- /dev/null +++ b/app/MindWork AI Studio/Provider/Fireworks/ResponseStreamLine.cs @@ -0,0 +1,24 @@ +namespace AIStudio.Provider.Fireworks; + +/// +/// Data model for a line in the response stream, for streaming completions. +/// +/// The id of the response. +/// The object describing the response. +/// The timestamp of the response. +/// The model used for the response. +/// The choices made by the AI. +public readonly record struct ResponseStreamLine(string Id, string Object, uint Created, string Model, IList Choices); + +/// +/// Data model for a choice made by the AI. +/// +/// The index of the choice. +/// The delta text of the choice. +public readonly record struct Choice(int Index, Delta Delta); + +/// +/// The delta text of a choice. +/// +/// The content of the delta text. +public readonly record struct Delta(string Content); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Providers.cs b/app/MindWork AI Studio/Provider/Providers.cs index 47e7ed93..530d0237 100644 --- a/app/MindWork AI Studio/Provider/Providers.cs +++ b/app/MindWork AI Studio/Provider/Providers.cs @@ -1,4 +1,5 @@ using AIStudio.Provider.Anthropic; +using AIStudio.Provider.Fireworks; using AIStudio.Provider.Mistral; using AIStudio.Provider.OpenAI; using AIStudio.Provider.SelfHosted; @@ -10,13 +11,15 @@ namespace AIStudio.Provider; /// public enum Providers { - NONE, + NONE = 0, - OPEN_AI, - ANTHROPIC, - MISTRAL, + OPEN_AI = 1, + ANTHROPIC = 2, + MISTRAL = 3, - SELF_HOSTED, + FIREWORKS = 5, + + SELF_HOSTED = 4, } /// @@ -37,6 +40,8 @@ public static class ExtensionsProvider Providers.ANTHROPIC => "Anthropic", Providers.MISTRAL => "Mistral", + Providers.FIREWORKS => "Fireworks.ai", + Providers.SELF_HOSTED => "Self-hosted", _ => "Unknown", @@ -56,9 +61,11 @@ public static class ExtensionsProvider Providers.OPEN_AI => new ProviderOpenAI { InstanceName = providerSettings.InstanceName }, Providers.ANTHROPIC => new ProviderAnthropic { InstanceName = providerSettings.InstanceName }, Providers.MISTRAL => new ProviderMistral { InstanceName = providerSettings.InstanceName }, - + + Providers.FIREWORKS => new ProviderFireworks { InstanceName = providerSettings.InstanceName }, + Providers.SELF_HOSTED => new ProviderSelfHosted(providerSettings) { InstanceName = providerSettings.InstanceName }, - + _ => new NoProvider(), }; } diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.8.3.md b/app/MindWork AI Studio/wwwroot/changelog/v0.8.3.md index 0df1aa40..0c9f98a0 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v0.8.3.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v0.8.3.md @@ -1,5 +1,6 @@ # v0.8.3 (WIP) -- Migrated UI framework from MudBlazor v6.x.x to v7.x.x - Added an option to configure the behavior of the navigation bar in the settings +- Added support for Fireworks.ai as provider, where you can use e.g., the llama 3.1 405b model - Improved the handling of self-hosted provider hostnames -- Improved the configured provider table: long model names are now truncated \ No newline at end of file +- Improved the configured provider table: long model names are now truncated +- Migrated UI framework from MudBlazor v6.x.x to v7.x.x \ No newline at end of file