Added Fireworks provider (#43)

This commit is contained in:
Thorsten Sommer 2024-07-25 15:29:44 +02:00 committed by GitHub
parent 4f5815272a
commit 9267ef865d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 335 additions and 38 deletions

View File

@ -5,7 +5,7 @@ MindWork AI Studio is a desktop application available for macOS, Windows, and Li
**Key advantages:**
- **Free of charge**: The app is free to use, both for personal and commercial purposes.
- **Independence**: Users are not tied to any single provider. Instead, they can choose the provider that best suits their needs. Right now, we support OpenAI (GPT4o etc.), Mistral, Anthropic (Claude), and self-hosted models using [llama.cpp](https://github.com/ggerganov/llama.cpp), [ollama](https://github.com/ollama/ollama), or [LM Studio](https://lmstudio.ai/). Support for Google Gemini, [Replicate](https://replicate.com/), and [Fireworks](https://fireworks.ai/) is planned.
- **Independence**: You are not tied to any single provider. Instead, you can choose the provider that best suits their needs. Right now, we support OpenAI (GPT4o etc.), Mistral, Anthropic (Claude), and self-hosted models using [llama.cpp](https://github.com/ggerganov/llama.cpp), [ollama](https://github.com/ollama/ollama), [LM Studio](https://lmstudio.ai/), or [Fireworks](https://fireworks.ai/). Support for Google Gemini, and [Replicate](https://replicate.com/) is planned.
- **Unrestricted usage**: Unlike services like ChatGPT, which impose limits after intensive use, MindWork AI Studio offers unlimited usage through the providers API.
- **Cost-effective**: You only pay for what you use, which can be cheaper than monthly subscription services like ChatGPT Plus, especially if used infrequently. But beware, here be dragons: For extremely intensive usage, the API costs can be significantly higher. Unfortunately, providers currently do not offer a way to display current costs in the app. Therefore, check your account with the respective provider to see how your costs are developing. When available, use prepaid and set a cost limit.
- **Privacy**: The data entered into the app is not used for training by the providers since we are using the provider's API.

View File

@ -13,6 +13,7 @@ public partial class Changelog
public static readonly Log[] LOGS =
[
new (165, "v0.8.3, build 165 (2024-07-25 13:25 UTC)", "v0.8.3.md"),
new (164, "v0.8.2, build 164 (2024-07-16 18:03 UTC)", "v0.8.2.md"),
new (163, "v0.8.1, build 163 (2024-07-16 08:32 UTC)", "v0.8.1.md"),
new (162, "v0.8.0, build 162 (2024-07-14 19:39 UTC)", "v0.8.0.md"),

View File

@ -31,7 +31,7 @@ public partial class Home : ComponentBase
private static readonly TextItem[] ITEMS_ADVANTAGES =
[
new TextItem("Free of charge", "The app is free to use, both for personal and commercial purposes."),
new TextItem("Independence", "You are not tied to any single provider. Instead, you might choose the provider that best suits your needs. Right now, we support OpenAI (GPT4o etc.), Mistral, Anthropic (Claude), and self-hosted models using llama.cpp, ollama, or LM Studio. Support for Google Gemini, Replicate, and Fireworks is planned."),
new TextItem("Independence", "You are not tied to any single provider. Instead, you might choose the provider that best suits your needs. Right now, we support OpenAI (GPT4o etc.), Mistral, Anthropic (Claude), and self-hosted models using llama.cpp, ollama, LM Studio, or Fireworks. Support for Google Gemini and Replicate is planned."),
new TextItem("Unrestricted usage", "Unlike services like ChatGPT, which impose limits after intensive use, MindWork AI Studio offers unlimited usage through the providers API."),
new TextItem("Cost-effective", "You only pay for what you use, which can be cheaper than monthly subscription services like ChatGPT Plus, especially if used infrequently. But beware, here be dragons: For extremely intensive usage, the API costs can be significantly higher. Unfortunately, providers currently do not offer a way to display current costs in the app. Therefore, check your account with the respective provider to see how your costs are developing. When available, use prepaid and set a cost limit."),
new TextItem("Privacy", "The data entered into the app is not used for training by the providers since we are using the provider's API."),

View File

@ -41,7 +41,7 @@
}
</MudTd>
<MudTd Style="text-align: left;">
<MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.OpenInBrowser" Class="ma-2" Href="@this.GetProviderDashboardURL(context.UsedProvider)" Target="_blank" Disabled="@(context.UsedProvider is Providers.NONE or Providers.SELF_HOSTED)">
<MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.OpenInBrowser" Class="ma-2" Href="@this.GetProviderDashboardURL(context.UsedProvider)" Target="_blank" Disabled="@(!this.HasDashboard(context.UsedProvider))">
Open Dashboard
</MudButton>
<MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Edit" Class="ma-2" OnClick="() => this.EditProvider(context)">

View File

@ -102,11 +102,22 @@ public partial class Settings : ComponentBase
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
}
private bool HasDashboard(Providers provider) => provider switch
{
Providers.OPEN_AI => true,
Providers.MISTRAL => true,
Providers.ANTHROPIC => true,
Providers.FIREWORKS => true,
_ => false,
};
private string GetProviderDashboardURL(Providers provider) => provider switch
{
Providers.OPEN_AI => "https://platform.openai.com/usage",
Providers.MISTRAL => "https://console.mistral.ai/usage/",
Providers.ANTHROPIC => "https://console.anthropic.com/settings/plans",
Providers.FIREWORKS => "https://fireworks.ai/account/billing",
_ => string.Empty,
};

View File

@ -0,0 +1,13 @@
namespace AIStudio.Provider.Fireworks;
/// <summary>
/// The Fireworks chat request model.
/// </summary>
/// <param name="Model">Which model to use for chat completion.</param>
/// <param name="Messages">The chat messages.</param>
/// <param name="Stream">Whether to stream the chat completion.</param>
public readonly record struct ChatRequest(
string Model,
IList<Message> Messages,
bool Stream
);

View File

@ -0,0 +1,8 @@
namespace AIStudio.Provider.Fireworks;
/// <summary>
/// Chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>
public readonly record struct Message(string Content, string Role);

View File

@ -0,0 +1,160 @@
using System.Net.Http.Headers;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Settings;
namespace AIStudio.Provider.Fireworks;
public class ProviderFireworks() : BaseProvider("https://api.fireworks.ai/inference/v1/"), IProvider
{
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
};
#region Implementation of IProvider
/// <inheritdoc />
public string Id => "Fireworks.ai";
/// <inheritdoc />
public string InstanceName { get; set; } = "Fireworks.ai";
/// <inheritdoc />
public async IAsyncEnumerable<string> StreamChatCompletion(IJSRuntime jsRuntime, SettingsManager settings, Model chatModel, ChatThread chatThread, [EnumeratorCancellation] CancellationToken token = default)
{
// Get the API key:
var requestedSecret = await settings.GetAPIKey(jsRuntime, this);
if(!requestedSecret.Success)
yield break;
// Prepare the system prompt:
var systemPrompt = new Message
{
Role = "system",
Content = chatThread.SystemPrompt,
};
// Prepare the Fireworks HTTP chat request:
var fireworksChatRequest = JsonSerializer.Serialize(new ChatRequest
{
Model = chatModel.Id,
// Build the messages:
// - First of all the system prompt
// - Then none-empty user and AI messages
Messages = [systemPrompt, ..chatThread.Blocks.Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text)).Select(n => new Message
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => text.Text,
_ => string.Empty,
}
}).ToList()],
// Right now, we only support streaming completions:
Stream = true,
}, JSON_SERIALIZER_OPTIONS);
// Build the HTTP post request:
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
// Set the authorization header:
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", requestedSecret.Secret);
// Set the content:
request.Content = new StringContent(fireworksChatRequest, Encoding.UTF8, "application/json");
// Send the request with the ResponseHeadersRead option.
// This allows us to read the stream as soon as the headers are received.
// This is important because we want to stream the responses.
var response = await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
// Open the response stream:
var fireworksStream = await response.Content.ReadAsStreamAsync(token);
// Add a stream reader to read the stream, line by line:
var streamReader = new StreamReader(fireworksStream);
// Read the stream, line by line:
while(!streamReader.EndOfStream)
{
// Check if the token is canceled:
if(token.IsCancellationRequested)
yield break;
// Read the next line:
var line = await streamReader.ReadLineAsync(token);
// Skip empty lines:
if(string.IsNullOrWhiteSpace(line))
continue;
// Skip lines that do not start with "data: ". Regard
// to the specification, we only want to read the data lines:
if(!line.StartsWith("data: ", StringComparison.InvariantCulture))
continue;
// Check if the line is the end of the stream:
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
yield break;
ResponseStreamLine fireworksResponse;
try
{
// We know that the line starts with "data: ". Hence, we can
// skip the first 6 characters to get the JSON data after that.
var jsonData = line[6..];
// Deserialize the JSON data:
fireworksResponse = JsonSerializer.Deserialize<ResponseStreamLine>(jsonData, JSON_SERIALIZER_OPTIONS);
}
catch
{
// Skip invalid JSON data:
continue;
}
// Skip empty responses:
if(fireworksResponse == default || fireworksResponse.Choices.Count == 0)
continue;
// Yield the response:
yield return fireworksResponse.Choices[0].Delta.Content;
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
/// <inheritdoc />
public async IAsyncEnumerable<ImageURL> StreamImageCompletion(IJSRuntime jsRuntime, SettingsManager settings, Model imageModel, string promptPositive, string promptNegative = FilterOperator.String.Empty, ImageURL referenceImageURL = default, [EnumeratorCancellation] CancellationToken token = default)
{
yield break;
}
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
/// <inheritdoc />
public Task<IEnumerable<Model>> GetTextModels(IJSRuntime jsRuntime, SettingsManager settings, string? apiKeyProvisional = null, CancellationToken token = default)
{
return Task.FromResult(Enumerable.Empty<Model>());
}
/// <inheritdoc />
public Task<IEnumerable<Model>> GetImageModels(IJSRuntime jsRuntime, SettingsManager settings, string? apiKeyProvisional = null, CancellationToken token = default)
{
return Task.FromResult(Enumerable.Empty<Model>());
}
#endregion
}

View File

@ -0,0 +1,24 @@
namespace AIStudio.Provider.Fireworks;
/// <summary>
/// Data model for a line in the response stream, for streaming completions.
/// </summary>
/// <param name="Id">The id of the response.</param>
/// <param name="Object">The object describing the response.</param>
/// <param name="Created">The timestamp of the response.</param>
/// <param name="Model">The model used for the response.</param>
/// <param name="Choices">The choices made by the AI.</param>
public readonly record struct ResponseStreamLine(string Id, string Object, uint Created, string Model, IList<Choice> Choices);
/// <summary>
/// Data model for a choice made by the AI.
/// </summary>
/// <param name="Index">The index of the choice.</param>
/// <param name="Delta">The delta text of the choice.</param>
public readonly record struct Choice(int Index, Delta Delta);
/// <summary>
/// The delta text of a choice.
/// </summary>
/// <param name="Content">The content of the delta text.</param>
public readonly record struct Delta(string Content);

View File

@ -97,7 +97,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
// Read the stream, line by line:
while(!streamReader.EndOfStream)
{
// Check if the token is cancelled:
// Check if the token is canceled:
if(token.IsCancellationRequested)
yield break;

View File

@ -1,4 +1,5 @@
using AIStudio.Provider.Anthropic;
using AIStudio.Provider.Fireworks;
using AIStudio.Provider.Mistral;
using AIStudio.Provider.OpenAI;
using AIStudio.Provider.SelfHosted;
@ -10,13 +11,15 @@ namespace AIStudio.Provider;
/// </summary>
public enum Providers
{
NONE,
NONE = 0,
OPEN_AI,
ANTHROPIC,
MISTRAL,
OPEN_AI = 1,
ANTHROPIC = 2,
MISTRAL = 3,
SELF_HOSTED,
FIREWORKS = 5,
SELF_HOSTED = 4,
}
/// <summary>
@ -37,6 +40,8 @@ public static class ExtensionsProvider
Providers.ANTHROPIC => "Anthropic",
Providers.MISTRAL => "Mistral",
Providers.FIREWORKS => "Fireworks.ai",
Providers.SELF_HOSTED => "Self-hosted",
_ => "Unknown",
@ -56,9 +61,11 @@ public static class ExtensionsProvider
Providers.OPEN_AI => new ProviderOpenAI { InstanceName = providerSettings.InstanceName },
Providers.ANTHROPIC => new ProviderAnthropic { InstanceName = providerSettings.InstanceName },
Providers.MISTRAL => new ProviderMistral { InstanceName = providerSettings.InstanceName },
Providers.FIREWORKS => new ProviderFireworks { InstanceName = providerSettings.InstanceName },
Providers.SELF_HOSTED => new ProviderSelfHosted(providerSettings) { InstanceName = providerSettings.InstanceName },
_ => new NoProvider(),
};
}

View File

@ -13,7 +13,7 @@
<MudSelectItem Value="@provider">@provider</MudSelectItem>
}
</MudSelect>
<MudButton Disabled="@this.IsSelfHostedOrNone" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.OpenInBrowser" Href="@this.GetProviderCreationURL()" Target="_blank">Create account</MudButton>
<MudButton Disabled="@(!this.ShowRegisterButton)" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.OpenInBrowser" Href="@this.GetProviderCreationURL()" Target="_blank">Create account</MudButton>
</MudStack>
@* ReSharper disable once CSharpWarnings::CS8974 *@
@ -21,7 +21,7 @@
T="string"
@bind-Text="@this.dataAPIKey"
Label="API Key"
Disabled="@this.IsSelfHostedOrNone"
Disabled="@(!this.NeedAPIKey)"
Class="mb-3"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.VpnKey"
@ -34,7 +34,7 @@
T="string"
@bind-Text="@this.DataHostname"
Label="Hostname"
Disabled="@this.IsCloudProvider"
Disabled="@(!this.NeedHostname)"
Class="mb-3"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.Dns"
@ -43,7 +43,7 @@
UserAttributes="@SPELLCHECK_ATTRIBUTES"
/>
<MudSelect Disabled="@this.IsCloudProvider" @bind-Value="@this.DataHost" Label="Host" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.ValidatingHost">
<MudSelect Disabled="@(!this.NeedHost)" @bind-Value="@this.DataHost" Label="Host" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.ValidatingHost">
@foreach (Host host in Enum.GetValues(typeof(Host)))
{
<MudSelectItem Value="@host">@host.Name()</MudSelectItem>
@ -51,13 +51,31 @@
</MudSelect>
<MudStack Row="@true" AlignItems="AlignItems.Center">
<MudButton Disabled="@(!this.CanLoadModels())" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.Refresh" OnClick="this.ReloadModels">Load</MudButton>
<MudSelect Disabled="@this.IsNoneProvider" @bind-Value="@this.DataModel" Label="Model" Class="mb-3" OpenIcon="@Icons.Material.Filled.FaceRetouchingNatural" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.ValidatingModel">
@foreach (var model in this.availableModels)
{
<MudSelectItem Value="@model">@model</MudSelectItem>
}
</MudSelect>
@if (this.ProvideModelManually)
{
<MudButton Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.OpenInBrowser" Href="@this.GetModelOverviewURL()" Target="_blank">Show available models</MudButton>
<MudTextField
T="string"
@bind-Text="@this.dataManuallyModel"
Label="Model"
Class="mb-3"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.Dns"
AdornmentColor="Color.Info"
Validation="@this.ValidateManuallyModel"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
/>
}
else
{
<MudButton Disabled="@(!this.CanLoadModels())" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.Refresh" OnClick="this.ReloadModels">Load</MudButton>
<MudSelect Disabled="@this.IsNoneProvider" @bind-Value="@this.DataModel" Label="Model" Class="mb-3" OpenIcon="@Icons.Material.Filled.FaceRetouchingNatural" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.ValidatingModel">
@foreach (var model in this.availableModels)
{
<MudSelectItem Value="@model">@model</MudSelectItem>
}
</MudSelect>
}
</MudStack>
@* ReSharper disable once CSharpWarnings::CS8974 *@

View File

@ -86,6 +86,7 @@ public partial class ProviderDialog : ComponentBase
private bool dataIsValid;
private string[] dataIssues = [];
private string dataAPIKey = string.Empty;
private string dataManuallyModel = string.Empty;
private string dataAPIKeyStorageIssue = string.Empty;
private string dataEditingPreviousInstanceName = string.Empty;
@ -100,7 +101,7 @@ public partial class ProviderDialog : ComponentBase
Id = this.DataId,
InstanceName = this.DataInstanceName,
UsedProvider = this.DataProvider,
Model = this.DataModel,
Model = this.DataProvider is Providers.FIREWORKS ? new Model(this.dataManuallyModel) : this.DataModel,
IsSelfHosted = this.DataProvider is Providers.SELF_HOSTED,
Hostname = this.DataHostname.EndsWith('/') ? this.DataHostname[..^1] : this.DataHostname,
Host = this.DataHost,
@ -220,6 +221,14 @@ public partial class ProviderDialog : ComponentBase
return null;
}
private string? ValidateManuallyModel(string manuallyModel)
{
if (this.DataProvider is Providers.FIREWORKS && string.IsNullOrWhiteSpace(manuallyModel))
return "Please enter a model name.";
return null;
}
private string? ValidatingModel(Model model)
{
if(this.DataProvider is Providers.SELF_HOSTED && this.DataHost == Host.LLAMACPP)
@ -354,11 +363,52 @@ public partial class ProviderDialog : ComponentBase
return true;
}
private bool IsCloudProvider => this.DataProvider is not Providers.SELF_HOSTED;
private bool ShowRegisterButton => this.DataProvider switch
{
Providers.OPEN_AI => true,
Providers.MISTRAL => true,
Providers.ANTHROPIC => true,
Providers.FIREWORKS => true,
_ => false,
};
private bool NeedAPIKey => this.DataProvider switch
{
Providers.OPEN_AI => true,
Providers.MISTRAL => true,
Providers.ANTHROPIC => true,
Providers.FIREWORKS => true,
_ => false,
};
private bool NeedHostname => this.DataProvider switch
{
Providers.SELF_HOSTED => true,
_ => false,
};
private bool IsSelfHostedOrNone => this.DataProvider is Providers.SELF_HOSTED or Providers.NONE;
private bool NeedHost => this.DataProvider switch
{
Providers.SELF_HOSTED => true,
_ => false,
};
private bool IsNoneProvider => this.DataProvider is Providers.NONE;
private bool ProvideModelManually => this.DataProvider switch
{
Providers.FIREWORKS => true,
_ => false,
};
private string GetModelOverviewURL() => this.DataProvider switch
{
Providers.FIREWORKS => "https://fireworks.ai/models?show=Serverless",
_ => string.Empty,
};
private string GetProviderCreationURL() => this.DataProvider switch
{
@ -366,6 +416,10 @@ public partial class ProviderDialog : ComponentBase
Providers.MISTRAL => "https://console.mistral.ai/",
Providers.ANTHROPIC => "https://console.anthropic.com/dashboard",
Providers.FIREWORKS => "https://fireworks.ai/login",
_ => string.Empty,
};
private bool IsNoneProvider => this.DataProvider is Providers.NONE;
}

View File

@ -163,6 +163,6 @@
"contentHash": "FHNOatmUq0sqJOkTx+UF/9YK1f180cnW5FVqnQMvYUN0elp6wFzbtPSiqbo1/ru8ICp43JM1i7kKkk6GsNGHlA=="
}
},
"net8.0/osx-arm64": {}
"net8.0/osx-x64": {}
}
}

View File

@ -1,5 +1,6 @@
# v0.8.3 (WIP)
- Migrated UI framework from MudBlazor v6.x.x to v7.x.x
# v0.8.3, build 165 (2024-07-25 13:25 UTC)
- Added an option to configure the behavior of the navigation bar in the settings
- Added support for Fireworks.ai as provider, where you can use e.g., the llama 3.1 405b model
- Improved the handling of self-hosted provider hostnames
- Improved the configured provider table: long model names are now truncated
- Improved the configured provider table: long model names are now truncated
- Migrated UI framework from MudBlazor v6.x.x to v7.x.x

View File

@ -1,9 +1,9 @@
0.8.2
2024-07-16 18:03:04 UTC
164
0.8.3
2024-07-25 13:25:12 UTC
165
8.0.107 (commit 1bdaef7265)
8.0.7 (commit 2aade6beb0)
1.79.0 (commit 129f3b996)
6.20.0
7.4.0
1.6.1
c92ce49af2d, release
72eb50d226f, release

2
runtime/Cargo.lock generated
View File

@ -2313,7 +2313,7 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mindwork-ai-studio"
version = "0.8.2"
version = "0.8.3"
dependencies = [
"arboard",
"flexi_logger",

View File

@ -1,6 +1,6 @@
[package]
name = "mindwork-ai-studio"
version = "0.8.2"
version = "0.8.3"
edition = "2021"
description = "MindWork AI Studio"
authors = ["Thorsten Sommer"]

View File

@ -6,7 +6,7 @@
},
"package": {
"productName": "MindWork AI Studio",
"version": "0.8.2"
"version": "0.8.3"
},
"tauri": {
"allowlist": {