mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-21 01:58:00 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9a65d9ca0 | ||
|
|
e4b1ea32fb | ||
|
|
e9975a5fbc | ||
|
|
2ef4d9d53f | ||
|
|
33a2728644 |
No files matched your search
@@ -35,6 +35,7 @@ Other News:
|
||||
|
||||
Features we have recently released:
|
||||
|
||||
- v0.9.25: Added [xAI](https://x.ai/) as a new provider. xAI provides their Grok models for generating content.
|
||||
- v0.9.23: Added support for OpenAI `o` models (`o1`, `o1-mini`, `o3`, etc.); added also an [ERI](https://github.com/MindWorkAI/ERI) server coding assistant as a preview feature behind the RAG feature flag. Your own ERI server can be used to gain access to, e.g., your enterprise data from within AI Studio.
|
||||
- v0.9.22: Added options for preview features; added embedding provider configuration for RAG (preview) and writer mode (experimental preview).
|
||||
- v0.9.18: Added the new Anthropic Heiku model; added Groq and Google Gemini as provider options.
|
||||
@@ -42,7 +43,6 @@ Features we have recently released:
|
||||
- v0.9.16: Added workspace display options & improved the layout of the app window.
|
||||
- v0.9.15: Added the bias-of-the-day assistant. Tells you about a cognitive bias every day.
|
||||
- v0.9.13: You can use `ollama` providers secured with API keys.
|
||||
- v0.9.12: Added a job posting assistant to the business category and improved grammar & spelling check and rewrite assistants.
|
||||
|
||||
## What is AI Studio?
|
||||
|
||||
@@ -53,7 +53,7 @@ MindWork AI Studio is a desktop application available for macOS, Windows, and Li
|
||||
|
||||
**Key advantages:**
|
||||
- **Free of charge**: The app is free to use, both for personal and commercial purposes.
|
||||
- **Independence**: You are not tied to any single provider. Instead, you can choose the provider that best suits their needs. Right now, we support OpenAI (GPT4o etc.), Mistral, Anthropic (Claude), Google Gemini, and self-hosted models using [llama.cpp](https://github.com/ggerganov/llama.cpp), [ollama](https://github.com/ollama/ollama), [LM Studio](https://lmstudio.ai/), [Groq](https://groq.com/), or [Fireworks](https://fireworks.ai/).
|
||||
- **Independence**: You are not tied to any single provider. Instead, you can choose the provider that best suits their needs. Right now, we support OpenAI (GPT4o etc.), Mistral, Anthropic (Claude), Google Gemini, xAI (Grok), and self-hosted models using [llama.cpp](https://github.com/ggerganov/llama.cpp), [ollama](https://github.com/ollama/ollama), [LM Studio](https://lmstudio.ai/), [Groq](https://groq.com/), or [Fireworks](https://fireworks.ai/).
|
||||
- **Unrestricted usage**: Unlike services like ChatGPT, which impose limits after intensive use, MindWork AI Studio offers unlimited usage through the providers API.
|
||||
- **Cost-effective**: You only pay for what you use, which can be cheaper than monthly subscription services like ChatGPT Plus, especially if used infrequently. But beware, here be dragons: For extremely intensive usage, the API costs can be significantly higher. Unfortunately, providers currently do not offer a way to display current costs in the app. Therefore, check your account with the respective provider to see how your costs are developing. When available, use prepaid and set a cost limit.
|
||||
- **Privacy**: The data entered into the app is not used for training by the providers since we are using the provider's API.
|
||||
|
||||
@@ -13,6 +13,7 @@ public partial class Changelog
|
||||
|
||||
public static readonly Log[] LOGS =
|
||||
[
|
||||
new (200, "v0.9.25, build 200 (2025-01-04 18:33 UTC)", "v0.9.25.md"),
|
||||
new (199, "v0.9.24, build 199 (2025-01-04 11:40 UTC)", "v0.9.24.md"),
|
||||
new (198, "v0.9.23, build 198 (2025-01-02 19:39 UTC)", "v0.9.23.md"),
|
||||
new (197, "v0.9.22, build 197 (2024-12-04 10:58 UTC)", "v0.9.22.md"),
|
||||
|
||||
@@ -76,6 +76,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
if (deferredContent is not null)
|
||||
{
|
||||
this.ChatThread = deferredContent;
|
||||
this.Logger.LogInformation($"The chat '{this.ChatThread.Name}' with {this.ChatThread.Blocks.Count} messages was deferred and will be rendered now.");
|
||||
await this.ChatThreadChanged.InvokeAsync(this.ChatThread);
|
||||
|
||||
if (this.ChatThread is not null)
|
||||
@@ -106,13 +107,13 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
await WorkspaceBehaviour.EnsureBiasWorkspace();
|
||||
}
|
||||
}
|
||||
|
||||
if (this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading)
|
||||
{
|
||||
this.mustScrollToBottomAfterRender = true;
|
||||
this.scrollRenderCountdown = 2;
|
||||
this.StateHasChanged();
|
||||
}
|
||||
}
|
||||
|
||||
if (this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading)
|
||||
{
|
||||
this.mustScrollToBottomAfterRender = true;
|
||||
this.scrollRenderCountdown = 4;
|
||||
this.StateHasChanged();
|
||||
}
|
||||
|
||||
var deferredLoading = MessageBus.INSTANCE.CheckDeferredMessages<LoadChat>(Event.LOAD_CHAT).FirstOrDefault();
|
||||
@@ -120,6 +121,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
{
|
||||
this.loadChat = deferredLoading;
|
||||
this.mustLoadChat = true;
|
||||
this.Logger.LogInformation($"The loading of the chat '{this.loadChat.ChatId}' was deferred and will be loaded now.");
|
||||
}
|
||||
|
||||
await this.SelectProviderWhenLoadingChat();
|
||||
@@ -144,18 +146,23 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
|
||||
if (firstRender && this.mustLoadChat)
|
||||
{
|
||||
this.Logger.LogInformation($"Try to load the chat '{this.loadChat.ChatId}' now.");
|
||||
this.mustLoadChat = false;
|
||||
this.ChatThread = await WorkspaceBehaviour.LoadChat(this.loadChat);
|
||||
|
||||
if(this.ChatThread is not null)
|
||||
{
|
||||
await this.ChatThreadChanged.InvokeAsync(this.ChatThread);
|
||||
this.Logger.LogInformation($"The chat '{this.ChatThread!.ChatId}' with title '{this.ChatThread.Name}' ({this.ChatThread.Blocks.Count} messages) was loaded successfully.");
|
||||
|
||||
this.currentWorkspaceName = await WorkspaceBehaviour.LoadWorkspaceName(this.ChatThread.WorkspaceId);
|
||||
this.WorkspaceName(this.currentWorkspaceName);
|
||||
await this.SelectProviderWhenLoadingChat();
|
||||
}
|
||||
else
|
||||
this.Logger.LogWarning($"The chat '{this.loadChat.ChatId}' could not be loaded.");
|
||||
|
||||
this.StateHasChanged();
|
||||
await this.ChatThreadChanged.InvokeAsync(this.ChatThread);
|
||||
}
|
||||
|
||||
if(this.mustScrollToBottomAfterRender)
|
||||
@@ -523,8 +530,6 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
this.WorkspaceName(this.currentWorkspaceName);
|
||||
|
||||
await this.SelectProviderWhenLoadingChat();
|
||||
|
||||
this.userInput = string.Empty;
|
||||
if (this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading)
|
||||
{
|
||||
this.mustScrollToBottomAfterRender = true;
|
||||
@@ -551,7 +556,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
{
|
||||
var chatProvider = this.ChatThread?.SelectedProvider;
|
||||
var chatProfile = this.ChatThread?.SelectedProfile;
|
||||
|
||||
|
||||
switch (this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior)
|
||||
{
|
||||
default:
|
||||
|
||||
@@ -33,7 +33,7 @@ public partial class Home : ComponentBase
|
||||
private static readonly TextItem[] ITEMS_ADVANTAGES =
|
||||
[
|
||||
new("Free of charge", "The app is free to use, both for personal and commercial purposes."),
|
||||
new("Independence", "You are not tied to any single provider. Instead, you might choose the provider that best suits your needs. Right now, we support OpenAI (GPT4o etc.), Mistral, Anthropic (Claude), Google Gemini, and self-hosted models using llama.cpp, ollama, LM Studio, Groq, or Fireworks."),
|
||||
new("Independence", "You are not tied to any single provider. Instead, you might choose the provider that best suits your needs. Right now, we support OpenAI (GPT4o etc.), Mistral, Anthropic (Claude), Google Gemini, xAI (Grok), and self-hosted models using llama.cpp, ollama, LM Studio, Groq, or Fireworks."),
|
||||
new("Unrestricted usage", "Unlike services like ChatGPT, which impose limits after intensive use, MindWork AI Studio offers unlimited usage through the providers API."),
|
||||
new("Cost-effective", "You only pay for what you use, which can be cheaper than monthly subscription services like ChatGPT Plus, especially if used infrequently. But beware, here be dragons: For extremely intensive usage, the API costs can be significantly higher. Unfortunately, providers currently do not offer a way to display current costs in the app. Therefore, check your account with the respective provider to see how your costs are developing. When available, use prepaid and set a cost limit."),
|
||||
new("Privacy", "The data entered into the app is not used for training by the providers since we are using the provider's API."),
|
||||
|
||||
@@ -10,11 +10,6 @@ namespace AIStudio.Provider.Anthropic;
|
||||
|
||||
public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://api.anthropic.com/v1/", logger)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
};
|
||||
|
||||
#region Implementation of IProvider
|
||||
|
||||
public override string Id => LLMProviders.ANTHROPIC.ToName();
|
||||
@@ -60,117 +55,24 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap
|
||||
Stream = true,
|
||||
}, JSON_SERIALIZER_OPTIONS);
|
||||
|
||||
StreamReader? streamReader = null;
|
||||
try
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "messages");
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "messages");
|
||||
|
||||
// Set the authorization header:
|
||||
request.Headers.Add("x-api-key", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
// Set the authorization header:
|
||||
request.Headers.Add("x-api-key", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
|
||||
// Set the Anthropic version:
|
||||
request.Headers.Add("anthropic-version", "2023-06-01");
|
||||
// Set the Anthropic version:
|
||||
request.Headers.Add("anthropic-version", "2023-06-01");
|
||||
|
||||
// Set the content:
|
||||
request.Content = new StringContent(chatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
// Send the request using exponential backoff:
|
||||
var responseData = await this.SendRequest(RequestBuilder, token);
|
||||
if(responseData.IsFailedAfterAllRetries)
|
||||
{
|
||||
this.logger.LogError($"Anthropic chat completion failed: {responseData.ErrorMessage}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Open the response stream:
|
||||
var stream = await responseData.Response!.Content.ReadAsStreamAsync(token);
|
||||
|
||||
// Add a stream reader to read the stream, line by line:
|
||||
streamReader = new StreamReader(stream);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to stream chat completion from Anthropic '{this.InstanceName}': {e.Message}");
|
||||
// Set the content:
|
||||
request.Content = new StringContent(chatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
if (streamReader is null)
|
||||
yield break;
|
||||
|
||||
// Read the stream, line by line:
|
||||
while(true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(streamReader.EndOfStream)
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogWarning($"Failed to read the end-of-stream state from Anthropic '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if the token is canceled:
|
||||
if(token.IsCancellationRequested)
|
||||
yield break;
|
||||
|
||||
// Read the next line:
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await streamReader.ReadLineAsync(token);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to read the stream from Anthropic '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if(string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
// Check for the end of the stream:
|
||||
if(line.StartsWith("event: message_stop", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
|
||||
// Skip lines that do not start with "data: ". Regard
|
||||
// to the specification, we only want to read the data lines:
|
||||
if(!line.StartsWith("data: ", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
// Ignore any type except "content_block_delta":
|
||||
if(!line.Contains("\"content_block_delta\"", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
ResponseStreamLine anthropicResponse;
|
||||
try
|
||||
{
|
||||
// We know that the line starts with "data: ". Hence, we can
|
||||
// skip the first 6 characters to get the JSON data after that.
|
||||
var jsonData = line[6..];
|
||||
|
||||
// Deserialize the JSON data:
|
||||
anthropicResponse = JsonSerializer.Deserialize<ResponseStreamLine>(jsonData, JSON_SERIALIZER_OPTIONS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid JSON data:
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty responses:
|
||||
if(anthropicResponse == default || string.IsNullOrWhiteSpace(anthropicResponse.Delta.Text))
|
||||
continue;
|
||||
|
||||
// Yield the response:
|
||||
yield return anthropicResponse.Delta.Text;
|
||||
}
|
||||
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Anthropic", RequestBuilder, token))
|
||||
yield return content;
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -7,7 +7,14 @@ namespace AIStudio.Provider.Anthropic;
|
||||
/// <param name="Type">The type of the response line.</param>
|
||||
/// <param name="Index">The index of the response line.</param>
|
||||
/// <param name="Delta">The delta of the response line.</param>
|
||||
public readonly record struct ResponseStreamLine(string Type, int Index, Delta Delta);
|
||||
public readonly record struct ResponseStreamLine(string Type, int Index, Delta Delta) : IResponseStreamLine
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public bool ContainsContent() => this != default && !string.IsNullOrWhiteSpace(this.Delta.Text);
|
||||
|
||||
/// <inheritdoc />
|
||||
public string GetContent() => this.Delta.Text;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The delta object of a response line.
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using System.Net;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Settings;
|
||||
@@ -31,6 +33,11 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
protected static readonly RustService RUST_SERVICE;
|
||||
|
||||
protected static readonly Encryption ENCRYPTION;
|
||||
|
||||
protected static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Constructor for the base provider.
|
||||
@@ -127,4 +134,105 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
|
||||
return new HttpRateLimitedStreamResult(true, false, string.Empty, response);
|
||||
}
|
||||
|
||||
protected async IAsyncEnumerable<string> StreamChatCompletionInternal<T>(string providerName, Func<Task<HttpRequestMessage>> requestBuilder, [EnumeratorCancellation] CancellationToken token = default) where T : struct, IResponseStreamLine
|
||||
{
|
||||
StreamReader? streamReader = null;
|
||||
try
|
||||
{
|
||||
// Send the request using exponential backoff:
|
||||
var responseData = await this.SendRequest(requestBuilder, token);
|
||||
if(responseData.IsFailedAfterAllRetries)
|
||||
{
|
||||
this.logger.LogError($"The {providerName} chat completion failed: {responseData.ErrorMessage}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Open the response stream:
|
||||
var providerStream = await responseData.Response!.Content.ReadAsStreamAsync(token);
|
||||
|
||||
// Add a stream reader to read the stream, line by line:
|
||||
streamReader = new StreamReader(providerStream);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to stream chat completion from {providerName} '{this.InstanceName}': {e.Message}");
|
||||
}
|
||||
|
||||
if (streamReader is null)
|
||||
yield break;
|
||||
|
||||
// Read the stream, line by line:
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(streamReader.EndOfStream)
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogWarning($"Failed to read the end-of-stream state from {providerName} '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if the token is canceled:
|
||||
if (token.IsCancellationRequested)
|
||||
{
|
||||
this.logger.LogWarning($"The user canceled the chat completion for {providerName} '{this.InstanceName}'.");
|
||||
streamReader.Close();
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Read the next line:
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await streamReader.ReadLineAsync(token);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to read the stream from {providerName} '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
// Skip lines that do not start with "data: ". Regard
|
||||
// to the specification, we only want to read the data lines:
|
||||
if (!line.StartsWith("data: ", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
// Check if the line is the end of the stream:
|
||||
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
|
||||
T providerResponse;
|
||||
try
|
||||
{
|
||||
// We know that the line starts with "data: ". Hence, we can
|
||||
// skip the first 6 characters to get the JSON data after that.
|
||||
var jsonData = line[6..];
|
||||
|
||||
// Deserialize the JSON data:
|
||||
providerResponse = JsonSerializer.Deserialize<T>(jsonData, JSON_SERIALIZER_OPTIONS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid JSON data:
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty responses:
|
||||
if (!providerResponse.ContainsContent())
|
||||
continue;
|
||||
|
||||
// Yield the response:
|
||||
yield return providerResponse.GetContent();
|
||||
}
|
||||
|
||||
streamReader.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -10,11 +10,6 @@ namespace AIStudio.Provider.Fireworks;
|
||||
|
||||
public class ProviderFireworks(ILogger logger) : BaseProvider("https://api.fireworks.ai/inference/v1/", logger)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
};
|
||||
|
||||
#region Implementation of IProvider
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -69,110 +64,21 @@ public class ProviderFireworks(ILogger logger) : BaseProvider("https://api.firew
|
||||
Stream = true,
|
||||
}, JSON_SERIALIZER_OPTIONS);
|
||||
|
||||
StreamReader? streamReader = null;
|
||||
try
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
|
||||
// Set the content:
|
||||
request.Content = new StringContent(fireworksChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
// Send the request using exponential backoff:
|
||||
var responseData = await this.SendRequest(RequestBuilder, token);
|
||||
if(responseData.IsFailedAfterAllRetries)
|
||||
{
|
||||
this.logger.LogError($"Fireworks chat completion failed: {responseData.ErrorMessage}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Open the response stream:
|
||||
var fireworksStream = await responseData.Response!.Content.ReadAsStreamAsync(token);
|
||||
|
||||
// Add a stream reader to read the stream, line by line:
|
||||
streamReader = new StreamReader(fireworksStream);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to stream chat completion from Fireworks '{this.InstanceName}': {e.Message}");
|
||||
// Set the content:
|
||||
request.Content = new StringContent(fireworksChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
if (streamReader is null)
|
||||
yield break;
|
||||
|
||||
// Read the stream, line by line:
|
||||
while(true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(streamReader.EndOfStream)
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogWarning($"Failed to read the end-of-stream state from Fireworks '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if the token is canceled:
|
||||
if(token.IsCancellationRequested)
|
||||
yield break;
|
||||
|
||||
// Read the next line:
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await streamReader.ReadLineAsync(token);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to read the stream from Fireworks '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if(string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
// Skip lines that do not start with "data: ". Regard
|
||||
// to the specification, we only want to read the data lines:
|
||||
if(!line.StartsWith("data: ", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
// Check if the line is the end of the stream:
|
||||
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
|
||||
ResponseStreamLine fireworksResponse;
|
||||
try
|
||||
{
|
||||
// We know that the line starts with "data: ". Hence, we can
|
||||
// skip the first 6 characters to get the JSON data after that.
|
||||
var jsonData = line[6..];
|
||||
|
||||
// Deserialize the JSON data:
|
||||
fireworksResponse = JsonSerializer.Deserialize<ResponseStreamLine>(jsonData, JSON_SERIALIZER_OPTIONS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid JSON data:
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty responses:
|
||||
if(fireworksResponse == default || fireworksResponse.Choices.Count == 0)
|
||||
continue;
|
||||
|
||||
// Yield the response:
|
||||
yield return fireworksResponse.Choices[0].Delta.Content;
|
||||
}
|
||||
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Fireworks", RequestBuilder, token))
|
||||
yield return content;
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -8,7 +8,14 @@ namespace AIStudio.Provider.Fireworks;
|
||||
/// <param name="Created">The timestamp of the response.</param>
|
||||
/// <param name="Model">The model used for the response.</param>
|
||||
/// <param name="Choices">The choices made by the AI.</param>
|
||||
public readonly record struct ResponseStreamLine(string Id, string Object, uint Created, string Model, IList<Choice> Choices);
|
||||
public readonly record struct ResponseStreamLine(string Id, string Object, uint Created, string Model, IList<Choice> Choices) : IResponseStreamLine
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public bool ContainsContent() => this != default && this.Choices.Count > 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string GetContent() => this.Choices[0].Delta.Content;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Data model for a choice made by the AI.
|
||||
|
||||
@@ -11,11 +11,6 @@ namespace AIStudio.Provider.Google;
|
||||
|
||||
public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativelanguage.googleapis.com/v1beta/", logger)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
};
|
||||
|
||||
#region Implementation of IProvider
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -70,110 +65,21 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
|
||||
Stream = true,
|
||||
}, JSON_SERIALIZER_OPTIONS);
|
||||
|
||||
StreamReader? streamReader = null;
|
||||
try
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
|
||||
// Set the content:
|
||||
request.Content = new StringContent(geminiChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
// Send the request using exponential backoff:
|
||||
var responseData = await this.SendRequest(RequestBuilder, token);
|
||||
if(responseData.IsFailedAfterAllRetries)
|
||||
{
|
||||
this.logger.LogError($"Google chat completion failed: {responseData.ErrorMessage}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Open the response stream:
|
||||
var geminiStream = await responseData.Response!.Content.ReadAsStreamAsync(token);
|
||||
|
||||
// Add a stream reader to read the stream, line by line:
|
||||
streamReader = new StreamReader(geminiStream);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to stream chat completion from Google '{this.InstanceName}': {e.Message}");
|
||||
// Set the content:
|
||||
request.Content = new StringContent(geminiChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
if (streamReader is null)
|
||||
yield break;
|
||||
|
||||
// Read the stream, line by line:
|
||||
while(true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(streamReader.EndOfStream)
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogWarning($"Failed to read the end-of-stream state from Google '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if the token is canceled:
|
||||
if(token.IsCancellationRequested)
|
||||
yield break;
|
||||
|
||||
// Read the next line:
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await streamReader.ReadLineAsync(token);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to read the stream from Google '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if(string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
// Skip lines that do not start with "data: ". Regard
|
||||
// to the specification, we only want to read the data lines:
|
||||
if(!line.StartsWith("data: ", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
// Check if the line is the end of the stream:
|
||||
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
|
||||
ResponseStreamLine geminiResponse;
|
||||
try
|
||||
{
|
||||
// We know that the line starts with "data: ". Hence, we can
|
||||
// skip the first 6 characters to get the JSON data after that.
|
||||
var jsonData = line[6..];
|
||||
|
||||
// Deserialize the JSON data:
|
||||
geminiResponse = JsonSerializer.Deserialize<ResponseStreamLine>(jsonData, JSON_SERIALIZER_OPTIONS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid JSON data:
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty responses:
|
||||
if(geminiResponse == default || geminiResponse.Choices.Count == 0)
|
||||
continue;
|
||||
|
||||
// Yield the response:
|
||||
yield return geminiResponse.Choices[0].Delta.Content;
|
||||
}
|
||||
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Google", RequestBuilder, token))
|
||||
yield return content;
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -11,11 +11,6 @@ namespace AIStudio.Provider.Groq;
|
||||
|
||||
public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/openai/v1/", logger)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
};
|
||||
|
||||
#region Implementation of IProvider
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -72,110 +67,21 @@ public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/o
|
||||
Stream = true,
|
||||
}, JSON_SERIALIZER_OPTIONS);
|
||||
|
||||
StreamReader? streamReader = null;
|
||||
try
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
|
||||
// Set the content:
|
||||
request.Content = new StringContent(groqChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
// Send the request using exponential backoff:
|
||||
var responseData = await this.SendRequest(RequestBuilder, token);
|
||||
if(responseData.IsFailedAfterAllRetries)
|
||||
{
|
||||
this.logger.LogError($"Groq chat completion failed: {responseData.ErrorMessage}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Open the response stream:
|
||||
var groqStream = await responseData.Response!.Content.ReadAsStreamAsync(token);
|
||||
|
||||
// Add a stream reader to read the stream, line by line:
|
||||
streamReader = new StreamReader(groqStream);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to stream chat completion from Groq '{this.InstanceName}': {e.Message}");
|
||||
// Set the content:
|
||||
request.Content = new StringContent(groqChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
if (streamReader is null)
|
||||
yield break;
|
||||
|
||||
// Read the stream, line by line:
|
||||
while(true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(streamReader.EndOfStream)
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogWarning($"Failed to read the end-of-stream state from Groq '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if the token is canceled:
|
||||
if(token.IsCancellationRequested)
|
||||
yield break;
|
||||
|
||||
// Read the next line:
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await streamReader.ReadLineAsync(token);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to read the stream from Groq '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if(string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
// Skip lines that do not start with "data: ". Regard
|
||||
// to the specification, we only want to read the data lines:
|
||||
if(!line.StartsWith("data: ", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
// Check if the line is the end of the stream:
|
||||
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
|
||||
ResponseStreamLine groqResponse;
|
||||
try
|
||||
{
|
||||
// We know that the line starts with "data: ". Hence, we can
|
||||
// skip the first 6 characters to get the JSON data after that.
|
||||
var jsonData = line[6..];
|
||||
|
||||
// Deserialize the JSON data:
|
||||
groqResponse = JsonSerializer.Deserialize<ResponseStreamLine>(jsonData, JSON_SERIALIZER_OPTIONS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid JSON data:
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty responses:
|
||||
if(groqResponse == default || groqResponse.Choices.Count == 0)
|
||||
continue;
|
||||
|
||||
// Yield the response:
|
||||
yield return groqResponse.Choices[0].Delta.Content;
|
||||
}
|
||||
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Groq", RequestBuilder, token))
|
||||
yield return content;
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace AIStudio.Provider;
|
||||
|
||||
public interface IResponseStreamLine
|
||||
{
|
||||
/// <summary>
|
||||
/// Checks if the response line contains any content.
|
||||
/// </summary>
|
||||
/// <returns>True when the response line contains content, false otherwise.</returns>
|
||||
public bool ContainsContent();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the content of the response line.
|
||||
/// </summary>
|
||||
/// <returns>The content of the response line.</returns>
|
||||
public string GetContent();
|
||||
}
|
||||
@@ -11,6 +11,7 @@ public enum LLMProviders
|
||||
ANTHROPIC = 2,
|
||||
MISTRAL = 3,
|
||||
GOOGLE = 7,
|
||||
X = 8,
|
||||
|
||||
FIREWORKS = 5,
|
||||
GROQ = 6,
|
||||
|
||||
@@ -5,6 +5,7 @@ using AIStudio.Provider.Groq;
|
||||
using AIStudio.Provider.Mistral;
|
||||
using AIStudio.Provider.OpenAI;
|
||||
using AIStudio.Provider.SelfHosted;
|
||||
using AIStudio.Provider.X;
|
||||
using AIStudio.Settings;
|
||||
|
||||
using Host = AIStudio.Provider.SelfHosted.Host;
|
||||
@@ -26,6 +27,7 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.ANTHROPIC => "Anthropic",
|
||||
LLMProviders.MISTRAL => "Mistral",
|
||||
LLMProviders.GOOGLE => "Google",
|
||||
LLMProviders.X => "xAI",
|
||||
|
||||
LLMProviders.GROQ => "Groq",
|
||||
LLMProviders.FIREWORKS => "Fireworks.ai",
|
||||
@@ -62,6 +64,8 @@ public static class LLMProvidersExtensions
|
||||
|
||||
LLMProviders.MISTRAL => Confidence.GDPR_NO_TRAINING.WithRegion("Europe, France").WithSources("https://mistral.ai/terms/#terms-of-service-la-plateforme").WithLevel(settingsManager.GetConfiguredConfidenceLevel(llmProvider)),
|
||||
|
||||
LLMProviders.X => Confidence.USA_NO_TRAINING.WithRegion("America, U.S.").WithSources("https://x.ai/legal/terms-of-service-enterprise").WithLevel(settingsManager.GetConfiguredConfidenceLevel(llmProvider)),
|
||||
|
||||
LLMProviders.SELF_HOSTED => Confidence.SELF_HOSTED.WithLevel(settingsManager.GetConfiguredConfidenceLevel(llmProvider)),
|
||||
|
||||
_ => Confidence.UNKNOWN.WithLevel(settingsManager.GetConfiguredConfidenceLevel(llmProvider)),
|
||||
@@ -87,6 +91,7 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.GROQ => false,
|
||||
LLMProviders.ANTHROPIC => false,
|
||||
LLMProviders.FIREWORKS => false,
|
||||
LLMProviders.X => false,
|
||||
|
||||
//
|
||||
// Self-hosted providers are treated as a special case anyway.
|
||||
@@ -128,6 +133,7 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.ANTHROPIC => new ProviderAnthropic(logger) { InstanceName = instanceName },
|
||||
LLMProviders.MISTRAL => new ProviderMistral(logger) { InstanceName = instanceName },
|
||||
LLMProviders.GOOGLE => new ProviderGoogle(logger) { InstanceName = instanceName },
|
||||
LLMProviders.X => new ProviderX(logger) { InstanceName = instanceName },
|
||||
|
||||
LLMProviders.GROQ => new ProviderGroq(logger) { InstanceName = instanceName },
|
||||
LLMProviders.FIREWORKS => new ProviderFireworks(logger) { InstanceName = instanceName },
|
||||
@@ -150,6 +156,7 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.MISTRAL => "https://console.mistral.ai/",
|
||||
LLMProviders.ANTHROPIC => "https://console.anthropic.com/dashboard",
|
||||
LLMProviders.GOOGLE => "https://console.cloud.google.com/",
|
||||
LLMProviders.X => "https://accounts.x.ai/sign-up",
|
||||
|
||||
LLMProviders.GROQ => "https://console.groq.com/",
|
||||
LLMProviders.FIREWORKS => "https://fireworks.ai/login",
|
||||
@@ -162,6 +169,7 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.OPEN_AI => "https://platform.openai.com/usage",
|
||||
LLMProviders.MISTRAL => "https://console.mistral.ai/usage/",
|
||||
LLMProviders.ANTHROPIC => "https://console.anthropic.com/settings/plans",
|
||||
LLMProviders.X => "https://console.x.ai/",
|
||||
LLMProviders.GROQ => "https://console.groq.com/settings/usage",
|
||||
LLMProviders.GOOGLE => "https://console.cloud.google.com/billing",
|
||||
LLMProviders.FIREWORKS => "https://fireworks.ai/account/billing",
|
||||
@@ -174,6 +182,7 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.OPEN_AI => true,
|
||||
LLMProviders.MISTRAL => true,
|
||||
LLMProviders.ANTHROPIC => true,
|
||||
LLMProviders.X => true,
|
||||
LLMProviders.GROQ => true,
|
||||
LLMProviders.FIREWORKS => true,
|
||||
LLMProviders.GOOGLE => true,
|
||||
@@ -217,6 +226,7 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.MISTRAL => true,
|
||||
LLMProviders.ANTHROPIC => true,
|
||||
LLMProviders.GOOGLE => true,
|
||||
LLMProviders.X => true,
|
||||
|
||||
LLMProviders.GROQ => true,
|
||||
LLMProviders.FIREWORKS => true,
|
||||
@@ -232,6 +242,7 @@ public static class LLMProvidersExtensions
|
||||
LLMProviders.MISTRAL => true,
|
||||
LLMProviders.ANTHROPIC => true,
|
||||
LLMProviders.GOOGLE => true,
|
||||
LLMProviders.X => true,
|
||||
|
||||
LLMProviders.GROQ => true,
|
||||
LLMProviders.FIREWORKS => true,
|
||||
|
||||
@@ -11,11 +11,6 @@ namespace AIStudio.Provider.Mistral;
|
||||
|
||||
public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.mistral.ai/v1/", logger)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
};
|
||||
|
||||
#region Implementation of IProvider
|
||||
|
||||
public override string Id => LLMProviders.MISTRAL.ToName();
|
||||
@@ -71,110 +66,21 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
|
||||
SafePrompt = false,
|
||||
}, JSON_SERIALIZER_OPTIONS);
|
||||
|
||||
StreamReader? streamReader = null;
|
||||
try
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
|
||||
// Set the content:
|
||||
request.Content = new StringContent(mistralChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
// Send the request using exponential backoff:
|
||||
var responseData = await this.SendRequest(RequestBuilder, token);
|
||||
if(responseData.IsFailedAfterAllRetries)
|
||||
{
|
||||
this.logger.LogError($"Mistral chat completion failed: {responseData.ErrorMessage}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Open the response stream:
|
||||
var mistralStream = await responseData.Response!.Content.ReadAsStreamAsync(token);
|
||||
|
||||
// Add a stream reader to read the stream, line by line:
|
||||
streamReader = new StreamReader(mistralStream);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to stream chat completion from Mistral '{this.InstanceName}': {e.Message}");
|
||||
// Set the content:
|
||||
request.Content = new StringContent(mistralChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
if (streamReader is null)
|
||||
yield break;
|
||||
|
||||
// Read the stream, line by line:
|
||||
while(true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(streamReader.EndOfStream)
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogWarning($"Failed to read the end-of-stream state from Mistral '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if the token is canceled:
|
||||
if(token.IsCancellationRequested)
|
||||
yield break;
|
||||
|
||||
// Read the next line:
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await streamReader.ReadLineAsync(token);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to read the stream from Mistral '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if(string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
// Skip lines that do not start with "data: ". Regard
|
||||
// to the specification, we only want to read the data lines:
|
||||
if(!line.StartsWith("data: ", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
// Check if the line is the end of the stream:
|
||||
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
|
||||
ResponseStreamLine mistralResponse;
|
||||
try
|
||||
{
|
||||
// We know that the line starts with "data: ". Hence, we can
|
||||
// skip the first 6 characters to get the JSON data after that.
|
||||
var jsonData = line[6..];
|
||||
|
||||
// Deserialize the JSON data:
|
||||
mistralResponse = JsonSerializer.Deserialize<ResponseStreamLine>(jsonData, JSON_SERIALIZER_OPTIONS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid JSON data:
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty responses:
|
||||
if(mistralResponse == default || mistralResponse.Choices.Count == 0)
|
||||
continue;
|
||||
|
||||
// Yield the response:
|
||||
yield return mistralResponse.Choices[0].Delta.Content;
|
||||
}
|
||||
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Mistral", RequestBuilder, token))
|
||||
yield return content;
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -13,11 +13,6 @@ namespace AIStudio.Provider.OpenAI;
|
||||
/// </summary>
|
||||
public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.openai.com/v1/", logger)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
};
|
||||
|
||||
#region Implementation of IProvider
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -99,110 +94,21 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
|
||||
Stream = true,
|
||||
}, JSON_SERIALIZER_OPTIONS);
|
||||
|
||||
StreamReader? streamReader = null;
|
||||
try
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
|
||||
// Set the content:
|
||||
request.Content = new StringContent(openAIChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
// Send the request using exponential backoff:
|
||||
var responseData = await this.SendRequest(RequestBuilder, token);
|
||||
if(responseData.IsFailedAfterAllRetries)
|
||||
{
|
||||
this.logger.LogError($"OpenAI chat completion failed: {responseData.ErrorMessage}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Open the response stream:
|
||||
var openAIStream = await responseData.Response!.Content.ReadAsStreamAsync(token);
|
||||
|
||||
// Add a stream reader to read the stream, line by line:
|
||||
streamReader = new StreamReader(openAIStream);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to stream chat completion from OpenAI '{this.InstanceName}': {e.Message}");
|
||||
// Set the content:
|
||||
request.Content = new StringContent(openAIChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
if (streamReader is null)
|
||||
yield break;
|
||||
|
||||
// Read the stream, line by line:
|
||||
while(true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(streamReader.EndOfStream)
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogWarning($"Failed to read the end-of-stream state from OpenAI '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if the token is canceled:
|
||||
if(token.IsCancellationRequested)
|
||||
yield break;
|
||||
|
||||
// Read the next line:
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await streamReader.ReadLineAsync(token);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to read the stream from OpenAI '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if(string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
// Skip lines that do not start with "data: ". Regard
|
||||
// to the specification, we only want to read the data lines:
|
||||
if(!line.StartsWith("data: ", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
// Check if the line is the end of the stream:
|
||||
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
|
||||
ResponseStreamLine openAIResponse;
|
||||
try
|
||||
{
|
||||
// We know that the line starts with "data: ". Hence, we can
|
||||
// skip the first 6 characters to get the JSON data after that.
|
||||
var jsonData = line[6..];
|
||||
|
||||
// Deserialize the JSON data:
|
||||
openAIResponse = JsonSerializer.Deserialize<ResponseStreamLine>(jsonData, JSON_SERIALIZER_OPTIONS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid JSON data:
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty responses:
|
||||
if(openAIResponse == default || openAIResponse.Choices.Count == 0)
|
||||
continue;
|
||||
|
||||
// Yield the response:
|
||||
yield return openAIResponse.Choices[0].Delta.Content;
|
||||
}
|
||||
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("OpenAI", RequestBuilder, token))
|
||||
yield return content;
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -9,7 +9,14 @@ namespace AIStudio.Provider.OpenAI;
|
||||
/// <param name="Model">The model used for the response.</param>
|
||||
/// <param name="SystemFingerprint">The system fingerprint; together with the seed, this allows you to reproduce the response.</param>
|
||||
/// <param name="Choices">The choices made by the AI.</param>
|
||||
public readonly record struct ResponseStreamLine(string Id, string Object, uint Created, string Model, string SystemFingerprint, IList<Choice> Choices);
|
||||
public readonly record struct ResponseStreamLine(string Id, string Object, uint Created, string Model, string SystemFingerprint, IList<Choice> Choices) : IResponseStreamLine
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public bool ContainsContent() => this != default && this.Choices.Count > 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string GetContent() => this.Choices[0].Delta.Content;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Data model for a choice made by the AI.
|
||||
|
||||
@@ -11,11 +11,6 @@ namespace AIStudio.Provider.SelfHosted;
|
||||
|
||||
public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostname) : BaseProvider($"{hostname}{host.BaseURL()}", logger)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
};
|
||||
|
||||
#region Implementation of IProvider
|
||||
|
||||
public override string Id => LLMProviders.SELF_HOSTED.ToName();
|
||||
@@ -67,113 +62,22 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam
|
||||
MaxTokens = -1,
|
||||
}, JSON_SERIALIZER_OPTIONS);
|
||||
|
||||
StreamReader? streamReader = null;
|
||||
try
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, host.ChatURL());
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, host.ChatURL());
|
||||
|
||||
// Set the authorization header:
|
||||
if (requestedSecret.Success)
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
// Set the authorization header:
|
||||
if (requestedSecret.Success)
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
|
||||
// Set the content:
|
||||
request.Content = new StringContent(providerChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
// Send the request using exponential backoff:
|
||||
var responseData = await this.SendRequest(RequestBuilder, token);
|
||||
if(responseData.IsFailedAfterAllRetries)
|
||||
{
|
||||
this.logger.LogError($"Self-hosted provider's chat completion failed: {responseData.ErrorMessage}");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Open the response stream:
|
||||
var providerStream = await responseData.Response!.Content.ReadAsStreamAsync(token);
|
||||
|
||||
// Add a stream reader to read the stream, line by line:
|
||||
streamReader = new StreamReader(providerStream);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to stream chat completion from self-hosted provider '{this.InstanceName}': {e.Message}");
|
||||
}
|
||||
|
||||
if (streamReader is null)
|
||||
yield break;
|
||||
|
||||
// Read the stream, line by line:
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(streamReader.EndOfStream)
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogWarning($"Failed to read the end-of-stream state from self-hosted provider '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check if the token is canceled:
|
||||
if (token.IsCancellationRequested)
|
||||
yield break;
|
||||
|
||||
// Read the next line:
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await streamReader.ReadLineAsync(token);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger.LogError($"Failed to read the stream from self-hosted provider '{this.InstanceName}': {e.Message}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
// Skip lines that do not start with "data: ". Regard
|
||||
// to the specification, we only want to read the data lines:
|
||||
if (!line.StartsWith("data: ", StringComparison.InvariantCulture))
|
||||
continue;
|
||||
|
||||
// Check if the line is the end of the stream:
|
||||
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
|
||||
ResponseStreamLine providerResponse;
|
||||
try
|
||||
{
|
||||
// We know that the line starts with "data: ". Hence, we can
|
||||
// skip the first 6 characters to get the JSON data after that.
|
||||
var jsonData = line[6..];
|
||||
|
||||
// Deserialize the JSON data:
|
||||
providerResponse = JsonSerializer.Deserialize<ResponseStreamLine>(jsonData, JSON_SERIALIZER_OPTIONS);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip invalid JSON data:
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty responses:
|
||||
if (providerResponse == default || providerResponse.Choices.Count == 0)
|
||||
continue;
|
||||
|
||||
// Yield the response:
|
||||
yield return providerResponse.Choices[0].Delta.Content;
|
||||
// Set the content:
|
||||
request.Content = new StringContent(providerChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
streamReader.Dispose();
|
||||
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("self-hosted provider", RequestBuilder, token))
|
||||
yield return content;
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
using System.Net.Http.Headers;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Provider.OpenAI;
|
||||
using AIStudio.Settings;
|
||||
|
||||
namespace AIStudio.Provider.X;
|
||||
|
||||
public sealed class ProviderX(ILogger logger) : BaseProvider("https://api.x.ai/v1/", logger)
|
||||
{
|
||||
#region Implementation of IProvider
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string Id => LLMProviders.X.ToName();
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string InstanceName { get; set; } = "xAI";
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
// Get the API key:
|
||||
var requestedSecret = await RUST_SERVICE.GetAPIKey(this);
|
||||
if(!requestedSecret.Success)
|
||||
yield break;
|
||||
|
||||
// Prepare the system prompt:
|
||||
var systemPrompt = new Message
|
||||
{
|
||||
Role = "system",
|
||||
Content = chatThread.PrepareSystemPrompt(settingsManager, chatThread, this.logger),
|
||||
};
|
||||
|
||||
// Prepare the xAI HTTP chat request:
|
||||
var xChatRequest = JsonSerializer.Serialize(new ChatRequest
|
||||
{
|
||||
Model = chatModel.Id,
|
||||
|
||||
// Build the messages:
|
||||
// - First of all the system prompt
|
||||
// - Then none-empty user and AI messages
|
||||
Messages = [systemPrompt, ..chatThread.Blocks.Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text)).Select(n => new Message
|
||||
{
|
||||
Role = n.Role switch
|
||||
{
|
||||
ChatRole.USER => "user",
|
||||
ChatRole.AI => "assistant",
|
||||
ChatRole.AGENT => "assistant",
|
||||
ChatRole.SYSTEM => "system",
|
||||
|
||||
_ => "user",
|
||||
},
|
||||
|
||||
Content = n.Content switch
|
||||
{
|
||||
ContentText text => text.Text,
|
||||
_ => string.Empty,
|
||||
}
|
||||
}).ToList()],
|
||||
|
||||
Seed = chatThread.Seed,
|
||||
|
||||
// Right now, we only support streaming completions:
|
||||
Stream = true,
|
||||
}, JSON_SERIALIZER_OPTIONS);
|
||||
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
|
||||
|
||||
// Set the authorization header:
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
|
||||
|
||||
// Set the content:
|
||||
request.Content = new StringContent(xChatRequest, Encoding.UTF8, "application/json");
|
||||
return request;
|
||||
}
|
||||
|
||||
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("xAI", RequestBuilder, token))
|
||||
yield return content;
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
/// <inheritdoc />
|
||||
public override async IAsyncEnumerable<ImageURL> StreamImageCompletion(Model imageModel, string promptPositive, string promptNegative = FilterOperator.String.Empty, ImageURL referenceImageURL = default, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<IEnumerable<Model>> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
{
|
||||
return this.LoadModels(["grok-"], token, apiKeyProvisional);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<IEnumerable<Model>> GetImageModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
{
|
||||
return Task.FromResult<IEnumerable<Model>>([]);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<IEnumerable<Model>> GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default)
|
||||
{
|
||||
return Task.FromResult<IEnumerable<Model>>([]);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private async Task<IEnumerable<Model>> LoadModels(string[] prefixes, CancellationToken token, string? apiKeyProvisional = null)
|
||||
{
|
||||
var secretKey = apiKeyProvisional switch
|
||||
{
|
||||
not null => apiKeyProvisional,
|
||||
_ => await RUST_SERVICE.GetAPIKey(this) switch
|
||||
{
|
||||
{ Success: true } result => await result.Secret.Decrypt(ENCRYPTION),
|
||||
_ => null,
|
||||
}
|
||||
};
|
||||
|
||||
if (secretKey is null)
|
||||
return [];
|
||||
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, "models");
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", secretKey);
|
||||
|
||||
var response = await this.httpClient.SendAsync(request, token);
|
||||
if(!response.IsSuccessStatusCode)
|
||||
return [];
|
||||
|
||||
var modelResponse = await response.Content.ReadFromJsonAsync<ModelsResponse>(token);
|
||||
|
||||
//
|
||||
// The API does not return the alias model names, so we have to add them manually:
|
||||
// Right now, the only alias to add is `grok-2-latest`.
|
||||
//
|
||||
return modelResponse.Data.Where(model => prefixes.Any(prefix => model.Id.StartsWith(prefix, StringComparison.InvariantCulture)))
|
||||
.Concat([
|
||||
new Model
|
||||
{
|
||||
Id = "grok-2-latest",
|
||||
DisplayName = "Grok 2.0 (latest)",
|
||||
}
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
# v0.9.25, build 200 (2025-01-04 18:33 UTC)
|
||||
- Added xAI as a new provider. xAI provides their Grok models for generating content.
|
||||
- Improved the stop generation button behavior to ensure that the AI stops generating content immediately (which will save compute time, energy and financial resources).
|
||||
- Fixed the chat scrolling behavior after toggling the workspaces.
|
||||
- Fixed an issue when loading the today's bias chat, triggered by the bias assistant.
|
||||
- Restructured the streaming network code to be centralized out of the individual providers. This will allow for easier maintenance and updates in the future.
|
||||
+4
-4
@@ -1,9 +1,9 @@
|
||||
0.9.24
|
||||
2025-01-04 11:40:00 UTC
|
||||
199
|
||||
0.9.25
|
||||
2025-01-04 18:33:07 UTC
|
||||
200
|
||||
8.0.111 (commit f3835a7fb4)
|
||||
8.0.11 (commit 9cb3b725e3)
|
||||
1.83.0 (commit 90b35a623)
|
||||
7.15.0
|
||||
1.8.1
|
||||
d6521850e95, release
|
||||
e4b1ea32fb6, release
|
||||
Generated
+1
-1
@@ -2319,7 +2319,7 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
|
||||
|
||||
[[package]]
|
||||
name = "mindwork-ai-studio"
|
||||
version = "0.9.24"
|
||||
version = "0.9.25"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"arboard",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mindwork-ai-studio"
|
||||
version = "0.9.24"
|
||||
version = "0.9.25"
|
||||
edition = "2021"
|
||||
description = "MindWork AI Studio"
|
||||
authors = ["Thorsten Sommer"]
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
},
|
||||
"package": {
|
||||
"productName": "MindWork AI Studio",
|
||||
"version": "0.9.24"
|
||||
"version": "0.9.25"
|
||||
},
|
||||
"tauri": {
|
||||
"allowlist": {
|
||||
|
||||
Reference in new issue
Block a user