diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index 5897b98..f4846e9 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -60,21 +60,24 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap Stream = true, }, JSON_SERIALIZER_OPTIONS); - async Task RequestBuilder() + StreamReader? streamReader = null; + try { - // Build the HTTP post request: - var request = new HttpRequestMessage(HttpMethod.Post, "messages"); + async Task RequestBuilder() + { + // Build the HTTP post request: + var request = new HttpRequestMessage(HttpMethod.Post, "messages"); - // Set the authorization header: - request.Headers.Add("x-api-key", await requestedSecret.Secret.Decrypt(ENCRYPTION)); + // Set the authorization header: + request.Headers.Add("x-api-key", await requestedSecret.Secret.Decrypt(ENCRYPTION)); - // Set the Anthropic version: - request.Headers.Add("anthropic-version", "2023-06-01"); + // Set the Anthropic version: + request.Headers.Add("anthropic-version", "2023-06-01"); - // Set the content: - request.Content = new StringContent(chatRequest, Encoding.UTF8, "application/json"); - return request; - } + // Set the content: + request.Content = new StringContent(chatRequest, Encoding.UTF8, "application/json"); + return request; + } // Send the request using exponential backoff: var responseData = await this.SendRequest(RequestBuilder, token); @@ -84,21 +87,49 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap yield break; } - // Open the response stream: - var stream = await responseData.Response!.Content.ReadAsStreamAsync(token); + // Open the response stream: + var stream = await responseData.Response!.Content.ReadAsStreamAsync(token); - // Add a stream reader to read the stream, line by line: - var streamReader = new StreamReader(stream); + // Add a stream reader to read the stream, line by line: + streamReader = new StreamReader(stream); + } + catch (Exception e) + { + this.logger.LogError($"Failed to stream chat completion from Anthropic '{this.InstanceName}': {e.Message}"); + } + + if (streamReader is null) + yield break; // Read the stream, line by line: - while(!streamReader.EndOfStream) + while(true) { + try + { + if(streamReader.EndOfStream) + break; + } + catch (Exception e) + { + this.logger.LogWarning($"Failed to read the end-of-stream state from Anthropic '{this.InstanceName}': {e.Message}"); + break; + } + // Check if the token is canceled: if(token.IsCancellationRequested) yield break; // Read the next line: - var line = await streamReader.ReadLineAsync(token); + string? line; + try + { + line = await streamReader.ReadLineAsync(token); + } + catch (Exception e) + { + this.logger.LogError($"Failed to read the stream from Anthropic '{this.InstanceName}': {e.Message}"); + break; + } // Skip empty lines: if(string.IsNullOrWhiteSpace(line)) diff --git a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs index b8b0357..1dd49d3 100644 --- a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs +++ b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs @@ -69,18 +69,21 @@ public class ProviderFireworks(ILogger logger) : BaseProvider("https://api.firew Stream = true, }, JSON_SERIALIZER_OPTIONS); - async Task RequestBuilder() + StreamReader? streamReader = null; + try { - // Build the HTTP post request: - var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); + async Task RequestBuilder() + { + // Build the HTTP post request: + var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); - // Set the authorization header: - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); + // Set the authorization header: + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); - // Set the content: - request.Content = new StringContent(fireworksChatRequest, Encoding.UTF8, "application/json"); - return request; - } + // Set the content: + request.Content = new StringContent(fireworksChatRequest, Encoding.UTF8, "application/json"); + return request; + } // Send the request using exponential backoff: var responseData = await this.SendRequest(RequestBuilder, token); @@ -90,21 +93,49 @@ public class ProviderFireworks(ILogger logger) : BaseProvider("https://api.firew yield break; } - // Open the response stream: - var fireworksStream = await responseData.Response!.Content.ReadAsStreamAsync(token); + // Open the response stream: + var fireworksStream = await responseData.Response!.Content.ReadAsStreamAsync(token); - // Add a stream reader to read the stream, line by line: - var streamReader = new StreamReader(fireworksStream); + // Add a stream reader to read the stream, line by line: + streamReader = new StreamReader(fireworksStream); + } + catch (Exception e) + { + this.logger.LogError($"Failed to stream chat completion from Fireworks '{this.InstanceName}': {e.Message}"); + } + + if (streamReader is null) + yield break; // Read the stream, line by line: - while(!streamReader.EndOfStream) + while(true) { + try + { + if(streamReader.EndOfStream) + break; + } + catch (Exception e) + { + this.logger.LogWarning($"Failed to read the end-of-stream state from Fireworks '{this.InstanceName}': {e.Message}"); + break; + } + // Check if the token is canceled: if(token.IsCancellationRequested) yield break; // Read the next line: - var line = await streamReader.ReadLineAsync(token); + string? line; + try + { + line = await streamReader.ReadLineAsync(token); + } + catch (Exception e) + { + this.logger.LogError($"Failed to read the stream from Fireworks '{this.InstanceName}': {e.Message}"); + break; + } // Skip empty lines: if(string.IsNullOrWhiteSpace(line)) diff --git a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs index 5110061..6293b08 100644 --- a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs +++ b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs @@ -70,18 +70,21 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela Stream = true, }, JSON_SERIALIZER_OPTIONS); - async Task RequestBuilder() + StreamReader? streamReader = null; + try { - // Build the HTTP post request: - var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); + async Task RequestBuilder() + { + // Build the HTTP post request: + var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); - // Set the authorization header: - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); + // Set the authorization header: + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); - // Set the content: - request.Content = new StringContent(geminiChatRequest, Encoding.UTF8, "application/json"); - return request; - } + // Set the content: + request.Content = new StringContent(geminiChatRequest, Encoding.UTF8, "application/json"); + return request; + } // Send the request using exponential backoff: var responseData = await this.SendRequest(RequestBuilder, token); @@ -91,21 +94,49 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela yield break; } - // Open the response stream: - var geminiStream = await responseData.Response!.Content.ReadAsStreamAsync(token); + // Open the response stream: + var geminiStream = await responseData.Response!.Content.ReadAsStreamAsync(token); - // Add a stream reader to read the stream, line by line: - var streamReader = new StreamReader(geminiStream); + // Add a stream reader to read the stream, line by line: + streamReader = new StreamReader(geminiStream); + } + catch (Exception e) + { + this.logger.LogError($"Failed to stream chat completion from Google '{this.InstanceName}': {e.Message}"); + } + + if (streamReader is null) + yield break; // Read the stream, line by line: - while(!streamReader.EndOfStream) + while(true) { + try + { + if(streamReader.EndOfStream) + break; + } + catch (Exception e) + { + this.logger.LogWarning($"Failed to read the end-of-stream state from Google '{this.InstanceName}': {e.Message}"); + break; + } + // Check if the token is canceled: if(token.IsCancellationRequested) yield break; // Read the next line: - var line = await streamReader.ReadLineAsync(token); + string? line; + try + { + line = await streamReader.ReadLineAsync(token); + } + catch (Exception e) + { + this.logger.LogError($"Failed to read the stream from Google '{this.InstanceName}': {e.Message}"); + break; + } // Skip empty lines: if(string.IsNullOrWhiteSpace(line)) diff --git a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs index ae4e5ff..7fd2931 100644 --- a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs +++ b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs @@ -72,18 +72,21 @@ public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/o Stream = true, }, JSON_SERIALIZER_OPTIONS); - async Task RequestBuilder() + StreamReader? streamReader = null; + try { - // Build the HTTP post request: - var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); + async Task RequestBuilder() + { + // Build the HTTP post request: + var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); - // Set the authorization header: - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); + // Set the authorization header: + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); - // Set the content: - request.Content = new StringContent(groqChatRequest, Encoding.UTF8, "application/json"); - return request; - } + // Set the content: + request.Content = new StringContent(groqChatRequest, Encoding.UTF8, "application/json"); + return request; + } // Send the request using exponential backoff: var responseData = await this.SendRequest(RequestBuilder, token); @@ -93,21 +96,49 @@ public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/o yield break; } - // Open the response stream: - var groqStream = await responseData.Response!.Content.ReadAsStreamAsync(token); + // Open the response stream: + var groqStream = await responseData.Response!.Content.ReadAsStreamAsync(token); - // Add a stream reader to read the stream, line by line: - var streamReader = new StreamReader(groqStream); + // Add a stream reader to read the stream, line by line: + streamReader = new StreamReader(groqStream); + } + catch (Exception e) + { + this.logger.LogError($"Failed to stream chat completion from Groq '{this.InstanceName}': {e.Message}"); + } + + if (streamReader is null) + yield break; // Read the stream, line by line: - while(!streamReader.EndOfStream) + while(true) { + try + { + if(streamReader.EndOfStream) + break; + } + catch (Exception e) + { + this.logger.LogWarning($"Failed to read the end-of-stream state from Groq '{this.InstanceName}': {e.Message}"); + break; + } + // Check if the token is canceled: if(token.IsCancellationRequested) yield break; // Read the next line: - var line = await streamReader.ReadLineAsync(token); + string? line; + try + { + line = await streamReader.ReadLineAsync(token); + } + catch (Exception e) + { + this.logger.LogError($"Failed to read the stream from Groq '{this.InstanceName}': {e.Message}"); + break; + } // Skip empty lines: if(string.IsNullOrWhiteSpace(line)) diff --git a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs index 561d316..b83dfa4 100644 --- a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs +++ b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs @@ -71,18 +71,21 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api. SafePrompt = false, }, JSON_SERIALIZER_OPTIONS); - async Task RequestBuilder() + StreamReader? streamReader = null; + try { - // Build the HTTP post request: - var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); + async Task RequestBuilder() + { + // Build the HTTP post request: + var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); - // Set the authorization header: - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); + // Set the authorization header: + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); - // Set the content: - request.Content = new StringContent(mistralChatRequest, Encoding.UTF8, "application/json"); - return request; - } + // Set the content: + request.Content = new StringContent(mistralChatRequest, Encoding.UTF8, "application/json"); + return request; + } // Send the request using exponential backoff: var responseData = await this.SendRequest(RequestBuilder, token); @@ -92,21 +95,49 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api. yield break; } - // Open the response stream: - var mistralStream = await responseData.Response!.Content.ReadAsStreamAsync(token); + // Open the response stream: + var mistralStream = await responseData.Response!.Content.ReadAsStreamAsync(token); - // Add a stream reader to read the stream, line by line: - var streamReader = new StreamReader(mistralStream); + // Add a stream reader to read the stream, line by line: + streamReader = new StreamReader(mistralStream); + } + catch (Exception e) + { + this.logger.LogError($"Failed to stream chat completion from Mistral '{this.InstanceName}': {e.Message}"); + } + + if (streamReader is null) + yield break; // Read the stream, line by line: - while(!streamReader.EndOfStream) + while(true) { + try + { + if(streamReader.EndOfStream) + break; + } + catch (Exception e) + { + this.logger.LogWarning($"Failed to read the end-of-stream state from Mistral '{this.InstanceName}': {e.Message}"); + break; + } + // Check if the token is canceled: if(token.IsCancellationRequested) yield break; // Read the next line: - var line = await streamReader.ReadLineAsync(token); + string? line; + try + { + line = await streamReader.ReadLineAsync(token); + } + catch (Exception e) + { + this.logger.LogError($"Failed to read the stream from Mistral '{this.InstanceName}': {e.Message}"); + break; + } // Skip empty lines: if(string.IsNullOrWhiteSpace(line)) diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index 62007f7..7635366 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -99,18 +99,21 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o Stream = true, }, JSON_SERIALIZER_OPTIONS); - async Task RequestBuilder() + StreamReader? streamReader = null; + try { - // Build the HTTP post request: - var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); + async Task RequestBuilder() + { + // Build the HTTP post request: + var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions"); - // Set the authorization header: - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); + // Set the authorization header: + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); - // Set the content: - request.Content = new StringContent(openAIChatRequest, Encoding.UTF8, "application/json"); - return request; - } + // Set the content: + request.Content = new StringContent(openAIChatRequest, Encoding.UTF8, "application/json"); + return request; + } // Send the request using exponential backoff: var responseData = await this.SendRequest(RequestBuilder, token); @@ -120,21 +123,49 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o yield break; } - // Open the response stream: - var openAIStream = await responseData.Response!.Content.ReadAsStreamAsync(token); + // Open the response stream: + var openAIStream = await responseData.Response!.Content.ReadAsStreamAsync(token); - // Add a stream reader to read the stream, line by line: - var streamReader = new StreamReader(openAIStream); + // Add a stream reader to read the stream, line by line: + streamReader = new StreamReader(openAIStream); + } + catch (Exception e) + { + this.logger.LogError($"Failed to stream chat completion from OpenAI '{this.InstanceName}': {e.Message}"); + } + + if (streamReader is null) + yield break; // Read the stream, line by line: - while(!streamReader.EndOfStream) + while(true) { + try + { + if(streamReader.EndOfStream) + break; + } + catch (Exception e) + { + this.logger.LogWarning($"Failed to read the end-of-stream state from OpenAI '{this.InstanceName}': {e.Message}"); + break; + } + // Check if the token is canceled: if(token.IsCancellationRequested) yield break; // Read the next line: - var line = await streamReader.ReadLineAsync(token); + string? line; + try + { + line = await streamReader.ReadLineAsync(token); + } + catch (Exception e) + { + this.logger.LogError($"Failed to read the stream from OpenAI '{this.InstanceName}': {e.Message}"); + break; + } // Skip empty lines: if(string.IsNullOrWhiteSpace(line)) diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs index ea9de2b..eb3c98b 100644 --- a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs +++ b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs @@ -67,7 +67,7 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam MaxTokens = -1, }, JSON_SERIALIZER_OPTIONS); - StreamReader? streamReader = default; + StreamReader? streamReader = null; try { async Task RequestBuilder() @@ -91,7 +91,7 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam this.logger.LogError($"Self-hosted provider's chat completion failed: {responseData.ErrorMessage}"); yield break; } - + // Open the response stream: var providerStream = await responseData.Response!.Content.ReadAsStreamAsync(token); @@ -103,55 +103,77 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam this.logger.LogError($"Failed to stream chat completion from self-hosted provider '{this.InstanceName}': {e.Message}"); } - if (streamReader is not null) + if (streamReader is null) + yield break; + + // Read the stream, line by line: + while (true) { - // Read the stream, line by line: - while (!streamReader.EndOfStream) + try { - // Check if the token is canceled: - if (token.IsCancellationRequested) - yield break; - - // Read the next line: - var line = await streamReader.ReadLineAsync(token); - - // Skip empty lines: - if (string.IsNullOrWhiteSpace(line)) - continue; - - // Skip lines that do not start with "data: ". Regard - // to the specification, we only want to read the data lines: - if (!line.StartsWith("data: ", StringComparison.InvariantCulture)) - continue; - - // Check if the line is the end of the stream: - if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture)) - yield break; - - ResponseStreamLine providerResponse; - try - { - // We know that the line starts with "data: ". Hence, we can - // skip the first 6 characters to get the JSON data after that. - var jsonData = line[6..]; - - // Deserialize the JSON data: - providerResponse = JsonSerializer.Deserialize(jsonData, JSON_SERIALIZER_OPTIONS); - } - catch - { - // Skip invalid JSON data: - continue; - } - - // Skip empty responses: - if (providerResponse == default || providerResponse.Choices.Count == 0) - continue; - - // Yield the response: - yield return providerResponse.Choices[0].Delta.Content; + if(streamReader.EndOfStream) + break; } + catch (Exception e) + { + this.logger.LogWarning($"Failed to read the end-of-stream state from self-hosted provider '{this.InstanceName}': {e.Message}"); + break; + } + + // Check if the token is canceled: + if (token.IsCancellationRequested) + yield break; + + // Read the next line: + string? line; + try + { + line = await streamReader.ReadLineAsync(token); + } + catch (Exception e) + { + this.logger.LogError($"Failed to read the stream from self-hosted provider '{this.InstanceName}': {e.Message}"); + break; + } + + // Skip empty lines: + if (string.IsNullOrWhiteSpace(line)) + continue; + + // Skip lines that do not start with "data: ". Regard + // to the specification, we only want to read the data lines: + if (!line.StartsWith("data: ", StringComparison.InvariantCulture)) + continue; + + // Check if the line is the end of the stream: + if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture)) + yield break; + + ResponseStreamLine providerResponse; + try + { + // We know that the line starts with "data: ". Hence, we can + // skip the first 6 characters to get the JSON data after that. + var jsonData = line[6..]; + + // Deserialize the JSON data: + providerResponse = JsonSerializer.Deserialize(jsonData, JSON_SERIALIZER_OPTIONS); + } + catch + { + // Skip invalid JSON data: + continue; + } + + // Skip empty responses: + if (providerResponse == default || providerResponse.Choices.Count == 0) + continue; + + // Yield the response: + yield return providerResponse.Choices[0].Delta.Content; } + + streamReader.Dispose(); } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.9.24.md b/app/MindWork AI Studio/wwwroot/changelog/v0.9.24.md index 58342a7..536b791 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v0.9.24.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v0.9.24.md @@ -2,5 +2,6 @@ - Added a button to remove a message from the chat thread. - Added a button to regenerate the last AI response. - Added a button to edit the last user message. -- Added a button to stop the AI from generating a response. +- Added a button to stop the AI from generating more tokens. +- Improved the stream handling for all providers. The stream handling is now very resilient and handles all kinds of network issues. - Fixed a streaming bug that was particularly visible with self-hosted providers. \ No newline at end of file