Improved the OpenAI provider (#548)
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Read metadata (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2025-09-03 10:08:04 +02:00
1 parent 4e167d58ea
commit 38ec098430
54 files changed
+1116 -220

No files matched your search

@@ -36,7 +36,7 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
};
// Prepare the AlibabaCloud HTTP chat request:
var alibabaCloudChatRequest = JsonSerializer.Serialize(new ChatRequest
var alibabaCloudChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
{
Model = chatModel.Id,
@@ -77,7 +77,7 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("AlibabaCloud", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, NoChatCompletionAnnotationStreamLine>("AlibabaCloud", RequestBuilder, token))
yield return content;
}
@@ -156,7 +156,9 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
Capability.AUDIO_INPUT, Capability.SPEECH_INPUT,
Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT, Capability.SPEECH_OUTPUT
Capability.TEXT_OUTPUT, Capability.SPEECH_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
// Check for Qwen 3:
@@ -166,7 +168,8 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING
Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
if(modelName.IndexOf("-vl-") is not -1)
@@ -174,6 +177,8 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
}
@@ -185,7 +190,8 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
}
@@ -197,7 +203,8 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING
Capability.ALWAYS_REASONING,
Capability.CHAT_COMPLETION_API,
];
}
@@ -207,7 +214,8 @@ public sealed class ProviderAlibabaCloud(ILogger logger) : BaseProvider("https:/
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
}
@@ -72,7 +72,7 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Anthropic", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine, NoChatCompletionAnnotationStreamLine>("Anthropic", RequestBuilder, token))
yield return content;
}
@@ -122,7 +122,9 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING];
Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// Claude 3.7 is able to do reasoning:
if(modelName.StartsWith("claude-3-7"))
@@ -130,7 +132,9 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING];
Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// All other 3.x models are able to process text and images as input:
if(modelName.StartsWith("claude-3-"))
@@ -138,13 +142,17 @@ public sealed class ProviderAnthropic(ILogger logger) : BaseProvider("https://ap
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING];
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// Any other model is able to process text only:
return [
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING];
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
}
#endregion
@@ -14,6 +14,21 @@ public readonly record struct ResponseStreamLine(string Type, int Index, Delta D
/// <inheritdoc />
public ContentStreamChunk GetContent() => new(this.Delta.Text, []);
#region Implementation of IAnnotationStreamLine
//
// Please note: Anthropic's API does not currently support sources in their
// OpenAI-compatible response stream.
//
/// <inheritdoc />
public bool ContainsSources() => false;
/// <inheritdoc />
public IList<ISource> GetSources() => [];
#endregion
}
/// <summary>
+246 -16
View File
@@ -3,6 +3,7 @@ using System.Runtime.CompilerServices;
using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Provider.OpenAI;
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
@@ -39,6 +40,7 @@ public abstract class BaseProvider : IProvider, ISecretId
protected static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
Converters = { new AnnotationConverter() }
};
/// <summary>
@@ -123,10 +125,12 @@ public abstract class BaseProvider : IProvider, ISecretId
break;
}
var errorBody = await nextResponse.Content.ReadAsStringAsync(token);
if (nextResponse.StatusCode is HttpStatusCode.Forbidden)
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Block, string.Format(TB("Tried to communicate with the LLM provider '{0}'. You might not be able to use this provider from your location. The provider message is: '{1}'"), this.InstanceName, nextResponse.ReasonPhrase)));
this.logger.LogError($"Failed request with status code {nextResponse.StatusCode} (message = '{nextResponse.ReasonPhrase}').");
this.logger.LogDebug($"Error body: {errorBody}");
errorMessage = nextResponse.ReasonPhrase;
break;
}
@@ -135,6 +139,7 @@ public abstract class BaseProvider : IProvider, ISecretId
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, string.Format(TB("Tried to communicate with the LLM provider '{0}'. The required message format might be changed. The provider message is: '{1}'"), this.InstanceName, nextResponse.ReasonPhrase)));
this.logger.LogError($"Failed request with status code {nextResponse.StatusCode} (message = '{nextResponse.ReasonPhrase}').");
this.logger.LogDebug($"Error body: {errorBody}");
errorMessage = nextResponse.ReasonPhrase;
break;
}
@@ -143,6 +148,7 @@ public abstract class BaseProvider : IProvider, ISecretId
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, string.Format(TB("Tried to communicate with the LLM provider '{0}'. Something was not found. The provider message is: '{1}'"), this.InstanceName, nextResponse.ReasonPhrase)));
this.logger.LogError($"Failed request with status code {nextResponse.StatusCode} (message = '{nextResponse.ReasonPhrase}').");
this.logger.LogDebug($"Error body: {errorBody}");
errorMessage = nextResponse.ReasonPhrase;
break;
}
@@ -151,6 +157,7 @@ public abstract class BaseProvider : IProvider, ISecretId
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Key, string.Format(TB("Tried to communicate with the LLM provider '{0}'. The API key might be invalid. The provider message is: '{1}'"), this.InstanceName, nextResponse.ReasonPhrase)));
this.logger.LogError($"Failed request with status code {nextResponse.StatusCode} (message = '{nextResponse.ReasonPhrase}').");
this.logger.LogDebug($"Error body: {errorBody}");
errorMessage = nextResponse.ReasonPhrase;
break;
}
@@ -159,6 +166,7 @@ public abstract class BaseProvider : IProvider, ISecretId
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, string.Format(TB("Tried to communicate with the LLM provider '{0}'. The server might be down or having issues. The provider message is: '{1}'"), this.InstanceName, nextResponse.ReasonPhrase)));
this.logger.LogError($"Failed request with status code {nextResponse.StatusCode} (message = '{nextResponse.ReasonPhrase}').");
this.logger.LogDebug($"Error body: {errorBody}");
errorMessage = nextResponse.ReasonPhrase;
break;
}
@@ -167,6 +175,7 @@ public abstract class BaseProvider : IProvider, ISecretId
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, string.Format(TB("Tried to communicate with the LLM provider '{0}'. The provider is overloaded. The message is: '{1}'"), this.InstanceName, nextResponse.ReasonPhrase)));
this.logger.LogError($"Failed request with status code {nextResponse.StatusCode} (message = '{nextResponse.ReasonPhrase}').");
this.logger.LogDebug($"Error body: {errorBody}");
errorMessage = nextResponse.ReasonPhrase;
break;
}
@@ -189,8 +198,20 @@ public abstract class BaseProvider : IProvider, ISecretId
return new HttpRateLimitedStreamResult(true, false, string.Empty, response);
}
protected async IAsyncEnumerable<ContentStreamChunk> StreamChatCompletionInternal<T>(string providerName, Func<Task<HttpRequestMessage>> requestBuilder, [EnumeratorCancellation] CancellationToken token = default) where T : struct, IResponseStreamLine
/// <summary>
/// Streams the chat completion from the provider using the Chat Completion API.
/// </summary>
/// <param name="providerName">The name of the provider.</param>
/// <param name="requestBuilder">A function that builds the request.</param>
/// <param name="token">The cancellation token to use.</param>
/// <typeparam name="TDelta">The type of the delta lines inside the stream.</typeparam>
/// <typeparam name="TAnnotation">The type of the annotation lines inside the stream.</typeparam>
/// <returns>The stream of content chunks.</returns>
protected async IAsyncEnumerable<ContentStreamChunk> StreamChatCompletionInternal<TDelta, TAnnotation>(string providerName, Func<Task<HttpRequestMessage>> requestBuilder, [EnumeratorCancellation] CancellationToken token = default) where TDelta : IResponseStreamLine where TAnnotation : IAnnotationStreamLine
{
// Check if annotations are supported:
var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
StreamReader? streamReader = null;
try
{
@@ -217,7 +238,9 @@ public abstract class BaseProvider : IProvider, ISecretId
if (streamReader is null)
yield break;
//
// Read the stream, line by line:
//
while (true)
{
try
@@ -240,7 +263,9 @@ public abstract class BaseProvider : IProvider, ISecretId
yield break;
}
//
// Read the next line:
//
string? line;
try
{
@@ -266,28 +291,233 @@ public abstract class BaseProvider : IProvider, ISecretId
if (line.StartsWith("data: [DONE]", StringComparison.InvariantCulture))
yield break;
T providerResponse;
//
// Process annotation lines:
//
if (annotationSupported && line.Contains("""
"annotations":[
""", StringComparison.InvariantCulture))
{
TAnnotation? providerResponse;
try
{
// We know that the line starts with "data: ". Hence, we can
// skip the first 6 characters to get the JSON data after that.
var jsonData = line[6..];
// Deserialize the JSON data:
providerResponse = JsonSerializer.Deserialize<TAnnotation>(jsonData, JSON_SERIALIZER_OPTIONS);
if (providerResponse is null)
continue;
}
catch
{
// Skip invalid JSON data:
continue;
}
// Skip empty responses:
if (!providerResponse.ContainsSources())
continue;
// Yield the response:
yield return new(string.Empty, providerResponse.GetSources());
}
//
// Process delta lines:
//
else
{
TDelta? providerResponse;
try
{
// We know that the line starts with "data: ". Hence, we can
// skip the first 6 characters to get the JSON data after that.
var jsonData = line[6..];
// Deserialize the JSON data:
providerResponse = JsonSerializer.Deserialize<TDelta>(jsonData, JSON_SERIALIZER_OPTIONS);
if (providerResponse is null)
continue;
}
catch
{
// Skip invalid JSON data:
continue;
}
// Skip empty responses:
if (!providerResponse.ContainsContent())
continue;
// Yield the response:
yield return providerResponse.GetContent();
}
}
streamReader.Dispose();
}
/// <summary>
/// Streams the chat completion from the provider using the Responses API.
/// </summary>
/// <param name="providerName">The name of the provider.</param>
/// <param name="requestBuilder">A function that builds the request.</param>
/// <param name="token">The cancellation token to use.</param>
/// <typeparam name="TDelta">The type of the delta lines inside the stream.</typeparam>
/// <typeparam name="TAnnotation">The type of the annotation lines inside the stream.</typeparam>
/// <returns>The stream of content chunks.</returns>
protected async IAsyncEnumerable<ContentStreamChunk> StreamResponsesInternal<TDelta, TAnnotation>(string providerName, Func<Task<HttpRequestMessage>> requestBuilder, [EnumeratorCancellation] CancellationToken token = default) where TDelta : IResponseStreamLine where TAnnotation : IAnnotationStreamLine
{
// Check if annotations are supported:
var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
StreamReader? streamReader = null;
try
{
// Send the request using exponential backoff:
var responseData = await this.SendRequest(requestBuilder, token);
if(responseData.IsFailedAfterAllRetries)
{
this.logger.LogError($"The {providerName} responses call failed: {responseData.ErrorMessage}");
yield break;
}
// Open the response stream:
var providerStream = await responseData.Response!.Content.ReadAsStreamAsync(token);
// Add a stream reader to read the stream, line by line:
streamReader = new StreamReader(providerStream);
}
catch(Exception e)
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Stream, string.Format(TB("Tried to communicate with the LLM provider '{0}'. There were some problems with the request. The provider message is: '{1}'"), this.InstanceName, e.Message)));
this.logger.LogError($"Failed to stream responses from {providerName} '{this.InstanceName}': {e.Message}");
}
if (streamReader is null)
yield break;
//
// Read the stream, line by line:
//
while (true)
{
try
{
// We know that the line starts with "data: ". Hence, we can
// skip the first 6 characters to get the JSON data after that.
var jsonData = line[6..];
// Deserialize the JSON data:
providerResponse = JsonSerializer.Deserialize<T>(jsonData, JSON_SERIALIZER_OPTIONS);
if(streamReader.EndOfStream)
break;
}
catch
catch (Exception e)
{
// Skip invalid JSON data:
continue;
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Stream, string.Format(TB("Tried to stream the LLM provider '{0}' answer. There were some problems with the stream. The message is: '{1}'"), this.InstanceName, e.Message)));
this.logger.LogWarning($"Failed to read the end-of-stream state from {providerName} '{this.InstanceName}': {e.Message}");
break;
}
// Check if the token is canceled:
if (token.IsCancellationRequested)
{
this.logger.LogWarning($"The user canceled the responses for {providerName} '{this.InstanceName}'.");
streamReader.Close();
yield break;
}
// Skip empty responses:
if (!providerResponse.ContainsContent())
continue;
//
// Read the next line:
//
string? line;
try
{
line = await streamReader.ReadLineAsync(token);
}
catch (Exception e)
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Stream, string.Format(TB("Tried to stream the LLM provider '{0}' answer. Was not able to read the stream. The message is: '{1}'"), this.InstanceName, e.Message)));
this.logger.LogError($"Failed to read the stream from {providerName} '{this.InstanceName}': {e.Message}");
break;
}
// Yield the response:
yield return providerResponse.GetContent();
// Skip empty lines:
if (string.IsNullOrWhiteSpace(line))
continue;
// Check if the line is the end of the stream:
if (line.StartsWith("event: response.completed", StringComparison.InvariantCulture))
yield break;
//
// Find delta lines:
//
if (line.StartsWith("""
data: {"type":"response.output_text.delta"
""", StringComparison.InvariantCulture))
{
TDelta? providerResponse;
try
{
// We know that the line starts with "data: ". Hence, we can
// skip the first 6 characters to get the JSON data after that.
var jsonData = line[6..];
// Deserialize the JSON data:
providerResponse = JsonSerializer.Deserialize<TDelta>(jsonData, JSON_SERIALIZER_OPTIONS);
if (providerResponse is null)
continue;
}
catch
{
// Skip invalid JSON data:
continue;
}
// Skip empty responses:
if (!providerResponse.ContainsContent())
continue;
// Yield the response:
yield return providerResponse.GetContent();
}
//
// Find annotation added lines:
//
else if (annotationSupported && line.StartsWith(
"""
data: {"type":"response.output_text.annotation.added"
""", StringComparison.InvariantCulture))
{
TAnnotation? providerResponse;
try
{
// We know that the line starts with "data: ". Hence, we can
// skip the first 6 characters to get the JSON data after that.
var jsonData = line[6..];
// Deserialize the JSON data:
providerResponse = JsonSerializer.Deserialize<TAnnotation>(jsonData, JSON_SERIALIZER_OPTIONS);
if (providerResponse is null)
continue;
}
catch
{
// Skip invalid JSON data:
continue;
}
// Skip empty responses:
if (!providerResponse.ContainsSources())
continue;
// Yield the response:
yield return new(string.Empty, providerResponse.GetSources());
}
}
streamReader.Dispose();
@@ -34,11 +34,17 @@ public static class CapabilitiesOpenSource
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// The old vision models cannot do function calling:
if (modelName.IndexOf("vision") is not -1)
return [Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.TEXT_OUTPUT];
return [
Capability.TEXT_INPUT,
Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
//
// All models >= 3.1 are able to do function calling:
@@ -53,10 +59,14 @@ public static class CapabilitiesOpenSource
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// All other llama models can only do text input and output:
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
return [
Capability.TEXT_INPUT, Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
}
//
@@ -66,9 +76,16 @@ public static class CapabilitiesOpenSource
{
if(modelName.IndexOf("deepseek-r1") is not -1 ||
modelName.IndexOf("deepseek r1") is not -1)
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, Capability.ALWAYS_REASONING];
return [
Capability.TEXT_INPUT, Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING,
Capability.CHAT_COMPLETION_API,
];
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
return [
Capability.TEXT_INPUT, Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
}
//
@@ -77,9 +94,16 @@ public static class CapabilitiesOpenSource
if (modelName.IndexOf("qwen") is not -1 || modelName.IndexOf("qwq") is not -1)
{
if (modelName.IndexOf("qwq") is not -1)
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, Capability.ALWAYS_REASONING];
return [
Capability.TEXT_INPUT, Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING,
Capability.CHAT_COMPLETION_API,
];
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
return [
Capability.TEXT_INPUT, Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
}
//
@@ -93,7 +117,8 @@ public static class CapabilitiesOpenSource
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
if (modelName.IndexOf("3.1") is not -1)
@@ -101,7 +126,8 @@ public static class CapabilitiesOpenSource
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// Default:
@@ -109,7 +135,8 @@ public static class CapabilitiesOpenSource
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
}
@@ -123,6 +150,7 @@ public static class CapabilitiesOpenSource
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
if(modelName.StartsWith("grok-3-mini"))
@@ -132,6 +160,7 @@ public static class CapabilitiesOpenSource
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
if(modelName.StartsWith("grok-3"))
@@ -141,10 +170,41 @@ public static class CapabilitiesOpenSource
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
}
//
// OpenAI models:
//
if (modelName.IndexOf("gpt-oss") is not -1 ||
modelName.IndexOf("gpt-3.5") is not -1)
{
if(modelName.IndexOf("gpt-oss") is not -1)
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.WEB_SEARCH,
Capability.CHAT_COMPLETION_API,
];
if(modelName.IndexOf("gpt-3.5") is not -1)
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
}
// Default:
return [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
return [
Capability.TEXT_INPUT, Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
}
}
@@ -94,4 +94,19 @@ public enum Capability
/// The AI model can perform function calling, such as invoking APIs or executing functions.
/// </summary>
FUNCTION_CALLING,
/// <summary>
/// The AI model can perform web search to retrieve information from the internet.
/// </summary>
WEB_SEARCH,
/// <summary>
/// The AI model is used via the Chat Completion API.
/// </summary>
CHAT_COMPLETION_API,
/// <summary>
/// The AI model is used via the Responses API.
/// </summary>
RESPONSES_API,
}
@@ -35,7 +35,7 @@ public sealed class ProviderDeepSeek(ILogger logger) : BaseProvider("https://api
};
// Prepare the DeepSeek HTTP chat request:
var deepSeekChatRequest = JsonSerializer.Serialize(new ChatRequest
var deepSeekChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
{
Model = chatModel.Id,
@@ -76,7 +76,7 @@ public sealed class ProviderDeepSeek(ILogger logger) : BaseProvider("https://api
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("DeepSeek", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, NoChatCompletionAnnotationStreamLine>("DeepSeek", RequestBuilder, token))
yield return content;
}
@@ -117,12 +117,14 @@ public sealed class ProviderDeepSeek(ILogger logger) : BaseProvider("https://api
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING,
Capability.CHAT_COMPLETION_API,
];
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
}
@@ -4,6 +4,7 @@ using System.Text;
using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Provider.OpenAI;
using AIStudio.Settings;
namespace AIStudio.Provider.Fireworks;
@@ -77,7 +78,7 @@ public class ProviderFireworks(ILogger logger) : BaseProvider("https://api.firew
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Fireworks", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine, ChatCompletionAnnotationStreamLine>("Fireworks", RequestBuilder, token))
yield return content;
}
@@ -15,6 +15,20 @@ public readonly record struct ResponseStreamLine(string Id, string Object, uint
/// <inheritdoc />
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []);
#region Implementation of IAnnotationStreamLine
//
// Currently, Fireworks does not provide source citations in their response stream.
//
/// <inheritdoc />
public bool ContainsSources() => false;
/// <inheritdoc />
public IList<ISource> GetSources() => [];
#endregion
}
/// <summary>
@@ -35,7 +35,7 @@ public sealed class ProviderGWDG(ILogger logger) : BaseProvider("https://chat-ai
};
// Prepare the GWDG HTTP chat request:
var gwdgChatRequest = JsonSerializer.Serialize(new ChatRequest
var gwdgChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
{
Model = chatModel.Id,
@@ -76,7 +76,7 @@ public sealed class ProviderGWDG(ILogger logger) : BaseProvider("https://chat-ai
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("GWDG", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, ChatCompletionAnnotationStreamLine>("GWDG", RequestBuilder, token))
yield return content;
}
@@ -78,7 +78,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Google", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, NoChatCompletionAnnotationStreamLine>("Google", RequestBuilder, token))
yield return content;
}
@@ -136,6 +136,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// Image generation:
@@ -146,6 +147,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
Capability.SPEECH_INPUT, Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT, Capability.IMAGE_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
// Realtime model:
@@ -158,6 +160,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
Capability.TEXT_OUTPUT, Capability.SPEECH_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// The 2.0 flash models cannot call functions:
@@ -168,6 +171,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
Capability.SPEECH_INPUT, Capability.VIDEO_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
// The old 1.0 pro vision model:
@@ -177,6 +181,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
// Default to all other Gemini models:
@@ -188,6 +193,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
}
@@ -199,6 +205,7 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
}
@@ -60,8 +60,6 @@ public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/o
_ => string.Empty,
}
}).ToList()],
Seed = chatThread.Seed,
// Right now, we only support streaming completions:
Stream = true,
@@ -80,7 +78,7 @@ public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/o
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Groq", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, ChatCompletionAnnotationStreamLine>("Groq", RequestBuilder, token))
yield return content;
}
@@ -35,7 +35,7 @@ public sealed class ProviderHelmholtz(ILogger logger) : BaseProvider("https://ap
};
// Prepare the Helmholtz HTTP chat request:
var helmholtzChatRequest = JsonSerializer.Serialize(new ChatRequest
var helmholtzChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
{
Model = chatModel.Id,
@@ -76,7 +76,7 @@ public sealed class ProviderHelmholtz(ILogger logger) : BaseProvider("https://ap
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Helmholtz", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, ChatCompletionAnnotationStreamLine>("Helmholtz", RequestBuilder, token))
yield return content;
}
@@ -40,7 +40,7 @@ public sealed class ProviderHuggingFace : BaseProvider
};
// Prepare the HuggingFace HTTP chat request:
var huggingfaceChatRequest = JsonSerializer.Serialize(new ChatRequest
var huggingfaceChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
{
Model = chatModel.Id,
@@ -81,7 +81,7 @@ public sealed class ProviderHuggingFace : BaseProvider
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("HuggingFace", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, ChatCompletionAnnotationStreamLine>("HuggingFace", RequestBuilder, token))
yield return content;
}
@@ -0,0 +1,19 @@
namespace AIStudio.Provider;
/// <summary>
/// A contract for a line in a response stream that can provide annotations such as sources.
/// </summary>
public interface IAnnotationStreamLine
{
/// <summary>
/// Checks if the response line contains any sources.
/// </summary>
/// <returns>True when the response line contains sources, false otherwise.</returns>
public bool ContainsSources();
/// <summary>
/// Gets the sources of the response line.
/// </summary>
/// <returns>The sources of the response line.</returns>
public IList<ISource> GetSources();
}
@@ -1,6 +1,9 @@
namespace AIStudio.Provider;
public interface IResponseStreamLine
/// <summary>
/// A contract for a streamed response line that may contain content and annotations.
/// </summary>
public interface IResponseStreamLine : IAnnotationStreamLine
{
/// <summary>
/// Checks if the response line contains any content.
@@ -13,16 +16,4 @@ public interface IResponseStreamLine
/// </summary>
/// <returns>The content of the response line.</returns>
public ContentStreamChunk GetContent();
/// <summary>
/// Checks if the response line contains any sources.
/// </summary>
/// <returns>True when the response line contains sources, false otherwise.</returns>
public bool ContainsSources() => false;
/// <summary>
/// Gets the sources of the response line.
/// </summary>
/// <returns>The sources of the response line.</returns>
public IList<ISource> GetSources() => [];
}
@@ -58,8 +58,6 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
_ => string.Empty,
}
}).ToList()],
RandomSeed = chatThread.Seed,
// Right now, we only support streaming completions:
Stream = true,
@@ -79,7 +77,7 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Mistral", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, NoChatCompletionAnnotationStreamLine>("Mistral", RequestBuilder, token))
yield return content;
}
@@ -134,6 +132,7 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// Mistral medium:
@@ -144,6 +143,7 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// Mistral small:
@@ -154,6 +154,7 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
// Mistral saba:
@@ -162,6 +163,7 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
// Default:
@@ -0,0 +1,15 @@
namespace AIStudio.Provider;
/// <summary>
/// A marker record indicating that no chat completion annotation line is expected in that stream.
/// </summary>
public sealed record NoChatCompletionAnnotationStreamLine : IAnnotationStreamLine
{
#region Implementation of IAnnotationStreamLine
public bool ContainsSources() => false;
public IList<ISource> GetSources() => [];
#endregion
}
@@ -0,0 +1,15 @@
namespace AIStudio.Provider;
/// <summary>
/// A marker record indicating that no annotation line is expected in that Responses API stream.
/// </summary>
public sealed record NoResponsesAnnotationStreamLine : IAnnotationStreamLine
{
#region Implementation of IAnnotationStreamLine
public bool ContainsSources() => false;
public IList<ISource> GetSources() => [];
#endregion
}
@@ -0,0 +1,7 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Represents an unknown annotation type.
/// </summary>
/// <param name="Type">The type of the unknown annotation.</param>
public sealed record AnnotatingUnknown(string Type) : Annotation(Type);
@@ -0,0 +1,10 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Base class for different types of annotations.
/// </summary>
/// <remarks>
/// We use this base class to represent various annotation types for all types of LLM providers.
/// </remarks>
/// <param name="Type">The type of the annotation.</param>
public abstract record Annotation(string Type);
@@ -0,0 +1,62 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Custom JSON converter for the annotation class to handle polymorphic deserialization.
/// </summary>
/// <remarks>
/// We use this converter for chat completion API and responses API annotation deserialization.
/// </remarks>
public sealed class AnnotationConverter : JsonConverter<Annotation>
{
public override Annotation? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
using var doc = JsonDocument.ParseValue(ref reader);
var root = doc.RootElement;
if (!root.TryGetProperty("type", out var typeElement))
return null;
var type = typeElement.GetString();
var rawText = root.GetRawText();
Annotation? annotation;
switch (type)
{
case "url_citation":
// Let's check the responses API data type first:
var responsesAnnotation = JsonSerializer.Deserialize<ResponsesAnnotatingUrlCitationData>(rawText, options);
// If it fails, let's try the chat completion API data type:
if(responsesAnnotation is null || string.IsNullOrWhiteSpace(responsesAnnotation.Title) || string.IsNullOrWhiteSpace(responsesAnnotation.URL))
{
// Try chat completion API data type:
var chatCompletionAnnotation = JsonSerializer.Deserialize<ChatCompletionAnnotatingURL>(rawText, options);
// If both fail, we return the unknown type:
if(chatCompletionAnnotation is null)
annotation = new AnnotatingUnknown(type);
else
annotation = chatCompletionAnnotation;
}
else
annotation = responsesAnnotation;
break;
default:
annotation = new AnnotatingUnknown(type ?? "unknown");
break;
}
return annotation;
}
public override void Write(Utf8JsonWriter writer, Annotation value, JsonSerializerOptions options)
{
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
}
@@ -0,0 +1,18 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// The OpenAI's legacy chat completion request model.
/// </summary>
/// <param name="Model">Which model to use for chat completion.</param>
/// <param name="Messages">The chat messages.</param>
/// <param name="Stream">Whether to stream the chat completion.</param>
public record ChatCompletionAPIRequest(
string Model,
IList<Message> Messages,
bool Stream
)
{
public ChatCompletionAPIRequest() : this(string.Empty, [], true)
{
}
}
@@ -0,0 +1,16 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data structure for URL annotation in chat completions.
/// </summary>
/// <remarks>
/// Although this class is not directly intended for the Responses API, it is
/// used there as a fallback solution. One day, one of the open source LLM
/// drivers may use this data structure for their responses API.
/// </remarks>
/// <param name="Type">The type of annotation, typically "url_citation".</param>
/// <param name="UrlCitation">The URL citation details.</param>
public sealed record ChatCompletionAnnotatingURL(
string Type,
ChatCompletionUrlCitationData UrlCitation
) : Annotation(Type);
@@ -0,0 +1,8 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data structure representing a choice in a chat completion annotation response.
/// </summary>
/// <param name="Index">The index of the choice.</param>
/// <param name="Delta">The delta information for the choice.</param>
public record ChatCompletionAnnotationChoice(int Index, ChatCompletionAnnotationDelta Delta);
@@ -0,0 +1,7 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data structure representing annotation deltas in chat completions.
/// </summary>
/// <param name="Annotations">The list of annotations, which can be null.</param>
public record ChatCompletionAnnotationDelta(IList<Annotation>? Annotations);
@@ -0,0 +1,57 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Represents a line of a chat completion annotation stream.
/// </summary>
/// <param name="Id">The unique identifier of the chat completion.</param>
/// <param name="Object">The type of object returned, typically "chat.completion".</param>
/// <param name="Created">The creation timestamp of the chat completion in Unix epoch format.</param>
/// <param name="Model">The model used for the chat completion.</param>
/// <param name="SystemFingerprint">The system fingerprint associated with the chat completion.</param>
/// <param name="Choices">The list of choices returned in the chat completion.</param>
public record ChatCompletionAnnotationStreamLine(string Id, string Object, uint Created, string Model, string SystemFingerprint, IList<ChatCompletionAnnotationChoice> Choices) : IAnnotationStreamLine
{
#region Implementation of IAnnotationStreamLine
/// <inheritdoc />
public bool ContainsSources() => this.Choices.Any(choice => choice.Delta.Annotations is not null && choice.Delta.Annotations.Any(annotation => annotation is not AnnotatingUnknown));
/// <inheritdoc />
public IList<ISource> GetSources()
{
var sources = new List<ISource>();
foreach (var choice in this.Choices)
{
if (choice.Delta.Annotations is null)
continue;
// Iterate through all annotations:
foreach (var annotation in choice.Delta.Annotations)
{
// Check if the annotation is of the expected type and extract the source information:
if (annotation is ChatCompletionAnnotatingURL urlAnnotation)
sources.Add(new Source(urlAnnotation.UrlCitation.Title, urlAnnotation.UrlCitation.URL));
//
// Check for the unexpected annotation type of the Responses API.
//
// This seems weird at first. But there are two possibilities why this could happen:
// - Anyone of the open source providers such as ollama, LM Studio, etc. could
// implement & use the Responses API data structures for annotations in their
// chat completion endpoint.
//
// - Our custom JSON converter checks for the Responses API data type first. If it
// fails, it checks for the chat completion API data type. So, when the Responses
// API data type is valid, it will be deserialized into that type, even though
// we are calling the chat completion endpoint.
//
if (annotation is ResponsesAnnotatingUrlCitationData citationData)
sources.Add(new Source(citationData.Title, citationData.URL));
}
}
return sources;
}
#endregion
}
@@ -0,0 +1,13 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data model for a choice made by the AI.
/// </summary>
/// <param name="Index">The index of the choice.</param>
/// <param name="Delta">The delta text of the choice.</param>
public record ChatCompletionChoice(int Index, ChatCompletionDelta Delta)
{
public ChatCompletionChoice() : this(0, new (string.Empty))
{
}
}
@@ -0,0 +1,12 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// The delta text of a choice.
/// </summary>
/// <param name="Content">The content of the delta text.</param>
public record ChatCompletionDelta(string Content)
{
public ChatCompletionDelta() : this(string.Empty)
{
}
}
@@ -0,0 +1,45 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data model for a delta line in the chat completion response stream.
/// </summary>
/// <param name="Id">The id of the response.</param>
/// <param name="Object">The object describing the response.</param>
/// <param name="Created">The timestamp of the response.</param>
/// <param name="Model">The model used for the response.</param>
/// <param name="SystemFingerprint">The system fingerprint; together with the seed, this allows you to reproduce the response.</param>
/// <param name="Choices">The choices made by the AI.</param>
public record ChatCompletionDeltaStreamLine(string Id, string Object, uint Created, string Model, string SystemFingerprint, IList<ChatCompletionChoice> Choices) : IResponseStreamLine
{
public ChatCompletionDeltaStreamLine() : this(string.Empty, string.Empty, 0, string.Empty, string.Empty, [])
{
}
/// <inheritdoc />
public bool ContainsContent() => this.Choices.Count > 0;
/// <inheritdoc />
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []);
#region Implementation of IAnnotationStreamLine
//
// Please note that there are multiple options where LLM providers might stream sources:
//
// - As part of the delta content while streaming. That would be part of this class.
// - By using a dedicated stream event and data structure. That would be another class implementing IResponseStreamLine.
//
// Right now, OpenAI uses the latter approach, so we don't have any sources here. And
// because no other provider does it yet, we don't have any implementation here either.
//
// One example where sources are part of the delta content is the Perplexity provider.
//
/// <inheritdoc />
public bool ContainsSources() => false;
/// <inheritdoc />
public IList<ISource> GetSources() => [];
#endregion
}
@@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Represents citation data for a URL in a chat completion response.
/// </summary>
/// <param name="EndIndex">The end index of the citation in the response text.</param>
/// <param name="StartIndex">The start index of the citation in the response text.</param>
/// <param name="Title">The title of the cited source.</param>
/// <param name="URL">The URL of the cited source.</param>
public sealed record ChatCompletionUrlCitationData(
int EndIndex,
int StartIndex,
string Title,
string URL);
@@ -1,21 +0,0 @@
using System.ComponentModel.DataAnnotations;
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// The OpenAI chat request model.
/// </summary>
/// <param name="Model">Which model to use for chat completion.</param>
/// <param name="Messages">The chat messages.</param>
/// <param name="Stream">Whether to stream the chat completion.</param>
/// <param name="Seed">The seed for the chat completion.</param>
/// <param name="FrequencyPenalty">The frequency penalty for the chat completion.</param>
public readonly record struct ChatRequest(
string Model,
IList<Message> Messages,
bool Stream,
int Seed,
[Range(-2.0f, 2.0f)]
float FrequencyPenalty
);
@@ -30,16 +30,20 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
yield break;
// Unfortunately, OpenAI changed the name of the system prompt based on the model.
// All models that start with "o" (the omni aka reasoning models) and all GPT4o models
// have the system prompt named "developer". All other models have the system prompt
// named "system". We need to check this to get the correct system prompt.
// All models that start with "o" (the omni aka reasoning models), all GPT4o models,
// and all newer models have the system prompt named "developer". All other models
// have the system prompt named "system". We need to check this to get the correct
// system prompt.
//
// To complicate it even more: The early versions of reasoning models, which are released
// before the 17th of December 2024, have no system prompt at all. We need to check this
// as well.
// Apply the basic rule first:
var systemPromptRole = chatModel.Id.StartsWith('o') || chatModel.Id.Contains("4o") ? "developer" : "system";
var systemPromptRole =
chatModel.Id.StartsWith('o') ||
chatModel.Id.StartsWith("gpt-5", StringComparison.Ordinal) ||
chatModel.Id.Contains("4o") ? "developer" : "system";
// Check if the model is an early version of the reasoning models:
systemPromptRole = chatModel.Id switch
@@ -51,53 +55,113 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
_ => systemPromptRole,
};
this.logger.LogInformation($"Using the system prompt role '{systemPromptRole}' for model '{chatModel.Id}'.");
// Read the model capabilities:
var modelCapabilities = this.GetModelCapabilities(chatModel);
// Check if we are using the Responses API or the Chat Completion API:
var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API);
// Prepare the request path based on the API we are using:
var requestPath = usingResponsesAPI ? "responses" : "chat/completions";
this.logger.LogInformation("Using the system prompt role '{SystemPromptRole}' and the '{RequestPath}' API for model '{ChatModelId}'.", systemPromptRole, requestPath, chatModel.Id);
// Prepare the system prompt:
var systemPrompt = new Message
{
Role = systemPromptRole,
Content = chatThread.PrepareSystemPrompt(settingsManager, chatThread, this.logger),
};
// Prepare the OpenAI HTTP chat request:
var openAIChatRequest = JsonSerializer.Serialize(new ChatRequest
//
// Prepare the tools we want to use:
//
IList<Tool> tools = modelCapabilities.Contains(Capability.WEB_SEARCH) switch
{
Model = chatModel.Id,
// Build the messages:
// - First of all the system prompt
// - Then none-empty user and AI messages
Messages = [systemPrompt, ..chatThread.Blocks.Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text)).Select(n => new Message
true => [ Tools.WEB_SEARCH ],
_ => []
};
//
// Create the request: either for the Responses API or the Chat Completion API
//
var openAIChatRequest = usingResponsesAPI switch
{
// Chat Completion API request:
false => JsonSerializer.Serialize(new ChatCompletionAPIRequest
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => systemPromptRole,
_ => "user",
},
Content = n.Content switch
{
ContentText text => text.Text,
_ => string.Empty,
}
}).ToList()],
Seed = chatThread.Seed,
Model = chatModel.Id,
// Right now, we only support streaming completions:
Stream = true,
}, JSON_SERIALIZER_OPTIONS);
// Build the messages:
// - First of all the system prompt
// - Then none-empty user and AI messages
Messages = [systemPrompt, ..chatThread.Blocks.Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text)).Select(n => new Message
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => systemPromptRole,
_ => "user",
},
Content = n.Content switch
{
ContentText text => text.Text,
_ => string.Empty,
}
}).ToList()],
// Right now, we only support streaming completions:
Stream = true,
}, JSON_SERIALIZER_OPTIONS),
// Responses API request:
true => JsonSerializer.Serialize(new ResponsesAPIRequest
{
Model = chatModel.Id,
// Build the messages:
// - First of all the system prompt
// - Then none-empty user and AI messages
Input = [systemPrompt, ..chatThread.Blocks.Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text)).Select(n => new Message
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => systemPromptRole,
_ => "user",
},
Content = n.Content switch
{
ContentText text => text.Text,
_ => string.Empty,
}
}).ToList()],
// Right now, we only support streaming completions:
Stream = true,
// We do not want to store any data on OpenAI's servers:
Store = false,
// Tools we want to use:
Tools = tools,
}, JSON_SERIALIZER_OPTIONS),
};
async Task<HttpRequestMessage> RequestBuilder()
{
// Build the HTTP post request:
var request = new HttpRequestMessage(HttpMethod.Post, "chat/completions");
var request = new HttpRequestMessage(HttpMethod.Post, requestPath);
// Set the authorization header:
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION));
@@ -106,29 +170,35 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
request.Content = new StringContent(openAIChatRequest, Encoding.UTF8, "application/json");
return request;
}
if (usingResponsesAPI)
await foreach (var content in this.StreamResponsesInternal<ResponsesDeltaStreamLine, ResponsesAnnotationStreamLine>("OpenAI", RequestBuilder, token))
yield return content;
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("OpenAI", RequestBuilder, token))
yield return content;
else
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, ChatCompletionAnnotationStreamLine>("OpenAI", RequestBuilder, token))
yield return content;
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
/// <inheritdoc />
public override async IAsyncEnumerable<ImageURL> StreamImageCompletion(Model imageModel, string promptPositive, string promptNegative = FilterOperator.String.Empty, ImageURL referenceImageURL = default, [EnumeratorCancellation] CancellationToken token = default)
{
yield break;
}
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
/// <inheritdoc />
public override async Task<IEnumerable<Model>> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
{
var models = await this.LoadModels(["gpt-", "o1-", "o3-", "o4-"], token, apiKeyProvisional);
var models = await this.LoadModels(["chatgpt-", "gpt-", "o1-", "o3-", "o4-"], token, apiKeyProvisional);
return models.Where(model => !model.Id.Contains("image", StringComparison.OrdinalIgnoreCase) &&
!model.Id.Contains("realtime", StringComparison.OrdinalIgnoreCase) &&
!model.Id.Contains("audio", StringComparison.OrdinalIgnoreCase) &&
!model.Id.Contains("tts", StringComparison.OrdinalIgnoreCase) &&
!model.Id.Contains("transcribe", StringComparison.OrdinalIgnoreCase) &&
!model.Id.Contains("o1-pro", StringComparison.OrdinalIgnoreCase));
!model.Id.Contains("transcribe", StringComparison.OrdinalIgnoreCase));
}
/// <inheritdoc />
@@ -147,6 +217,26 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
{
var modelName = model.Id.ToLowerInvariant().AsSpan();
if (modelName is "gpt-4o-search-preview")
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.WEB_SEARCH,
Capability.CHAT_COMPLETION_API,
];
if (modelName is "gpt-4o-mini-search-preview")
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.WEB_SEARCH,
Capability.CHAT_COMPLETION_API,
];
if (modelName.StartsWith("o1-mini"))
return
[
@@ -154,32 +244,63 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING,
Capability.CHAT_COMPLETION_API,
];
if(modelName is "gpt-3.5-turbo")
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.RESPONSES_API,
];
if(modelName.StartsWith("gpt-3.5"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API,
];
if (modelName.StartsWith("chatgpt-4o-"))
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.RESPONSES_API,
];
if (modelName.StartsWith("o3-mini"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING,
Capability.RESPONSES_API,
];
if (modelName.StartsWith("o4-mini") || modelName.StartsWith("o1") || modelName.StartsWith("o3"))
if (modelName.StartsWith("o4-mini") || modelName.StartsWith("o3"))
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING,
Capability.WEB_SEARCH,
Capability.RESPONSES_API,
];
if(modelName.StartsWith("gpt-3.5"))
if (modelName.StartsWith("o1"))
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
];
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING,
Capability.RESPONSES_API,
];
if(modelName.StartsWith("gpt-4-turbo"))
return
@@ -187,7 +308,8 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING
Capability.FUNCTION_CALLING,
Capability.RESPONSES_API,
];
if(modelName is "gpt-4" || modelName.StartsWith("gpt-4-"))
@@ -195,14 +317,37 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.RESPONSES_API,
];
if(modelName.StartsWith("gpt-5-nano"))
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING, Capability.ALWAYS_REASONING,
Capability.RESPONSES_API,
];
if(modelName is "gpt-5" || modelName.StartsWith("gpt-5-"))
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING, Capability.ALWAYS_REASONING,
Capability.WEB_SEARCH,
Capability.RESPONSES_API,
];
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.RESPONSES_API,
];
}
@@ -1,32 +0,0 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data model for a line in the response stream, for streaming completions.
/// </summary>
/// <param name="Id">The id of the response.</param>
/// <param name="Object">The object describing the response.</param>
/// <param name="Created">The timestamp of the response.</param>
/// <param name="Model">The model used for the response.</param>
/// <param name="SystemFingerprint">The system fingerprint; together with the seed, this allows you to reproduce the response.</param>
/// <param name="Choices">The choices made by the AI.</param>
public readonly record struct ResponseStreamLine(string Id, string Object, uint Created, string Model, string SystemFingerprint, IList<Choice> Choices) : IResponseStreamLine
{
/// <inheritdoc />
public bool ContainsContent() => this != default && this.Choices.Count > 0;
/// <inheritdoc />
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []);
}
/// <summary>
/// Data model for a choice made by the AI.
/// </summary>
/// <param name="Index">The index of the choice.</param>
/// <param name="Delta">The delta text of the choice.</param>
public readonly record struct Choice(int Index, Delta Delta);
/// <summary>
/// The delta text of a choice.
/// </summary>
/// <param name="Content">The content of the delta text.</param>
public readonly record struct Delta(string Content);
@@ -0,0 +1,21 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// The request body for the Responses API.
/// </summary>
/// <param name="Model">Which model to use.</param>
/// <param name="Input">The chat messages.</param>
/// <param name="Stream">Whether to stream the response.</param>
/// <param name="Store">Whether to store the response on the server (usually OpenAI's infrastructure).</param>
/// <param name="Tools">The tools to use for the request.</param>
public record ResponsesAPIRequest(
string Model,
IList<Message> Input,
bool Stream,
bool Store,
IList<Tool> Tools)
{
public ResponsesAPIRequest() : this(string.Empty, [], true, false, [])
{
}
}
@@ -0,0 +1,16 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data structure for URL citation annotations in the OpenAI Responses API.
/// </summary>
/// <param name="Type">The type of annotation, typically "url_citation".</param>
/// <param name="EndIndex">The end index of the annotated text in the response.</param>
/// <param name="StartIndex">The start index of the annotated text in the response.</param>
/// <param name="Title">The title of the cited URL.</param>
/// <param name="URL">The URL being cited.</param>
public sealed record ResponsesAnnotatingUrlCitationData(
string Type,
int EndIndex,
int StartIndex,
string Title,
string URL) : Annotation(Type);
@@ -0,0 +1,45 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data structure for a line in the response stream of the Responses API, containing an annotation.
/// </summary>
/// <param name="Type">The type of the annotation.</param>
/// <param name="AnnotationIndex">The continuous index of the annotation in the response.</param>
/// <param name="Annotation">The annotation details.</param>
public sealed record ResponsesAnnotationStreamLine(string Type, int AnnotationIndex, Annotation Annotation) : IAnnotationStreamLine
{
#region Implementation of IAnnotationStreamLine
/// <inheritdoc />
public bool ContainsSources()
{
return this.Annotation is not AnnotatingUnknown;
}
/// <inheritdoc />
public IList<ISource> GetSources()
{
//
// Check for the unexpected annotation type of the chat completion API.
//
// This seems weird at first. But there are two possibilities why this could happen:
// - Anyone of the open source providers such as ollama, LM Studio, etc. could
// implement and use the chat completion API data structures for annotations in their
// Responses API endpoint.
//
// - Our custom JSON converter checks for all possible annotation data types. So,
// when the streamed data is valid for any annotation type, it will be deserialized
// into that type, even though we are calling the Responses API endpoint.
//
if (this.Annotation is ChatCompletionAnnotatingURL urlAnnotation)
return [new Source(urlAnnotation.UrlCitation.Title, urlAnnotation.UrlCitation.URL)];
// Check for the expected annotation type of the Responses API:
if (this.Annotation is ResponsesAnnotatingUrlCitationData urlCitationData)
return [new Source(urlCitationData.Title, urlCitationData.URL)];
return [];
}
#endregion
}
@@ -0,0 +1,39 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Data model for a delta line in the Response API chat completion stream.
/// </summary>
/// <param name="Type">The type of the response.</param>
/// <param name="Delta">The delta content of the response.</param>
public record ResponsesDeltaStreamLine(
string Type,
string Delta) : IResponseStreamLine
{
#region Implementation of IResponseStreamLine
/// <inheritdoc />
public bool ContainsContent() => !string.IsNullOrWhiteSpace(this.Delta);
/// <inheritdoc />
public ContentStreamChunk GetContent() => new(this.Delta, this.GetSources());
//
// Please note that there are multiple options where LLM providers might stream sources:
//
// - As part of the delta content while streaming. That would be part of this class.
// - By using a dedicated stream event and data structure. That would be another class implementing IResponseStreamLine.
//
// Right now, OpenAI uses the latter approach, so we don't have any sources here. And
// because no other provider does it yet, we don't have any implementation here either.
//
// One example where sources are part of the delta content is the Perplexity provider.
//
/// <inheritdoc />
public bool ContainsSources() => false;
/// <inheritdoc />
public IList<ISource> GetSources() => [];
#endregion
}
@@ -0,0 +1,12 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Represents a tool used by the AI model.
/// </summary>
/// <remarks>
/// Right now, only our OpenAI provider is using tools. Thus, this class is located in the
/// OpenAI namespace. In the future, when other providers also support tools, this class can
/// be moved into the provider namespace.
/// </remarks>
/// <param name="Type">The type of the tool.</param>
public record Tool(string Type);
@@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Known tools for LLM providers.
/// </summary>
/// <remarks>
/// Right now, only our OpenAI provider is using tools. Thus, this class is located in the
/// OpenAI namespace. In the future, when other providers also support tools, this class can
/// be moved into the provider namespace.
/// </remarks>
public static class Tools
{
public static readonly Tool WEB_SEARCH = new("web_search");
}
@@ -0,0 +1,8 @@
namespace AIStudio.Provider.Perplexity;
/// <summary>
/// Data model for a choice made by the AI.
/// </summary>
/// <param name="Index">The index of the choice.</param>
/// <param name="Delta">The delta text of the choice.</param>
public readonly record struct Choice(int Index, Delta Delta);
@@ -0,0 +1,7 @@
namespace AIStudio.Provider.Perplexity;
/// <summary>
/// The delta text of a choice.
/// </summary>
/// <param name="Content">The content of the delta text.</param>
public readonly record struct Delta(string Content);
@@ -44,7 +44,7 @@ public sealed class ProviderPerplexity(ILogger logger) : BaseProvider("https://a
};
// Prepare the Perplexity HTTP chat request:
var perplexityChatRequest = JsonSerializer.Serialize(new ChatRequest
var perplexityChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
{
Model = chatModel.Id,
@@ -85,7 +85,7 @@ public sealed class ProviderPerplexity(ILogger logger) : BaseProvider("https://a
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("Perplexity", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine, NoChatCompletionAnnotationStreamLine>("Perplexity", RequestBuilder, token))
yield return content;
}
@@ -130,6 +130,8 @@ public sealed class ProviderPerplexity(ILogger logger) : BaseProvider("https://a
Capability.IMAGE_OUTPUT,
Capability.ALWAYS_REASONING,
Capability.WEB_SEARCH,
Capability.CHAT_COMPLETION_API,
];
return
@@ -139,6 +141,9 @@ public sealed class ProviderPerplexity(ILogger logger) : BaseProvider("https://a
Capability.TEXT_OUTPUT,
Capability.IMAGE_OUTPUT,
Capability.WEB_SEARCH,
Capability.CHAT_COMPLETION_API,
];
}
@@ -22,24 +22,4 @@ public readonly record struct ResponseStreamLine(string Id, string Object, uint
/// <inheritdoc />
public IList<ISource> GetSources() => this.SearchResults.Cast<ISource>().ToList();
}
/// <summary>
/// Data model for a choice made by the AI.
/// </summary>
/// <param name="Index">The index of the choice.</param>
/// <param name="Delta">The delta text of the choice.</param>
public readonly record struct Choice(int Index, Delta Delta);
/// <summary>
/// The delta text of a choice.
/// </summary>
/// <param name="Content">The content of the delta text.</param>
public readonly record struct Delta(string Content);
/// <summary>
/// Data model for a search result.
/// </summary>
/// <param name="Title">The title of the search result.</param>
/// <param name="URL">The URL of the search result.</param>
public sealed record SearchResult(string Title, string URL) : Source(Title, URL);
}
@@ -0,0 +1,8 @@
namespace AIStudio.Provider.Perplexity;
/// <summary>
/// Data model for a search result.
/// </summary>
/// <param name="Title">The title of the search result.</param>
/// <param name="URL">The URL of the search result.</param>
public sealed record SearchResult(string Title, string URL) : Source(Title, URL);
@@ -75,7 +75,7 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("self-hosted provider", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, ChatCompletionAnnotationStreamLine>("self-hosted provider", RequestBuilder, token))
yield return content;
}
@@ -35,7 +35,7 @@ public sealed class ProviderX(ILogger logger) : BaseProvider("https://api.x.ai/v
};
// Prepare the xAI HTTP chat request:
var xChatRequest = JsonSerializer.Serialize(new ChatRequest
var xChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
{
Model = chatModel.Id,
@@ -60,8 +60,6 @@ public sealed class ProviderX(ILogger logger) : BaseProvider("https://api.x.ai/v
_ => string.Empty,
}
}).ToList()],
Seed = chatThread.Seed,
// Right now, we only support streaming completions:
Stream = true,
@@ -80,7 +78,7 @@ public sealed class ProviderX(ILogger logger) : BaseProvider("https://api.x.ai/v
return request;
}
await foreach (var content in this.StreamChatCompletionInternal<ResponseStreamLine>("xAI", RequestBuilder, token))
await foreach (var content in this.StreamChatCompletionInternal<ChatCompletionDeltaStreamLine, NoChatCompletionAnnotationStreamLine>("xAI", RequestBuilder, token))
yield return content;
}