Added support for images (#609)
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Build and Release / Read metadata (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2025-12-30 18:30:32 +01:00
1 parent ed4c7d215a
commit e9485ca8ea
80 files changed
+1300 -495

No files matched your search

@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.AlibabaCloud;
public sealed class ProviderAlibabaCloud() : BaseProvider("https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", LOGGER)
public sealed class ProviderAlibabaCloud() : BaseProvider(LLMProviders.ALIBABA_CLOUD, "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", LOGGER)
{
private static readonly ILogger<ProviderAlibabaCloud> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderAlibabaCloud>();
@@ -40,24 +40,7 @@ public sealed class ProviderAlibabaCloud() : BaseProvider("https://dashscope-int
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the AlibabaCloud HTTP chat request:
var alibabaCloudChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
@@ -0,0 +1,9 @@
namespace AIStudio.Provider.Anthropic;
public interface ISubContentImageSource
{
/// <summary>
/// The type of the sub-content image.
/// </summary>
public SubContentImageType Type { get; }
}
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Anthropic;
public sealed class ProviderAnthropic() : BaseProvider("https://api.anthropic.com/v1/", LOGGER)
public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, "https://api.anthropic.com/v1/", LOGGER)
{
private static readonly ILogger<ProviderAnthropic> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderAnthropic>();
@@ -31,9 +31,11 @@ public sealed class ProviderAnthropic() : BaseProvider("https://api.anthropic.co
var apiParameters = this.ParseAdditionalApiParameters("system");
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
var messages = await chatThread.Blocks.BuildMessagesAsync(
this.Provider, chatModel,
// Anthropic-specific role mapping:
role => role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
@@ -41,13 +43,26 @@ public sealed class ProviderAnthropic() : BaseProvider("https://api.anthropic.co
_ => "user",
},
Content = n.Content switch
// Anthropic uses the standard text sub-content:
text => new SubContentText
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
Text = text,
},
// Anthropic-specific image sub-content:
async attachment => new SubContentImage
{
Source = new SubContentBase64Image
{
Data = await attachment.TryAsBase64(token: token) is (true, var base64Content)
? base64Content
: string.Empty,
MediaType = attachment.DetermineMimeType(),
}
}
});
);
// Prepare the Anthropic HTTP chat request:
var chatRequest = JsonSerializer.Serialize(new ChatRequest
@@ -0,0 +1,10 @@
namespace AIStudio.Provider.Anthropic;
public record SubContentBase64Image : ISubContentImageSource
{
public SubContentImageType Type => SubContentImageType.BASE64;
public string MediaType { get; init; } = string.Empty;
public string Data { get; init; } = string.Empty;
}
@@ -0,0 +1,10 @@
using AIStudio.Provider.OpenAI;
namespace AIStudio.Provider.Anthropic;
public record SubContentImage(SubContentType Type, ISubContentImageSource Source) : ISubContent
{
public SubContentImage() : this(SubContentType.IMAGE, new SubContentImageUrl())
{
}
}
@@ -0,0 +1,32 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace AIStudio.Provider.Anthropic;
/// <summary>
/// Custom JSON converter for the ISubContentImageSource interface to handle polymorphic serialization.
/// </summary>
/// <remarks>
/// This converter ensures that when serializing ISubContentImageSource objects, all properties
/// of the concrete implementation (e.g., SubContentBase64Image, SubContentImageUrl) are serialized,
/// not just the properties defined in the ISubContentImageSource interface.
/// </remarks>
public sealed class SubContentImageSourceConverter : JsonConverter<ISubContentImageSource>
{
private static readonly ILogger<SubContentImageSourceConverter> LOGGER = Program.LOGGER_FACTORY.CreateLogger<SubContentImageSourceConverter>();
public override ISubContentImageSource? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
// Deserialization is not needed for request objects, as sub-content image sources are only serialized
// when sending requests to LLM providers.
LOGGER.LogError("Deserializing ISubContentImageSource is not supported. This converter is only used for serializing request messages.");
return null;
}
public override void Write(Utf8JsonWriter writer, ISubContentImageSource value, JsonSerializerOptions options)
{
// Serialize the actual concrete type (e.g., SubContentBase64Image, SubContentImageUrl) instead of just the ISubContentImageSource interface.
// This ensures all properties of the concrete type are included in the JSON output.
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
}
@@ -0,0 +1,7 @@
namespace AIStudio.Provider.Anthropic;
public enum SubContentImageType
{
URL,
BASE64
}
@@ -0,0 +1,8 @@
namespace AIStudio.Provider.Anthropic;
public record SubContentImageUrl : ISubContentImageSource
{
public SubContentImageType Type => SubContentImageType.URL;
public string Url { get; init; } = string.Empty;
}
@@ -1,8 +1,10 @@
using System.Net;
using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Text.Json.Serialization;
using AIStudio.Chat;
using AIStudio.Provider.Anthropic;
using AIStudio.Provider.OpenAI;
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
@@ -40,18 +42,28 @@ public abstract class BaseProvider : IProvider, ISecretId
protected static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
Converters = { new AnnotationConverter(), new MessageBaseConverter() },
Converters =
{
new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower),
new AnnotationConverter(),
new MessageBaseConverter(),
new SubContentConverter(),
new SubContentImageSourceConverter(),
new SubContentImageUrlConverter(),
},
AllowTrailingCommas = false
};
/// <summary>
/// Constructor for the base provider.
/// </summary>
/// <param name="provider">The provider enum value.</param>
/// <param name="url">The base URL for the provider.</param>
/// <param name="logger">The logger to use.</param>
protected BaseProvider(string url, ILogger logger)
protected BaseProvider(LLMProviders provider, string url, ILogger logger)
{
this.logger = logger;
this.Provider = provider;
// Set the base URL:
this.httpClient.BaseAddress = new(url);
@@ -59,6 +71,9 @@ public abstract class BaseProvider : IProvider, ISecretId
#region Handling of IProvider, which all providers must implement
/// <inheritdoc />
public LLMProviders Provider { get; }
/// <inheritdoc />
public abstract string Id { get; }
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.DeepSeek;
public sealed class ProviderDeepSeek() : BaseProvider("https://api.deepseek.com/", LOGGER)
public sealed class ProviderDeepSeek() : BaseProvider(LLMProviders.DEEP_SEEK, "https://api.deepseek.com/", LOGGER)
{
private static readonly ILogger<ProviderDeepSeek> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderDeepSeek>();
@@ -40,24 +40,7 @@ public sealed class ProviderDeepSeek() : BaseProvider("https://api.deepseek.com/
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel);
// Prepare the DeepSeek HTTP chat request:
var deepSeekChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Fireworks;
public class ProviderFireworks() : BaseProvider("https://api.fireworks.ai/inference/v1/", LOGGER)
public class ProviderFireworks() : BaseProvider(LLMProviders.FIREWORKS, "https://api.fireworks.ai/inference/v1/", LOGGER)
{
private static readonly ILogger<ProviderFireworks> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderFireworks>();
@@ -40,24 +40,7 @@ public class ProviderFireworks() : BaseProvider("https://api.fireworks.ai/infere
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the Fireworks HTTP chat request:
var fireworksChatRequest = JsonSerializer.Serialize(new ChatRequest
@@ -1,13 +0,0 @@
namespace AIStudio.Provider.Fireworks;
/// <summary>
/// Chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>
public record TextMessage(string Content, string Role) : IMessage<string>
{
public TextMessage() : this(string.Empty, string.Empty)
{
}
}
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.GWDG;
public sealed class ProviderGWDG() : BaseProvider("https://chat-ai.academiccloud.de/v1/", LOGGER)
public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, "https://chat-ai.academiccloud.de/v1/", LOGGER)
{
private static readonly ILogger<ProviderGWDG> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGWDG>();
@@ -40,24 +40,7 @@ public sealed class ProviderGWDG() : BaseProvider("https://chat-ai.academiccloud
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the GWDG HTTP chat request:
var gwdgChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Google;
public class ProviderGoogle() : BaseProvider("https://generativelanguage.googleapis.com/v1beta/", LOGGER)
public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, "https://generativelanguage.googleapis.com/v1beta/", LOGGER)
{
private static readonly ILogger<ProviderGoogle> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGoogle>();
@@ -40,24 +40,7 @@ public class ProviderGoogle() : BaseProvider("https://generativelanguage.googlea
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the Google HTTP chat request:
var geminiChatRequest = JsonSerializer.Serialize(new ChatRequest
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Groq;
public class ProviderGroq() : BaseProvider("https://api.groq.com/openai/v1/", LOGGER)
public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, "https://api.groq.com/openai/v1/", LOGGER)
{
private static readonly ILogger<ProviderGroq> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGroq>();
@@ -40,24 +40,7 @@ public class ProviderGroq() : BaseProvider("https://api.groq.com/openai/v1/", LO
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the OpenAI HTTP chat request:
var groqChatRequest = JsonSerializer.Serialize(new ChatRequest
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Helmholtz;
public sealed class ProviderHelmholtz() : BaseProvider("https://api.helmholtz-blablador.fz-juelich.de/v1/", LOGGER)
public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, "https://api.helmholtz-blablador.fz-juelich.de/v1/", LOGGER)
{
private static readonly ILogger<ProviderHelmholtz> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderHelmholtz>();
@@ -40,24 +40,7 @@ public sealed class ProviderHelmholtz() : BaseProvider("https://api.helmholtz-bl
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the Helmholtz HTTP chat request:
var helmholtzChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
@@ -13,9 +13,9 @@ public sealed class ProviderHuggingFace : BaseProvider
{
private static readonly ILogger<ProviderHuggingFace> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderHuggingFace>();
public ProviderHuggingFace(HFInferenceProvider hfProvider, Model model) : base($"https://router.huggingface.co/{hfProvider.Endpoints(model)}", LOGGER)
public ProviderHuggingFace(HFInferenceProvider hfProvider, Model model) : base(LLMProviders.HUGGINGFACE, $"https://router.huggingface.co/{hfProvider.Endpoints(model)}", LOGGER)
{
LOGGER.LogInformation($"We use the inferende provider '{hfProvider}'. Thus we use the base URL 'https://router.huggingface.co/{hfProvider.Endpoints(model)}'.");
LOGGER.LogInformation($"We use the inference provider '{hfProvider}'. Thus we use the base URL 'https://router.huggingface.co/{hfProvider.Endpoints(model)}'.");
}
#region Implementation of IProvider
@@ -45,24 +45,7 @@ public sealed class ProviderHuggingFace : BaseProvider
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var message = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var message = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the HuggingFace HTTP chat request:
var huggingfaceChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
@@ -8,6 +8,11 @@ namespace AIStudio.Provider;
/// </summary>
public interface IProvider
{
/// <summary>
/// The provider type.
/// </summary>
public LLMProviders Provider { get; }
/// <summary>
/// The provider's ID.
/// </summary>
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Mistral;
public sealed class ProviderMistral() : BaseProvider("https://api.mistral.ai/v1/", LOGGER)
public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, "https://api.mistral.ai/v1/", LOGGER)
{
private static readonly ILogger<ProviderMistral> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderMistral>();
@@ -38,24 +38,7 @@ public sealed class ProviderMistral() : BaseProvider("https://api.mistral.ai/v1/
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel);
// Prepare the Mistral HTTP chat request:
var mistralChatRequest = JsonSerializer.Serialize(new ChatRequest
@@ -1,13 +0,0 @@
namespace AIStudio.Provider.Mistral;
/// <summary>
/// Text chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>
public record TextMessage(string Content, string Role) : IMessage<string>
{
public TextMessage() : this(string.Empty, string.Empty)
{
}
}
@@ -9,6 +9,8 @@ public class NoProvider : IProvider
{
#region Implementation of IProvider
public LLMProviders Provider => LLMProviders.NONE;
public string Id => "none";
public string InstanceName { get; set; } = "None";
@@ -0,0 +1,12 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Contract for sub-content in multimodal messages.
/// </summary>
public interface ISubContent
{
/// <summary>
/// The type of the sub-content.
/// </summary>
public SubContentType Type { get; init; }
}
@@ -0,0 +1,19 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Contract for nested image URL sub-content.
/// </summary>
/// <remarks>
/// Some providers use a nested object format for image URLs:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This interface represents the inner object with the "url" property.
/// </remarks>
public interface ISubContentImageUrl
{
/// <summary>
/// The URL or base64-encoded data URI of the image.
/// </summary>
public string Url { get; init; }
}
@@ -0,0 +1,13 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// A multimodal chat message model that can contain various types of content.
/// </summary>
/// <param name="Content">The list of sub-contents in the message.</param>
/// <param name="Role">The role of the message.</param>
public record MultimodalMessage(List<ISubContent> Content, string Role) : IMessage<List<ISubContent>>
{
public MultimodalMessage() : this([], string.Empty)
{
}
}
@@ -11,7 +11,7 @@ namespace AIStudio.Provider.OpenAI;
/// <summary>
/// The OpenAI provider.
/// </summary>
public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/", LOGGER)
public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, "https://api.openai.com/v1/", LOGGER)
{
private static readonly ILogger<ProviderOpenAI> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderOpenAI>();
@@ -59,7 +59,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
};
// Read the model capabilities:
var modelCapabilities = ProviderExtensions.GetModelCapabilitiesOpenAI(chatModel);
var modelCapabilities = this.Provider.GetModelCapabilities(chatModel);
// Check if we are using the Responses API or the Chat Completion API:
var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API);
@@ -90,9 +90,11 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
var apiParameters = this.ParseAdditionalApiParameters("input", "store", "tools");
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
var messages = await chatThread.Blocks.BuildMessagesAsync(
this.Provider, chatModel,
// OpenAI-specific role mapping:
role => role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
@@ -102,12 +104,46 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
_ => "user",
},
Content = n.Content switch
// OpenAI's text sub-content depends on the model, whether we are using
// the Responses API or the Chat Completion API:
text => usingResponsesAPI switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Responses API uses INPUT_TEXT:
true => new SubContentInputText
{
Text = text,
},
// Chat Completion API uses TEXT:
false => new SubContentText
{
Text = text,
},
},
// OpenAI's image sub-content depends on the model as well,
// whether we are using the Responses API or the Chat Completion API:
async attachment => usingResponsesAPI switch
{
// Responses API uses INPUT_IMAGE:
true => new SubContentInputImage
{
ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content)
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
: string.Empty,
},
// Chat Completion API uses IMAGE_URL:
false => new SubContentImageUrlNested
{
ImageUrl = new SubContentImageUrlData
{
Url = await attachment.TryAsBase64(token: token) is (true, var base64Content)
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
: string.Empty,
},
}
});
//
// Create the request: either for the Responses API or the Chat Completion API
@@ -149,7 +185,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
}, JSON_SERIALIZER_OPTIONS),
};
async Task<HttpRequestMessage> RequestBuilder()
{
// Build the HTTP post request:
@@ -0,0 +1,11 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image sub-content for multimodal messages.
/// </summary>
public record SubContentImageUrl(SubContentType Type, string ImageUrl) : ISubContent
{
public SubContentImageUrl() : this(SubContentType.IMAGE_URL, string.Empty)
{
}
}
@@ -0,0 +1,17 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Represents the inner object of a nested image URL sub-content.
/// </summary>
/// <remarks>
/// This record is used when the provider expects the format:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This class represents the inner <c>{ "url": "..." }</c> part.
/// </remarks>
public record SubContentImageUrlData : ISubContentImageUrl
{
/// <inheritdoc />
public string Url { get; init; } = string.Empty;
}
@@ -0,0 +1,18 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image sub-content for multimodal messages using nested URL format.
/// </summary>
/// <remarks>
/// This record is used when the provider expects the format:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// Used by LM Studio, VLLM, and other OpenAI-compatible providers.
/// </remarks>
public record SubContentImageUrlNested(SubContentType Type, ISubContentImageUrl ImageUrl) : ISubContent
{
public SubContentImageUrlNested() : this(SubContentType.IMAGE_URL, new SubContentImageUrlData())
{
}
}
@@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image input sub-content for multimodal messages.
/// </summary>
/// <remarks>
/// Right now, this is used only by OpenAI in its responses API.
/// </remarks>
public record SubContentInputImage(SubContentType Type, string ImageUrl) : ISubContent
{
public SubContentInputImage() : this(SubContentType.INPUT_IMAGE, string.Empty)
{
}
}
@@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Text input sub-content for multimodal messages.
/// </summary>
/// <remarks>
/// Right now, this is used only by OpenAI in its responses API.
/// </remarks>
public record SubContentInputText(SubContentType Type, string Text) : ISubContent
{
public SubContentInputText() : this(SubContentType.INPUT_TEXT, string.Empty)
{
}
}
@@ -0,0 +1,11 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Text sub-content for multimodal messages.
/// </summary>
public record SubContentText(SubContentType Type, string Text) : ISubContent
{
public SubContentText() : this(SubContentType.TEXT, string.Empty)
{
}
}
@@ -1,7 +1,7 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Chat message model.
/// Standard text-based chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.OpenRouter;
public sealed class ProviderOpenRouter() : BaseProvider("https://openrouter.ai/api/v1/", LOGGER)
public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER, "https://openrouter.ai/api/v1/", LOGGER)
{
private const string PROJECT_WEBSITE = "https://github.com/MindWorkAI/AI-Studio";
private const string PROJECT_NAME = "MindWork AI Studio";
@@ -43,24 +43,7 @@ public sealed class ProviderOpenRouter() : BaseProvider("https://openrouter.ai/a
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the OpenRouter HTTP chat request:
var openRouterChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Perplexity;
public sealed class ProviderPerplexity() : BaseProvider("https://api.perplexity.ai/", LOGGER)
public sealed class ProviderPerplexity() : BaseProvider(LLMProviders.PERPLEXITY, "https://api.perplexity.ai/", LOGGER)
{
private static readonly ILogger<ProviderPerplexity> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderPerplexity>();
@@ -49,24 +49,7 @@ public sealed class ProviderPerplexity() : BaseProvider("https://api.perplexity.
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage()
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the Perplexity HTTP chat request:
var perplexityChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.SelfHosted;
public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvider($"{hostname}{host.BaseURL()}", LOGGER)
public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvider(LLMProviders.SELF_HOSTED, $"{hostname}{host.BaseURL()}", LOGGER)
{
private static readonly ILogger<ProviderSelfHosted> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderSelfHosted>();
@@ -34,26 +34,15 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide
// Parse the API parameters:
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
// Build the list of messages. The image format depends on the host:
// - Ollama uses the direct image URL format: { "type": "image_url", "image_url": "data:..." }
// - LM Studio, vLLM, and llama.cpp use the nested image URL format: { "type": "image_url", "image_url": { "url": "data:..." } }
var messages = host switch
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel),
_ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel),
};
// Prepare the OpenAI HTTP chat request:
var providerChatRequest = JsonSerializer.Serialize(new ChatRequest
@@ -1,13 +0,0 @@
namespace AIStudio.Provider.SelfHosted;
/// <summary>
/// Chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>
public record TextMessage(string Content, string Role) : IMessage<string>
{
public TextMessage() : this(string.Empty, string.Empty)
{
}
}
@@ -0,0 +1,34 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using AIStudio.Provider.OpenAI;
namespace AIStudio.Provider;
/// <summary>
/// Custom JSON converter for the ISubContent interface to handle polymorphic serialization.
/// </summary>
/// <remarks>
/// This converter ensures that when serializing ISubContent objects, all properties
/// of the concrete implementation (e.g., SubContentText, SubContentImageUrl) are serialized,
/// not just the properties defined in the ISubContent interface.
/// </remarks>
public sealed class SubContentConverter : JsonConverter<ISubContent>
{
private static readonly ILogger<SubContentConverter> LOGGER = Program.LOGGER_FACTORY.CreateLogger<SubContentConverter>();
public override ISubContent? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
// Deserialization is not needed for request objects, as sub-content is only serialized
// when sending requests to LLM providers.
LOGGER.LogError("Deserializing ISubContent is not supported. This converter is only used for serializing request messages.");
return null;
}
public override void Write(Utf8JsonWriter writer, ISubContent value, JsonSerializerOptions options)
{
// Serialize the actual concrete type (e.g., SubContentText, SubContentImageUrl) instead of just the ISubContent interface.
// This ensures all properties of the concrete type are included in the JSON output.
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
}
@@ -0,0 +1,34 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using AIStudio.Provider.OpenAI;
namespace AIStudio.Provider;
/// <summary>
/// Custom JSON converter for the ISubContentImageUrl interface to handle polymorphic serialization.
/// </summary>
/// <remarks>
/// This converter ensures that when serializing ISubContentImageUrl objects, all properties
/// of the concrete implementation (e.g., SubContentImageUrlData) are serialized,
/// not just the properties defined in the ISubContentImageUrl interface.
/// </remarks>
public sealed class SubContentImageUrlConverter : JsonConverter<ISubContentImageUrl>
{
private static readonly ILogger<SubContentImageUrlConverter> LOGGER = Program.LOGGER_FACTORY.CreateLogger<SubContentImageUrlConverter>();
public override ISubContentImageUrl? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
// Deserialization is not needed for request objects, as sub-content image URLs are only serialized
// when sending requests to LLM providers.
LOGGER.LogError("Deserializing ISubContentImageUrl is not supported. This converter is only used for serializing request messages.");
return null;
}
public override void Write(Utf8JsonWriter writer, ISubContentImageUrl value, JsonSerializerOptions options)
{
// Serialize the actual concrete type (e.g., SubContentImageUrlData) instead of just the ISubContentImageUrl interface.
// This ensures all properties of the concrete type are included in the JSON output.
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
}
@@ -0,0 +1,39 @@
namespace AIStudio.Provider;
/// <summary>
/// Sub content types for OpenAI-compatible API interactions when using multimodal messages.
/// </summary>
public enum SubContentType
{
/// <summary>
/// Default type for user prompts in multimodal messages. This type is supported across all providers.
/// </summary>
TEXT,
/// <summary>
/// Right now only supported by OpenAI and it's responses API. Even other providers that support multimodal messages
/// and the responses API do not support this type. They use TEXT instead.
/// </summary>
INPUT_TEXT,
/// <summary>
/// Right now only supported by OpenAI and it's responses API. Even other providers that support multimodal messages
/// and the responses API do not support this type. They use IMAGE_URL instead.
/// </summary>
INPUT_IMAGE,
/// <summary>
/// Default type for images in multimodal messages. This type is supported across all providers.
/// </summary>
IMAGE_URL,
/// <summary>
/// The image type is used exclusively by Anthropic's messages API.
/// </summary>
IMAGE,
/// <summary>
/// Right now only supported by OpenAI (responses & chat completion API), Google (chat completions API), and Mistral (chat completions API).
/// </summary>
INPUT_AUDIO,
}
+2 -19
View File
@@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.X;
public sealed class ProviderX() : BaseProvider("https://api.x.ai/v1/", LOGGER)
public sealed class ProviderX() : BaseProvider(LLMProviders.X, "https://api.x.ai/v1/", LOGGER)
{
private static readonly ILogger<ProviderX> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderX>();
@@ -40,24 +40,7 @@ public sealed class ProviderX() : BaseProvider("https://api.x.ai/v1/", LOGGER)
var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
// Prepare the xAI HTTP chat request:
var xChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest