Added support for images (#609)
Build and Release / Read metadata (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2025-12-30 18:30:32 +01:00
1 parent ed4c7d215a
commit e9485ca8ea
80 files changed
+1300 -495

No files matched your search

@@ -0,0 +1,12 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Contract for sub-content in multimodal messages.
/// </summary>
public interface ISubContent
{
/// <summary>
/// The type of the sub-content.
/// </summary>
public SubContentType Type { get; init; }
}
@@ -0,0 +1,19 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Contract for nested image URL sub-content.
/// </summary>
/// <remarks>
/// Some providers use a nested object format for image URLs:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This interface represents the inner object with the "url" property.
/// </remarks>
public interface ISubContentImageUrl
{
/// <summary>
/// The URL or base64-encoded data URI of the image.
/// </summary>
public string Url { get; init; }
}
@@ -0,0 +1,13 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// A multimodal chat message model that can contain various types of content.
/// </summary>
/// <param name="Content">The list of sub-contents in the message.</param>
/// <param name="Role">The role of the message.</param>
public record MultimodalMessage(List<ISubContent> Content, string Role) : IMessage<List<ISubContent>>
{
public MultimodalMessage() : this([], string.Empty)
{
}
}
@@ -11,7 +11,7 @@ namespace AIStudio.Provider.OpenAI;
/// <summary>
/// The OpenAI provider.
/// </summary>
public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/", LOGGER)
public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, "https://api.openai.com/v1/", LOGGER)
{
private static readonly ILogger<ProviderOpenAI> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderOpenAI>();
@@ -59,7 +59,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
};
// Read the model capabilities:
var modelCapabilities = ProviderExtensions.GetModelCapabilitiesOpenAI(chatModel);
var modelCapabilities = this.Provider.GetModelCapabilities(chatModel);
// Check if we are using the Responses API or the Chat Completion API:
var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API);
@@ -90,9 +90,11 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
var apiParameters = this.ParseAdditionalApiParameters("input", "store", "tools");
// Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
{
Role = n.Role switch
var messages = await chatThread.Blocks.BuildMessagesAsync(
this.Provider, chatModel,
// OpenAI-specific role mapping:
role => role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
@@ -102,12 +104,46 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
_ => "user",
},
Content = n.Content switch
// OpenAI's text sub-content depends on the model, whether we are using
// the Responses API or the Chat Completion API:
text => usingResponsesAPI switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Responses API uses INPUT_TEXT:
true => new SubContentInputText
{
Text = text,
},
// Chat Completion API uses TEXT:
false => new SubContentText
{
Text = text,
},
},
// OpenAI's image sub-content depends on the model as well,
// whether we are using the Responses API or the Chat Completion API:
async attachment => usingResponsesAPI switch
{
// Responses API uses INPUT_IMAGE:
true => new SubContentInputImage
{
ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content)
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
: string.Empty,
},
// Chat Completion API uses IMAGE_URL:
false => new SubContentImageUrlNested
{
ImageUrl = new SubContentImageUrlData
{
Url = await attachment.TryAsBase64(token: token) is (true, var base64Content)
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
: string.Empty,
},
}
});
//
// Create the request: either for the Responses API or the Chat Completion API
@@ -149,7 +185,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
}, JSON_SERIALIZER_OPTIONS),
};
async Task<HttpRequestMessage> RequestBuilder()
{
// Build the HTTP post request:
@@ -0,0 +1,11 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image sub-content for multimodal messages.
/// </summary>
public record SubContentImageUrl(SubContentType Type, string ImageUrl) : ISubContent
{
public SubContentImageUrl() : this(SubContentType.IMAGE_URL, string.Empty)
{
}
}
@@ -0,0 +1,17 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Represents the inner object of a nested image URL sub-content.
/// </summary>
/// <remarks>
/// This record is used when the provider expects the format:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This class represents the inner <c>{ "url": "..." }</c> part.
/// </remarks>
public record SubContentImageUrlData : ISubContentImageUrl
{
/// <inheritdoc />
public string Url { get; init; } = string.Empty;
}
@@ -0,0 +1,18 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image sub-content for multimodal messages using nested URL format.
/// </summary>
/// <remarks>
/// This record is used when the provider expects the format:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// Used by LM Studio, VLLM, and other OpenAI-compatible providers.
/// </remarks>
public record SubContentImageUrlNested(SubContentType Type, ISubContentImageUrl ImageUrl) : ISubContent
{
public SubContentImageUrlNested() : this(SubContentType.IMAGE_URL, new SubContentImageUrlData())
{
}
}
@@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image input sub-content for multimodal messages.
/// </summary>
/// <remarks>
/// Right now, this is used only by OpenAI in its responses API.
/// </remarks>
public record SubContentInputImage(SubContentType Type, string ImageUrl) : ISubContent
{
public SubContentInputImage() : this(SubContentType.INPUT_IMAGE, string.Empty)
{
}
}
@@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Text input sub-content for multimodal messages.
/// </summary>
/// <remarks>
/// Right now, this is used only by OpenAI in its responses API.
/// </remarks>
public record SubContentInputText(SubContentType Type, string Text) : ISubContent
{
public SubContentInputText() : this(SubContentType.INPUT_TEXT, string.Empty)
{
}
}
@@ -0,0 +1,11 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Text sub-content for multimodal messages.
/// </summary>
public record SubContentText(SubContentType Type, string Text) : ISubContent
{
public SubContentText() : this(SubContentType.TEXT, string.Empty)
{
}
}
@@ -1,7 +1,7 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Chat message model.
/// Standard text-based chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>