mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-13 05:13:47 +00:00
Added support for images (#609)
Build and Release / Read metadata (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Build and Release / Read metadata (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
This commit is contained in:
1 parent
ed4c7d215a
commit
e9485ca8ea
80 files changed
+1300
-495
No files matched your search
@@ -0,0 +1,12 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Contract for sub-content in multimodal messages.
|
||||
/// </summary>
|
||||
public interface ISubContent
|
||||
{
|
||||
/// <summary>
|
||||
/// The type of the sub-content.
|
||||
/// </summary>
|
||||
public SubContentType Type { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Contract for nested image URL sub-content.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Some providers use a nested object format for image URLs:
|
||||
/// <code>
|
||||
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
|
||||
/// </code>
|
||||
/// This interface represents the inner object with the "url" property.
|
||||
/// </remarks>
|
||||
public interface ISubContentImageUrl
|
||||
{
|
||||
/// <summary>
|
||||
/// The URL or base64-encoded data URI of the image.
|
||||
/// </summary>
|
||||
public string Url { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// A multimodal chat message model that can contain various types of content.
|
||||
/// </summary>
|
||||
/// <param name="Content">The list of sub-contents in the message.</param>
|
||||
/// <param name="Role">The role of the message.</param>
|
||||
public record MultimodalMessage(List<ISubContent> Content, string Role) : IMessage<List<ISubContent>>
|
||||
{
|
||||
public MultimodalMessage() : this([], string.Empty)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ namespace AIStudio.Provider.OpenAI;
|
||||
/// <summary>
|
||||
/// The OpenAI provider.
|
||||
/// </summary>
|
||||
public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/", LOGGER)
|
||||
public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, "https://api.openai.com/v1/", LOGGER)
|
||||
{
|
||||
private static readonly ILogger<ProviderOpenAI> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderOpenAI>();
|
||||
|
||||
@@ -59,7 +59,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
|
||||
};
|
||||
|
||||
// Read the model capabilities:
|
||||
var modelCapabilities = ProviderExtensions.GetModelCapabilitiesOpenAI(chatModel);
|
||||
var modelCapabilities = this.Provider.GetModelCapabilities(chatModel);
|
||||
|
||||
// Check if we are using the Responses API or the Chat Completion API:
|
||||
var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API);
|
||||
@@ -90,9 +90,11 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
|
||||
var apiParameters = this.ParseAdditionalApiParameters("input", "store", "tools");
|
||||
|
||||
// Build the list of messages:
|
||||
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage
|
||||
{
|
||||
Role = n.Role switch
|
||||
var messages = await chatThread.Blocks.BuildMessagesAsync(
|
||||
this.Provider, chatModel,
|
||||
|
||||
// OpenAI-specific role mapping:
|
||||
role => role switch
|
||||
{
|
||||
ChatRole.USER => "user",
|
||||
ChatRole.AI => "assistant",
|
||||
@@ -102,12 +104,46 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
|
||||
_ => "user",
|
||||
},
|
||||
|
||||
Content = n.Content switch
|
||||
// OpenAI's text sub-content depends on the model, whether we are using
|
||||
// the Responses API or the Chat Completion API:
|
||||
text => usingResponsesAPI switch
|
||||
{
|
||||
ContentText text => await text.PrepareTextContentForAI(),
|
||||
_ => string.Empty,
|
||||
}
|
||||
});
|
||||
// Responses API uses INPUT_TEXT:
|
||||
true => new SubContentInputText
|
||||
{
|
||||
Text = text,
|
||||
},
|
||||
|
||||
// Chat Completion API uses TEXT:
|
||||
false => new SubContentText
|
||||
{
|
||||
Text = text,
|
||||
},
|
||||
},
|
||||
|
||||
// OpenAI's image sub-content depends on the model as well,
|
||||
// whether we are using the Responses API or the Chat Completion API:
|
||||
async attachment => usingResponsesAPI switch
|
||||
{
|
||||
// Responses API uses INPUT_IMAGE:
|
||||
true => new SubContentInputImage
|
||||
{
|
||||
ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content)
|
||||
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
|
||||
: string.Empty,
|
||||
},
|
||||
|
||||
// Chat Completion API uses IMAGE_URL:
|
||||
false => new SubContentImageUrlNested
|
||||
{
|
||||
ImageUrl = new SubContentImageUrlData
|
||||
{
|
||||
Url = await attachment.TryAsBase64(token: token) is (true, var base64Content)
|
||||
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
|
||||
: string.Empty,
|
||||
},
|
||||
}
|
||||
});
|
||||
|
||||
//
|
||||
// Create the request: either for the Responses API or the Chat Completion API
|
||||
@@ -149,7 +185,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
|
||||
|
||||
}, JSON_SERIALIZER_OPTIONS),
|
||||
};
|
||||
|
||||
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
// Build the HTTP post request:
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Image sub-content for multimodal messages.
|
||||
/// </summary>
|
||||
public record SubContentImageUrl(SubContentType Type, string ImageUrl) : ISubContent
|
||||
{
|
||||
public SubContentImageUrl() : this(SubContentType.IMAGE_URL, string.Empty)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the inner object of a nested image URL sub-content.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This record is used when the provider expects the format:
|
||||
/// <code>
|
||||
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
|
||||
/// </code>
|
||||
/// This class represents the inner <c>{ "url": "..." }</c> part.
|
||||
/// </remarks>
|
||||
public record SubContentImageUrlData : ISubContentImageUrl
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public string Url { get; init; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Image sub-content for multimodal messages using nested URL format.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This record is used when the provider expects the format:
|
||||
/// <code>
|
||||
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
|
||||
/// </code>
|
||||
/// Used by LM Studio, VLLM, and other OpenAI-compatible providers.
|
||||
/// </remarks>
|
||||
public record SubContentImageUrlNested(SubContentType Type, ISubContentImageUrl ImageUrl) : ISubContent
|
||||
{
|
||||
public SubContentImageUrlNested() : this(SubContentType.IMAGE_URL, new SubContentImageUrlData())
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Image input sub-content for multimodal messages.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Right now, this is used only by OpenAI in its responses API.
|
||||
/// </remarks>
|
||||
public record SubContentInputImage(SubContentType Type, string ImageUrl) : ISubContent
|
||||
{
|
||||
public SubContentInputImage() : this(SubContentType.INPUT_IMAGE, string.Empty)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Text input sub-content for multimodal messages.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Right now, this is used only by OpenAI in its responses API.
|
||||
/// </remarks>
|
||||
public record SubContentInputText(SubContentType Type, string Text) : ISubContent
|
||||
{
|
||||
public SubContentInputText() : this(SubContentType.INPUT_TEXT, string.Empty)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Text sub-content for multimodal messages.
|
||||
/// </summary>
|
||||
public record SubContentText(SubContentType Type, string Text) : ISubContent
|
||||
{
|
||||
public SubContentText() : this(SubContentType.TEXT, string.Empty)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Chat message model.
|
||||
/// Standard text-based chat message model.
|
||||
/// </summary>
|
||||
/// <param name="Content">The text content of the message.</param>
|
||||
/// <param name="Role">The role of the message.</param>
|
||||
|
||||
Reference in new issue
Block a user