Added support for images (#609)
Some checks are pending
Build and Release / Read metadata (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis updater) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage deb updater) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions

This commit is contained in:
Thorsten Sommer 2025-12-30 18:30:32 +01:00 committed by GitHub
parent ed4c7d215a
commit e9485ca8ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
80 changed files with 1300 additions and 495 deletions

View File

@ -159,7 +159,9 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
ContentText text => text.Text, ContentText text => text.Text,
// Image prompts may be empty, e.g., when the image is too large: // Image prompts may be empty, e.g., when the image is too large:
ContentImage image => await image.AsBase64(token), ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
? base64Image
: string.Empty,
// Other content types are not supported yet: // Other content types are not supported yet:
_ => string.Empty, _ => string.Empty,

View File

@ -219,7 +219,9 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
ContentText text => text.Text, ContentText text => text.Text,
// Image prompts may be empty, e.g., when the image is too large: // Image prompts may be empty, e.g., when the image is too large:
ContentImage image => await image.AsBase64(token), ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
? base64Image
: string.Empty,
// Other content types are not supported yet: // Other content types are not supported yet:
_ => string.Empty, _ => string.Empty,

View File

@ -217,12 +217,13 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
return chatId; return chatId;
} }
protected DateTimeOffset AddUserRequest(string request, bool hideContentFromUser = false) protected DateTimeOffset AddUserRequest(string request, bool hideContentFromUser = false, params List<FileAttachment> attachments)
{ {
var time = DateTimeOffset.Now; var time = DateTimeOffset.Now;
this.lastUserPrompt = new ContentText this.lastUserPrompt = new ContentText
{ {
Text = request, Text = request,
FileAttachments = attachments,
}; };
this.chatThread!.Blocks.Add(new ContentBlock this.chatThread!.Blocks.Add(new ContentBlock

View File

@ -103,7 +103,7 @@ else
@T("Documents for the analysis") @T("Documents for the analysis")
</MudText> </MudText>
<AttachDocuments Name="Document Analysis Files" @bind-DocumentPaths="@this.loadedDocumentPaths" CatchAllDocuments="true" UseSmallForm="false"/> <AttachDocuments Name="Document Analysis Files" @bind-DocumentPaths="@this.loadedDocumentPaths" CatchAllDocuments="true" UseSmallForm="false" Provider="@this.providerSettings"/>
</ExpansionPanel> </ExpansionPanel>
</MudExpansionPanels> </MudExpansionPanels>

View File

@ -1,3 +1,5 @@
using System.Text;
using AIStudio.Chat; using AIStudio.Chat;
using AIStudio.Dialogs; using AIStudio.Dialogs;
using AIStudio.Dialogs.Settings; using AIStudio.Dialogs.Settings;
@ -34,11 +36,13 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore<SettingsDialo
DOCUMENTS: the only content you may analyze. DOCUMENTS: the only content you may analyze.
Maybe, there are image files attached. IMAGES may contain important information. Use them as part of your analysis.
{this.GetDocumentTaskDescription()} {this.GetDocumentTaskDescription()}
# Scope and precedence # Scope and precedence
Use only information explicitly contained in DOCUMENTS and/or POLICY_*. Use only information explicitly contained in DOCUMENTS, IMAGES, and/or POLICY_*.
You may paraphrase but must not add facts, assumptions, or outside knowledge. You may paraphrase but must not add facts, assumptions, or outside knowledge.
Content decisions are governed by POLICY_ANALYSIS_RULES; formatting is governed by POLICY_OUTPUT_RULES. Content decisions are governed by POLICY_ANALYSIS_RULES; formatting is governed by POLICY_OUTPUT_RULES.
If there is a conflict between DOCUMENTS and POLICY_*, follow POLICY_ANALYSIS_RULES for analysis and POLICY_OUTPUT_RULES for formatting. Do not invent reconciliations. If there is a conflict between DOCUMENTS and POLICY_*, follow POLICY_ANALYSIS_RULES for analysis and POLICY_OUTPUT_RULES for formatting. Do not invent reconciliations.
@ -46,7 +50,7 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore<SettingsDialo
# Process # Process
1) Read POLICY_ANALYSIS_RULES and POLICY_OUTPUT_RULES end to end. 1) Read POLICY_ANALYSIS_RULES and POLICY_OUTPUT_RULES end to end.
2) Extract only the information from DOCUMENTS that POLICY_ANALYSIS_RULES permits. 2) Extract only the information from DOCUMENTS and IMAGES that POLICY_ANALYSIS_RULES permits.
3) Perform the analysis strictly according to POLICY_ANALYSIS_RULES. 3) Perform the analysis strictly according to POLICY_ANALYSIS_RULES.
4) Produce the final answer strictly according to POLICY_OUTPUT_RULES. 4) Produce the final answer strictly according to POLICY_OUTPUT_RULES.
@ -74,16 +78,33 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore<SettingsDialo
# Selfcheck before sending # Selfcheck before sending
Verify the answer matches POLICY_OUTPUT_RULES exactly. Verify the answer matches POLICY_OUTPUT_RULES exactly.
Verify every statement is attributable to DOCUMENTS or POLICY_*. Verify every statement is attributable to DOCUMENTS, IMAGES, or POLICY_*.
Remove any text not required by POLICY_OUTPUT_RULES. Remove any text not required by POLICY_OUTPUT_RULES.
{this.PromptGetActivePolicy()} {this.PromptGetActivePolicy()}
"""; """;
private string GetDocumentTaskDescription() => private string GetDocumentTaskDescription()
this.loadedDocumentPaths.Count > 1 {
? $"Your task is to analyze {this.loadedDocumentPaths.Count} DOCUMENTS. Different DOCUMENTS are divided by a horizontal rule in markdown formatting followed by the name of the document." var numDocuments = this.loadedDocumentPaths.Count(x => x is { Exists: true, IsImage: false });
: "Your task is to analyze a single document."; var numImages = this.loadedDocumentPaths.Count(x => x is { Exists: true, IsImage: true });
return (numDocuments, numImages) switch
{
(0, 1) => "Your task is to analyze a single image file attached as a document.",
(0, > 1) => $"Your task is to analyze {numImages} image file(s) attached as documents.",
(1, 0) => "Your task is to analyze a single DOCUMENT.",
(1, 1) => "Your task is to analyze a single DOCUMENT and 1 image file attached as a document.",
(1, > 1) => $"Your task is to analyze a single DOCUMENT and {numImages} image file(s) attached as documents.",
(> 0, 0) => $"Your task is to analyze {numDocuments} DOCUMENTS. Different DOCUMENTS are divided by a horizontal rule in markdown formatting followed by the name of the document.",
(> 0, 1) => $"Your task is to analyze {numDocuments} DOCUMENTS and 1 image file attached as a document. Different DOCUMENTS are divided by a horizontal rule in Markdown formatting followed by the name of the document.",
(> 0, > 0) => $"Your task is to analyze {numDocuments} DOCUMENTS and {numImages} image file(s) attached as documents. Different DOCUMENTS are divided by a horizontal rule in Markdown formatting followed by the name of the document.",
_ => "Your task is to analyze a single DOCUMENT."
};
}
protected override IReadOnlyList<IButtonData> FooterButtons => []; protected override IReadOnlyList<IButtonData> FooterButtons => [];
@ -327,37 +348,68 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore<SettingsDialo
if (this.loadedDocumentPaths.Count == 0) if (this.loadedDocumentPaths.Count == 0)
return string.Empty; return string.Empty;
var documentSections = new List<string>(); var documents = this.loadedDocumentPaths.Where(n => n is { Exists: true, IsImage: false }).ToList();
var count = 1; var sb = new StringBuilder();
foreach (var fileAttachment in this.loadedDocumentPaths) if (documents.Count > 0)
{ {
if (fileAttachment.IsForbidden) sb.AppendLine("""
# DOCUMENTS:
""");
}
var numDocuments = 1;
foreach (var document in documents)
{ {
this.Logger.LogWarning($"Skipping forbidden file: '{fileAttachment.FilePath}'."); if (document.IsForbidden)
{
this.Logger.LogWarning($"Skipping forbidden file: '{document.FilePath}'.");
continue; continue;
} }
var fileContent = await this.RustService.ReadArbitraryFileData(fileAttachment.FilePath, int.MaxValue); var fileContent = await this.RustService.ReadArbitraryFileData(document.FilePath, int.MaxValue);
sb.AppendLine($"""
documentSections.Add($""" ## DOCUMENT {numDocuments}:
## DOCUMENT {count}: File path: {document.FilePath}
File path: {fileAttachment.FilePath}
Content: Content:
``` ```
{fileContent} {fileContent}
``` ```
--- ---
"""); """);
count++; numDocuments++;
} }
return $""" var numImages = this.loadedDocumentPaths.Count(x => x is { IsImage: true, Exists: true });
# DOCUMENTS: if (numImages > 0)
{
if (documents.Count == 0)
{
sb.AppendLine($"""
{string.Join("\n", documentSections)} There are {numImages} image file(s) attached as documents.
"""; Please consider them as documents as well and use them to
answer accordingly.
""");
}
else
{
sb.AppendLine($"""
Additionally, there are {numImages} image file(s) attached.
Please consider them as documents as well and use them to
answer accordingly.
""");
}
}
return sb.ToString();
} }
private async Task Analyze() private async Task Analyze()
@ -370,7 +422,9 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore<SettingsDialo
this.CreateChatThread(); this.CreateChatThread();
var userRequest = this.AddUserRequest( var userRequest = this.AddUserRequest(
$"{await this.PromptLoadDocumentsContent()}", hideContentFromUser:true); await this.PromptLoadDocumentsContent(),
hideContentFromUser: true,
this.loadedDocumentPaths.Where(n => n is { Exists: true, IsImage: true }).ToList());
await this.AddAIResponseAsync(userRequest); await this.AddAIResponseAsync(userRequest);
} }

View File

@ -1513,6 +1513,12 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T4188329028"] = "No, kee
-- Export Chat to Microsoft Word -- Export Chat to Microsoft Word
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T861873672"] = "Export Chat to Microsoft Word" UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T861873672"] = "Export Chat to Microsoft Word"
-- The local image file does not exist. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "The local image file does not exist. Skipping the image."
-- Failed to download the image from the URL. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T2996654916"] = "Failed to download the image from the URL. Skipping the image."
-- The local image file is too large (>10 MB). Skipping the image. -- The local image file is too large (>10 MB). Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T3219823625"] = "The local image file is too large (>10 MB). Skipping the image." UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T3219823625"] = "The local image file is too large (>10 MB). Skipping the image."
@ -2968,6 +2974,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdo
-- Load file -- Load file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Load file" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Load file"
-- Image View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2199753423"] = "Image View"
-- See how we load your file. Review the content before we process it further. -- See how we load your file. Review the content before we process it further.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "See how we load your file. Review the content before we process it further." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "See how we load your file. Review the content before we process it further."
@ -2986,6 +2995,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T652739927"] = "This is
-- File Path -- File Path
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "File Path" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "File Path"
-- The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T973777830"] = "The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible."
-- Embedding Name -- Embedding Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Embedding Name" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Embedding Name"
@ -3436,12 +3448,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T1746160064"] = "He
-- There aren't any file attachments available right now. -- There aren't any file attachments available right now.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T2111340711"] = "There aren't any file attachments available right now." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T2111340711"] = "There aren't any file attachments available right now."
-- Document Preview
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T285154968"] = "Document Preview"
-- The file was deleted, renamed, or moved. -- The file was deleted, renamed, or moved.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3083729256"] = "The file was deleted, renamed, or moved." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3083729256"] = "The file was deleted, renamed, or moved."
-- Your attached file. -- Your attached file.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3154198222"] = "Your attached file." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3154198222"] = "Your attached file."
-- Preview what we send to the AI.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3160778981"] = "Preview what we send to the AI."
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3448155331"] = "Close"
-- Your attached files -- Your attached files
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Your attached files" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Your attached files"
@ -6058,9 +6079,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T29289275
-- Images are not supported yet -- Images are not supported yet
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T298062956"] = "Images are not supported yet" UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T298062956"] = "Images are not supported yet"
-- Images are not supported at this place
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T305247150"] = "Images are not supported at this place"
-- Executables are not allowed -- Executables are not allowed
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T4167762413"] = "Executables are not allowed" UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T4167762413"] = "Executables are not allowed"
-- Images are not supported by the selected provider and model
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T999194030"] = "Images are not supported by the selected provider and model"
-- The hostname is not a valid HTTP(S) URL. -- The hostname is not a valid HTTP(S) URL.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::PROVIDERVALIDATION::T1013354736"] = "The hostname is not a valid HTTP(S) URL." UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::PROVIDERVALIDATION::T1013354736"] = "The hostname is not a valid HTTP(S) URL."

View File

@ -238,7 +238,7 @@ public sealed record ChatThread
{ {
var (contentData, contentType) = block.Content switch var (contentData, contentType) = block.Content switch
{ {
ContentImage image => (await image.AsBase64(token), Tools.ERIClient.DataModel.ContentType.IMAGE), ContentImage image => (await image.TryAsBase64(token) is (success: true, { } base64Image) ? base64Image : string.Empty, Tools.ERIClient.DataModel.ContentType.IMAGE),
ContentText text => (text.Text, Tools.ERIClient.DataModel.ContentType.TEXT), ContentText text => (text.Text, Tools.ERIClient.DataModel.ContentType.TEXT),
_ => (string.Empty, Tools.ERIClient.DataModel.ContentType.UNKNOWN), _ => (string.Empty, Tools.ERIClient.DataModel.ContentType.UNKNOWN),

View File

@ -47,6 +47,8 @@ public sealed class ContentImage : IContent, IImageSource
InitialRemoteWait = this.InitialRemoteWait, InitialRemoteWait = this.InitialRemoteWait,
IsStreaming = this.IsStreaming, IsStreaming = this.IsStreaming,
SourceType = this.SourceType, SourceType = this.SourceType,
Sources = [..this.Sources],
FileAttachments = [..this.FileAttachments],
}; };
#endregion #endregion

View File

@ -195,6 +195,14 @@ public sealed class ContentText : IContent
sb.AppendLine(await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue)); sb.AppendLine(await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue));
sb.AppendLine("````"); sb.AppendLine("````");
} }
var numImages = this.FileAttachments.Count(x => x is { IsImage: true, Exists: true });
if (numImages > 0)
{
sb.AppendLine();
sb.AppendLine($"Additionally, there are {numImages} image file(s) attached to this message. ");
sb.AppendLine("Please consider them as part of the message content and use them to answer accordingly.");
}
} }
} }
} }

View File

@ -1,3 +1,5 @@
using System.Text.Json.Serialization;
using AIStudio.Tools.Rust; using AIStudio.Tools.Rust;
namespace AIStudio.Chat; namespace AIStudio.Chat;
@ -9,22 +11,47 @@ namespace AIStudio.Chat;
/// <param name="FileName">The name of the file, including extension.</param> /// <param name="FileName">The name of the file, including extension.</param>
/// <param name="FilePath">The full path to the file, including the filename and extension.</param> /// <param name="FilePath">The full path to the file, including the filename and extension.</param>
/// <param name="FileSizeBytes">The size of the file in bytes.</param> /// <param name="FileSizeBytes">The size of the file in bytes.</param>
public readonly record struct FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes) [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
[JsonDerivedType(typeof(FileAttachment), typeDiscriminator: "file")]
[JsonDerivedType(typeof(FileAttachmentImage), typeDiscriminator: "image")]
public record FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes)
{ {
/// <summary>
/// Gets a value indicating whether the file still exists on the file system.
/// </summary>
public bool Exists => File.Exists(this.FilePath);
/// <summary> /// <summary>
/// Gets a value indicating whether the file type is forbidden and should not be attached. /// Gets a value indicating whether the file type is forbidden and should not be attached.
/// </summary> /// </summary>
public bool IsForbidden => this.Type == FileAttachmentType.FORBIDDEN; /// <remarks>
/// The state is determined once during construction and does not change.
/// </remarks>
public bool IsForbidden { get; } = Type == FileAttachmentType.FORBIDDEN;
/// <summary> /// <summary>
/// Gets a value indicating whether the file type is valid and allowed to be attached. /// Gets a value indicating whether the file type is valid and allowed to be attached.
/// </summary> /// </summary>
public bool IsValid => this.Type != FileAttachmentType.FORBIDDEN; /// <remarks>
/// The state is determined once during construction and does not change.
/// </remarks>
public bool IsValid { get; } = Type != FileAttachmentType.FORBIDDEN;
/// <summary>
/// Gets a value indicating whether the file type is an image.
/// </summary>
/// <remarks>
/// The state is determined once during construction and does not change.
/// </remarks>
public bool IsImage { get; } = Type == FileAttachmentType.IMAGE;
/// <summary>
/// Gets the file path for loading the file from the web browser-side (Blazor).
/// </summary>
public string FilePathAsUrl { get; } = FileHandler.CreateFileUrl(FilePath);
/// <summary>
/// Gets a value indicating whether the file still exists on the file system.
/// </summary>
/// <remarks>
/// This property checks the file system each time it is accessed.
/// </remarks>
public bool Exists => File.Exists(this.FilePath);
/// <summary> /// <summary>
/// Creates a FileAttachment from a file path by automatically determining the type, /// Creates a FileAttachment from a file path by automatically determining the type,
@ -38,7 +65,13 @@ public readonly record struct FileAttachment(FileAttachmentType Type, string Fil
var fileSize = File.Exists(filePath) ? new FileInfo(filePath).Length : 0; var fileSize = File.Exists(filePath) ? new FileInfo(filePath).Length : 0;
var type = DetermineFileType(filePath); var type = DetermineFileType(filePath);
return new FileAttachment(type, fileName, filePath, fileSize); return type switch
{
FileAttachmentType.DOCUMENT => new FileAttachment(type, fileName, filePath, fileSize),
FileAttachmentType.IMAGE => new FileAttachmentImage(fileName, filePath, fileSize),
_ => new FileAttachment(type, fileName, filePath, fileSize),
};
} }
/// <summary> /// <summary>

View File

@ -0,0 +1,17 @@
namespace AIStudio.Chat;
public record FileAttachmentImage(string FileName, string FilePath, long FileSizeBytes) : FileAttachment(FileAttachmentType.IMAGE, FileName, FilePath, FileSizeBytes), IImageSource
{
/// <summary>
/// The type of the image source.
/// </summary>
/// <remarks>
/// Is the image source a URL, a local file path, a base64 string, etc.?
/// </remarks>
public ContentImageSource SourceType { get; init; } = ContentImageSource.LOCAL_PATH;
/// <summary>
/// The image source.
/// </summary>
public string Source { get; set; } = FilePath;
}

View File

@ -6,27 +6,89 @@ public static class IImageSourceExtensions
{ {
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IImageSourceExtensions).Namespace, nameof(IImageSourceExtensions)); private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IImageSourceExtensions).Namespace, nameof(IImageSourceExtensions));
public static string DetermineMimeType(this IImageSource image)
{
switch (image.SourceType)
{
case ContentImageSource.BASE64:
{
// Try to detect the mime type from the base64 string:
var base64Data = image.Source;
if (base64Data.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
{
var mimeEnd = base64Data.IndexOf(';');
if (mimeEnd > 5)
{
return base64Data[5..mimeEnd];
}
}
// Fallback:
return "application/octet-stream";
}
case ContentImageSource.URL:
{
// Try to detect the mime type from the URL extension:
var uri = new Uri(image.Source);
var extension = Path.GetExtension(uri.AbsolutePath).ToLowerInvariant();
return extension switch
{
".png" => "image/png",
".jpg" or ".jpeg" => "image/jpeg",
".gif" => "image/gif",
".bmp" => "image/bmp",
".webp" => "image/webp",
_ => "application/octet-stream"
};
}
case ContentImageSource.LOCAL_PATH:
{
var extension = Path.GetExtension(image.Source).ToLowerInvariant();
return extension switch
{
".png" => "image/png",
".jpg" or ".jpeg" => "image/jpeg",
".gif" => "image/gif",
".bmp" => "image/bmp",
".webp" => "image/webp",
_ => "application/octet-stream"
};
}
default:
return "application/octet-stream";
}
}
/// <summary> /// <summary>
/// Read the image content as a base64 string. /// Read the image content as a base64 string.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// The images are directly converted to base64 strings. The maximum /// The images are directly converted to base64 strings. The maximum
/// size of the image is around 10 MB. If the image is larger, the method /// size of the image is around 10 MB. If the image is larger, the method
/// returns an empty string. /// returns an empty string.<br/>
/// /// <br/>
/// As of now, this method does no sort of image processing. LLMs usually /// As of now, this method does no sort of image processing. LLMs usually
/// do not work with arbitrary image sizes. In the future, we might have /// do not work with arbitrary image sizes. In the future, we might have
/// to resize the images before sending them to the model. /// to resize the images before sending them to the model.<br/>
/// <br/>
/// Note as well that this method returns just the base64 string without
/// any data URI prefix (like "data:image/png;base64,"). The caller has
/// to take care of that if needed.
/// </remarks> /// </remarks>
/// <param name="image">The image source.</param> /// <param name="image">The image source.</param>
/// <param name="token">The cancellation token.</param> /// <param name="token">The cancellation token.</param>
/// <returns>The image content as a base64 string; might be empty.</returns> /// <returns>The image content as a base64 string; might be empty.</returns>
public static async Task<string> AsBase64(this IImageSource image, CancellationToken token = default) public static async Task<(bool success, string base64Content)> TryAsBase64(this IImageSource image, CancellationToken token = default)
{ {
switch (image.SourceType) switch (image.SourceType)
{ {
case ContentImageSource.BASE64: case ContentImageSource.BASE64:
return image.Source; return (success: true, image.Source);
case ContentImageSource.URL: case ContentImageSource.URL:
{ {
@ -39,14 +101,15 @@ public static class IImageSourceExtensions
if(lengthBytes > 10_000_000) if(lengthBytes > 10_000_000)
{ {
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The image at the URL is too large (>10 MB). Skipping the image."))); await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The image at the URL is too large (>10 MB). Skipping the image.")));
return string.Empty; return (success: false, string.Empty);
} }
var bytes = await response.Content.ReadAsByteArrayAsync(token); var bytes = await response.Content.ReadAsByteArrayAsync(token);
return Convert.ToBase64String(bytes); return (success: true, Convert.ToBase64String(bytes));
} }
return string.Empty; await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("Failed to download the image from the URL. Skipping the image.")));
return (success: false, string.Empty);
} }
case ContentImageSource.LOCAL_PATH: case ContentImageSource.LOCAL_PATH:
@ -57,17 +120,18 @@ public static class IImageSourceExtensions
if(length > 10_000_000) if(length > 10_000_000)
{ {
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file is too large (>10 MB). Skipping the image."))); await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file is too large (>10 MB). Skipping the image.")));
return string.Empty; return (success: false, string.Empty);
} }
var bytes = await File.ReadAllBytesAsync(image.Source, token); var bytes = await File.ReadAllBytesAsync(image.Source, token);
return Convert.ToBase64String(bytes); return (success: true, Convert.ToBase64String(bytes));
} }
return string.Empty; await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file does not exist. Skipping the image.")));
return (success: false, string.Empty);
default: default:
return string.Empty; return (success: false, string.Empty);
} }
} }
} }

View File

@ -1,4 +1,6 @@
using AIStudio.Provider; using AIStudio.Provider;
using AIStudio.Provider.OpenAI;
using AIStudio.Settings;
namespace AIStudio.Chat; namespace AIStudio.Chat;
@ -8,19 +10,171 @@ public static class ListContentBlockExtensions
/// Processes a list of content blocks by transforming them into a collection of message results asynchronously. /// Processes a list of content blocks by transforming them into a collection of message results asynchronously.
/// </summary> /// </summary>
/// <param name="blocks">The list of content blocks to process.</param> /// <param name="blocks">The list of content blocks to process.</param>
/// <param name="transformer">A function that transforms each content block into a message result asynchronously.</param> /// <param name="roleTransformer">A function that transforms each content block into a message result asynchronously.</param>
/// <param name="selectedProvider">The selected LLM provider.</param>
/// <param name="selectedModel">The selected model.</param>
/// <param name="textSubContentFactory">A factory function to create text sub-content.</param>
/// <param name="imageSubContentFactory">A factory function to create image sub-content.</param>
/// <returns>An asynchronous task that resolves to a list of transformed results.</returns> /// <returns>An asynchronous task that resolves to a list of transformed results.</returns>
public static async Task<IList<IMessageBase>> BuildMessages(this List<ContentBlock> blocks, Func<ContentBlock, Task<IMessageBase>> transformer) public static async Task<IList<IMessageBase>> BuildMessagesAsync(
this List<ContentBlock> blocks,
LLMProviders selectedProvider,
Model selectedModel,
Func<ChatRole, string> roleTransformer,
Func<string, ISubContent> textSubContentFactory,
Func<FileAttachmentImage, Task<ISubContent>> imageSubContentFactory)
{ {
var messages = blocks var capabilities = selectedProvider.GetModelCapabilities(selectedModel);
.Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text)) var canProcessImages = capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT) ||
.Select(transformer) capabilities.Contains(Capability.SINGLE_IMAGE_INPUT);
.ToList();
var messageTaskList = new List<Task<IMessageBase>>(blocks.Count);
foreach (var block in blocks)
{
switch (block.Content)
{
// The prompt may or may not contain image(s), but the provider/model cannot process images.
// Thus, we treat it as a regular text message.
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && !canProcessImages:
messageTaskList.Add(CreateTextMessageAsync(block, text));
break;
// The regular case for text content without images:
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && !text.FileAttachments.ContainsImages():
messageTaskList.Add(CreateTextMessageAsync(block, text));
break;
// Text prompt with images as attachments, and the provider/model can process images:
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && text.FileAttachments.ContainsImages():
messageTaskList.Add(CreateMultimodalMessageAsync(block, text, textSubContentFactory, imageSubContentFactory));
break;
}
}
// Await all messages: // Await all messages:
await Task.WhenAll(messages); await Task.WhenAll(messageTaskList);
// Select all results: // Select all results:
return messages.Select(n => n.Result).ToList(); return messageTaskList.Select(n => n.Result).ToList();
// Local function to create a text message asynchronously.
Task<IMessageBase> CreateTextMessageAsync(ContentBlock block, ContentText text)
{
return Task.Run(async () => new TextMessage
{
Role = roleTransformer(block.Role),
Content = await text.PrepareTextContentForAI(),
} as IMessageBase);
} }
// Local function to create a multimodal message asynchronously.
Task<IMessageBase> CreateMultimodalMessageAsync(
ContentBlock block,
ContentText text,
Func<string, ISubContent> innerTextSubContentFactory,
Func<FileAttachmentImage, Task<ISubContent>> innerImageSubContentFactory)
{
return Task.Run(async () =>
{
var imagesTasks = text.FileAttachments
.Where(x => x is { IsImage: true, Exists: true })
.Cast<FileAttachmentImage>()
.Select(innerImageSubContentFactory)
.ToList();
Task.WaitAll(imagesTasks);
var images = imagesTasks.Select(t => t.Result).ToList();
return new MultimodalMessage
{
Role = roleTransformer(block.Role),
Content =
[
innerTextSubContentFactory(await text.PrepareTextContentForAI()),
..images,
]
} as IMessageBase;
});
}
}
/// <summary>
/// Processes a list of content blocks using direct image URL format to create message results asynchronously.
/// </summary>
/// <param name="blocks">The list of content blocks to process.</param>
/// <param name="selectedProvider">The selected LLM provider.</param>
/// <param name="selectedModel">The selected model.</param>
/// <returns>An asynchronous task that resolves to a list of transformed message results.</returns>
/// <remarks>
/// Uses direct image URL format where the image data is placed directly in the image_url field:
/// <code>
/// { "type": "image_url", "image_url": "data:image/jpeg;base64,..." }
/// </code>
/// This format is used by OpenAI, Mistral, and Ollama.
/// </remarks>
public static async Task<IList<IMessageBase>> BuildMessagesUsingDirectImageUrlAsync(
this List<ContentBlock> blocks,
LLMProviders selectedProvider,
Model selectedModel) => await blocks.BuildMessagesAsync(
selectedProvider,
selectedModel,
StandardRoleTransformer,
StandardTextSubContentFactory,
DirectImageSubContentFactory);
/// <summary>
/// Processes a list of content blocks using nested image URL format to create message results asynchronously.
/// </summary>
/// <param name="blocks">The list of content blocks to process.</param>
/// <param name="selectedProvider">The selected LLM provider.</param>
/// <param name="selectedModel">The selected model.</param>
/// <returns>An asynchronous task that resolves to a list of transformed message results.</returns>
/// <remarks>
/// Uses nested image URL format where the image data is wrapped in an object:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This format is used by LM Studio, VLLM, llama.cpp, and other OpenAI-compatible providers.
/// </remarks>
public static async Task<IList<IMessageBase>> BuildMessagesUsingNestedImageUrlAsync(
this List<ContentBlock> blocks,
LLMProviders selectedProvider,
Model selectedModel) => await blocks.BuildMessagesAsync(
selectedProvider,
selectedModel,
StandardRoleTransformer,
StandardTextSubContentFactory,
NestedImageSubContentFactory);
private static ISubContent StandardTextSubContentFactory(string text) => new SubContentText
{
Text = text,
};
private static async Task<ISubContent> DirectImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrl
{
ImageUrl = await attachment.TryAsBase64() is (true, var base64Content)
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
: string.Empty,
};
private static async Task<ISubContent> NestedImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrlNested
{
ImageUrl = new SubContentImageUrlData
{
Url = await attachment.TryAsBase64() is (true, var base64Content)
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
: string.Empty,
},
};
private static string StandardRoleTransformer(ChatRole role) => role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
};
} }

View File

@ -0,0 +1,6 @@
namespace AIStudio.Chat;
public static class ListFileAttachmentExtensions
{
public static bool ContainsImages(this List<FileAttachment> attachments) => attachments.Any(attachment => attachment.IsImage);
}

View File

@ -36,6 +36,9 @@ public partial class AttachDocuments : MSGComponentBase
[Parameter] [Parameter]
public bool UseSmallForm { get; set; } public bool UseSmallForm { get; set; }
[Parameter]
public AIStudio.Settings.Provider? Provider { get; set; }
[Inject] [Inject]
private ILogger<AttachDocuments> Logger { get; set; } = null!; private ILogger<AttachDocuments> Logger { get; set; } = null!;
@ -114,7 +117,7 @@ public partial class AttachDocuments : MSGComponentBase
foreach (var path in paths) foreach (var path in paths)
{ {
if(!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(path)) if(!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, this.Provider))
continue; continue;
this.DocumentPaths.Add(FileAttachment.FromPath(path)); this.DocumentPaths.Add(FileAttachment.FromPath(path));
@ -158,7 +161,7 @@ public partial class AttachDocuments : MSGComponentBase
if (!File.Exists(selectedFilePath)) if (!File.Exists(selectedFilePath))
continue; continue;
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(selectedFilePath)) if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, selectedFilePath, this.Provider))
continue; continue;
this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath)); this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath));
@ -216,7 +219,7 @@ public partial class AttachDocuments : MSGComponentBase
{ {
var dialogParameters = new DialogParameters<DocumentCheckDialog> var dialogParameters = new DialogParameters<DocumentCheckDialog>
{ {
{ x => x.FilePath, fileAttachment.FilePath }, { x => x.Document, fileAttachment },
}; };
await this.DialogService.ShowAsync<DocumentCheckDialog>(T("Document Preview"), dialogParameters, DialogOptions.FULLSCREEN); await this.DialogService.ShowAsync<DocumentCheckDialog>(T("Document Preview"), dialogParameters, DialogOptions.FULLSCREEN);

View File

@ -83,7 +83,7 @@
<ChatTemplateSelection CanChatThreadBeUsedForTemplate="@this.CanThreadBeSaved" CurrentChatThread="@this.ChatThread" CurrentChatTemplate="@this.currentChatTemplate" CurrentChatTemplateChanged="@this.ChatTemplateWasChanged"/> <ChatTemplateSelection CanChatThreadBeUsedForTemplate="@this.CanThreadBeSaved" CurrentChatThread="@this.ChatThread" CurrentChatTemplate="@this.currentChatTemplate" CurrentChatTemplateChanged="@this.ChatTemplateWasChanged"/>
<AttachDocuments Name="File Attachments" @bind-DocumentPaths="@this.chatDocumentPaths" CatchAllDocuments="true" UseSmallForm="true"/> <AttachDocuments Name="File Attachments" @bind-DocumentPaths="@this.chatDocumentPaths" CatchAllDocuments="true" UseSmallForm="true" Provider="@this.Provider"/>
@if (this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY) @if (this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY)
{ {

View File

@ -932,12 +932,19 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
} }
if (this.cancellationTokenSource is not null) if (this.cancellationTokenSource is not null)
{
try
{ {
if(!this.cancellationTokenSource.IsCancellationRequested) if(!this.cancellationTokenSource.IsCancellationRequested)
await this.cancellationTokenSource.CancelAsync(); await this.cancellationTokenSource.CancelAsync();
this.cancellationTokenSource.Dispose(); this.cancellationTokenSource.Dispose();
} }
catch
{
// ignored
}
}
} }
#endregion #endregion

View File

@ -55,7 +55,7 @@ public partial class ReadFileContent : MSGComponentBase
return; return;
} }
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(selectedFile.SelectedFilePath)) if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.DIRECTLY_LOADING_CONTENT, selectedFile.SelectedFilePath))
{ {
this.Logger.LogWarning("User attempted to load unsupported file: {FilePath}", selectedFile.SelectedFilePath); this.Logger.LogWarning("User attempted to load unsupported file: {FilePath}", selectedFile.SelectedFilePath);
return; return;

View File

@ -6,7 +6,7 @@
@T("See how we load your file. Review the content before we process it further.") @T("See how we load your file. Review the content before we process it further.")
</MudJustifiedText> </MudJustifiedText>
@if (string.IsNullOrWhiteSpace(this.FilePath)) @if (this.Document is null)
{ {
<ReadFileContent Text="@T("Load file")" @bind-FileContent="@this.FileContent"/> <ReadFileContent Text="@T("Load file")" @bind-FileContent="@this.FileContent"/>
} }
@ -14,7 +14,7 @@
{ {
<MudTextField <MudTextField
T="string" T="string"
@bind-Text="@this.FilePath" Text="@this.Document.FilePath"
AdornmentIcon="@Icons.Material.Filled.FileOpen" AdornmentIcon="@Icons.Material.Filled.FileOpen"
Adornment="Adornment.Start" Adornment="Adornment.Start"
Immediate="@true" Immediate="@true"
@ -27,7 +27,23 @@
/> />
} }
@if (!this.Document?.Exists ?? false)
{
<MudAlert Severity="Severity.Error" Variant="Variant.Filled" Class="my-2">
@T("The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.")
</MudAlert>
}
else
{
<MudTabs Elevation="0" Rounded="true" ApplyEffectsToContainer="true" Outlined="true" PanelClass="pa-2" Class="mb-2"> <MudTabs Elevation="0" Rounded="true" ApplyEffectsToContainer="true" Outlined="true" PanelClass="pa-2" Class="mb-2">
@if (this.Document?.IsImage ?? false)
{
<MudTabPanel Text="@T("Image View")" Icon="@Icons.Material.Filled.Image">
<MudImage ObjectFit="ObjectFit.ScaleDown" Style="width: 100%;" Src="@this.Document.FilePathAsUrl"/>
</MudTabPanel>
}
else
{
<MudTabPanel Text="@T("Markdown View")" Icon="@Icons.Material.Filled.TextSnippet"> <MudTabPanel Text="@T("Markdown View")" Icon="@Icons.Material.Filled.TextSnippet">
<MudField <MudField
Variant="Variant.Outlined" Variant="Variant.Outlined"
@ -36,8 +52,7 @@
Label="@T("Loaded Content")" Label="@T("Loaded Content")"
FullWidth="true" FullWidth="true"
Class="ma-2 pe-4" Class="ma-2 pe-4"
HelperText="@T("This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.")" HelperText="@T("This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.")">
>
<div style="max-height: 40vh; overflow-y: auto;"> <div style="max-height: 40vh; overflow-y: auto;">
<MudMarkdown Value="@this.FileContent" Props="Markdown.DefaultConfig" Styling="@this.MarkdownStyling"/> <MudMarkdown Value="@this.FileContent" Props="Markdown.DefaultConfig" Styling="@this.MarkdownStyling"/>
</div> </div>
@ -59,11 +74,13 @@
Class="ma-2" Class="ma-2"
HelperText="@T("This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.")"/> HelperText="@T("This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.")"/>
</MudTabPanel> </MudTabPanel>
}
</MudTabs> </MudTabs>
}
</DialogContent> </DialogContent>
<DialogActions> <DialogActions>
<MudButton OnClick="@this.Close" Variant="Variant.Filled"> <MudButton OnClick="@this.Close" Variant="Variant.Filled" Color="Color.Primary">
@T("Close") @T("Close")
</MudButton> </MudButton>
</DialogActions> </DialogActions>

View File

@ -1,4 +1,5 @@
using AIStudio.Components; using AIStudio.Chat;
using AIStudio.Components;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components;
@ -13,7 +14,7 @@ public partial class DocumentCheckDialog : MSGComponentBase
private IMudDialogInstance MudDialog { get; set; } = null!; private IMudDialogInstance MudDialog { get; set; } = null!;
[Parameter] [Parameter]
public string FilePath { get; set; } = string.Empty; public FileAttachment? Document { get; set; }
private void Close() => this.MudDialog.Cancel(); private void Close() => this.MudDialog.Cancel();
@ -27,27 +28,30 @@ public partial class DocumentCheckDialog : MSGComponentBase
private IDialogService DialogService { get; init; } = null!; private IDialogService DialogService { get; init; } = null!;
[Inject] [Inject]
private ILogger<ReadFileContent> Logger { get; init; } = null!; private ILogger<DocumentCheckDialog> Logger { get; init; } = null!;
protected override async Task OnAfterRenderAsync(bool firstRender) protected override async Task OnAfterRenderAsync(bool firstRender)
{ {
if (firstRender && !string.IsNullOrWhiteSpace(this.FilePath)) if (firstRender && this.Document is not null)
{ {
try try
{ {
var fileContent = await UserFile.LoadFileData(this.FilePath, this.RustService, this.DialogService); if (!this.Document.IsImage)
{
var fileContent = await UserFile.LoadFileData(this.Document.FilePath, this.RustService, this.DialogService);
this.FileContent = fileContent; this.FileContent = fileContent;
this.StateHasChanged(); }
} }
catch (Exception ex) catch (Exception ex)
{ {
this.Logger.LogError(ex, "Failed to load file content from '{FilePath}'", this.FilePath); this.Logger.LogError(ex, "Failed to load file content from '{FilePath}'", this.Document);
this.FileContent = string.Empty; this.FileContent = string.Empty;
}
this.StateHasChanged(); this.StateHasChanged();
} }
}
else if (firstRender) else if (firstRender)
this.Logger.LogWarning("Document check dialog opened without a valid file path"); this.Logger.LogWarning("Document check dialog opened without a valid file path.");
} }
private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default; private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default;

View File

@ -46,19 +46,24 @@
</MudTooltip> </MudTooltip>
</div> </div>
<MudToolBar WrapContent="true" Gutters="false" Class="ml-2" Style="flex-shrink: 0; min-height: 1em;">
<MudTooltip Text="@T("Preview what we send to the AI.")" Placement="Placement.Bottom">
<MudIconButton Icon="@Icons.Material.Filled.Search"
Color="Color.Primary"
OnClick="@(() => this.InvestigateFile(fileAttachment))"/>
</MudTooltip>
<MudTooltip Text="@T("Remove this attachment.")" Placement="Placement.Bottom"> <MudTooltip Text="@T("Remove this attachment.")" Placement="Placement.Bottom">
<MudIconButton Icon="@Icons.Material.Filled.Delete" <MudIconButton Icon="@Icons.Material.Filled.Delete"
Color="Color.Error" Color="Color.Error"
Class="ml-2"
Style="flex-shrink: 0;"
OnClick="@(() => this.DeleteAttachment(fileAttachment))"/> OnClick="@(() => this.DeleteAttachment(fileAttachment))"/>
</MudTooltip> </MudTooltip>
</MudToolBar>
</MudStack> </MudStack>
} }
else else
{ {
<MudStack Row Justify="Justify.SpaceBetween" AlignItems="AlignItems.Center" Class="ms-3 mb-2"> <MudStack Row Justify="Justify.SpaceBetween" AlignItems="AlignItems.Center" Class="ms-3">
<div style="min-width: 0; flex: 1; overflow: hidden;"> <div style="min-width: 0; flex: 1; overflow: hidden;">
<MudTooltip Text="@T("The file was deleted, renamed, or moved.")" Placement="Placement.Bottom"> <MudTooltip Text="@T("The file was deleted, renamed, or moved.")" Placement="Placement.Bottom">
<span class="d-inline-flex align-items-center" style="overflow: hidden; width: 100%;"> <span class="d-inline-flex align-items-center" style="overflow: hidden; width: 100%;">
@ -70,13 +75,17 @@
</MudTooltip> </MudTooltip>
</div> </div>
<MudToolBar WrapContent="true" Gutters="false" Class="ml-2" Style="flex-shrink: 0; min-height: 1em;">
<MudIconButton Icon="@Icons.Material.Filled.Search"
Color="Color.Primary"
Disabled="true"/>
<MudTooltip Text="@T("Remove this attachment.")" Placement="Placement.Bottom"> <MudTooltip Text="@T("Remove this attachment.")" Placement="Placement.Bottom">
<MudIconButton Icon="@Icons.Material.Filled.Delete" <MudIconButton Icon="@Icons.Material.Filled.Delete"
Color="Color.Error" Color="Color.Error"
Class="ml-2"
Style="flex-shrink: 0;"
OnClick="@(() => this.DeleteAttachment(fileAttachment))"/> OnClick="@(() => this.DeleteAttachment(fileAttachment))"/>
</MudTooltip> </MudTooltip>
</MudToolBar>
</MudStack> </MudStack>
} }
} }
@ -85,7 +94,7 @@
</DialogContent> </DialogContent>
<DialogActions> <DialogActions>
<MudButton OnClick="@this.Close" Variant="Variant.Filled" Color="Color.Primary"> <MudButton OnClick="@this.Close" Variant="Variant.Filled" Color="Color.Primary">
Close @T("Close")
</MudButton> </MudButton>
</DialogActions> </DialogActions>
</MudDialog> </MudDialog>

View File

@ -46,4 +46,18 @@ public partial class ReviewAttachmentsDialog : MSGComponentBase
this.StateHasChanged(); this.StateHasChanged();
} }
} }
/// <summary>
/// The user might want to check what we actually extract from his file and therefore give the LLM as an input.
/// </summary>
/// <param name="fileAttachment">The file to check.</param>
private async Task InvestigateFile(FileAttachment fileAttachment)
{
var dialogParameters = new DialogParameters<DocumentCheckDialog>
{
{ x => x.Document, fileAttachment },
};
await this.DialogService.ShowAsync<DocumentCheckDialog>(T("Document Preview"), dialogParameters, DialogOptions.FULLSCREEN);
}
} }

View File

@ -0,0 +1,80 @@
using Microsoft.AspNetCore.StaticFiles;
namespace AIStudio;
internal static class FileHandler
{
private const string ENDPOINT = "/local/file";
private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(nameof(FileHandler));
internal static string CreateFileUrl(string filePath)
{
var encodedPath = Uri.EscapeDataString(filePath);
return $"{ENDPOINT}?path={encodedPath}";
}
internal static async Task HandlerAsync(HttpContext context, Func<Task> nextHandler)
{
var requestPath = context.Request.Path.Value;
if (string.IsNullOrWhiteSpace(requestPath) || !requestPath.Equals(ENDPOINT, StringComparison.Ordinal))
{
await nextHandler();
return;
}
// Extract the file path from the query parameter:
// Format: /local/file?path={url-encoded-path}
if (!context.Request.Query.TryGetValue("path", out var pathValues) || pathValues.Count == 0)
{
context.Response.StatusCode = StatusCodes.Status400BadRequest;
LOGGER.LogWarning("No file path provided in the request. Using ?path={{url-encoded-path}} format.");
return;
}
// The query parameter is automatically URL-decoded by ASP.NET Core:
var filePath = pathValues[0];
if (string.IsNullOrWhiteSpace(filePath))
{
context.Response.StatusCode = StatusCodes.Status400BadRequest;
LOGGER.LogWarning("Empty file path provided in the request.");
return;
}
// Security check: Prevent path traversal attacks:
var fullPath = Path.GetFullPath(filePath);
if (fullPath != filePath && !filePath.StartsWith('/'))
{
// On Windows, absolute paths may differ, so we do an additional check
// to ensure no path traversal sequences are present:
if (filePath.Contains(".."))
{
context.Response.StatusCode = StatusCodes.Status403Forbidden;
LOGGER.LogWarning("Path traversal attempt detected: {FilePath}", filePath);
return;
}
}
// Check if the file exists:
if (!File.Exists(filePath))
{
context.Response.StatusCode = StatusCodes.Status404NotFound;
LOGGER.LogWarning("Requested file not found: '{FilePath}'", filePath);
return;
}
// Determine the content type:
var contentTypeProvider = new FileExtensionContentTypeProvider();
if (!contentTypeProvider.TryGetContentType(filePath, out var contentType))
contentType = "application/octet-stream";
// Set response headers:
context.Response.ContentType = contentType;
context.Response.Headers.ContentDisposition = $"inline; filename=\"{Path.GetFileName(filePath)}\"";
// Stream the file to the response:
await using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 64 * 1024, useAsync: true);
context.Response.ContentLength = fileStream.Length;
await fileStream.CopyToAsync(context.Response.Body);
}
}

View File

@ -49,7 +49,7 @@ LANG_NAME = "Deutsch (Deutschland)"
UI_TEXT_CONTENT = {} UI_TEXT_CONTENT = {}
-- Objective -- Objective
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::AGENDA::ASSISTANTAGENDA::T1121586136"] = "Ziel" UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::AGENDA::ASSISTANTAGENDA::T1121586136"] = "Zielsetzung"
-- Describe the topic of the meeting, seminar, etc. Is it about quantum computing, software engineering, or is it a general business meeting? -- Describe the topic of the meeting, seminar, etc. Is it about quantum computing, software engineering, or is it a general business meeting?
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::AGENDA::ASSISTANTAGENDA::T12079368"] = "Beschreiben Sie das Thema des Treffens, Seminars usw. Geht es um Quantencomputing, Softwareentwicklung oder handelt es sich um ein allgemeines Geschäftstreffen?" UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::AGENDA::ASSISTANTAGENDA::T12079368"] = "Beschreiben Sie das Thema des Treffens, Seminars usw. Geht es um Quantencomputing, Softwareentwicklung oder handelt es sich um ein allgemeines Geschäftstreffen?"
@ -1515,6 +1515,12 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T4188329028"] = "Nein, b
-- Export Chat to Microsoft Word -- Export Chat to Microsoft Word
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T861873672"] = "Chat in Microsoft Word exportieren" UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T861873672"] = "Chat in Microsoft Word exportieren"
-- The local image file does not exist. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "Die lokale Bilddatei existiert nicht. Das Bild wird übersprungen."
-- Failed to download the image from the URL. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T2996654916"] = "Das Bild konnte nicht von der URL heruntergeladen werden. Das Bild wird übersprungen."
-- The local image file is too large (>10 MB). Skipping the image. -- The local image file is too large (>10 MB). Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T3219823625"] = "Die lokale Bilddatei ist zu groß (>10 MB). Das Bild wird übersprungen." UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T3219823625"] = "Die lokale Bilddatei ist zu groß (>10 MB). Das Bild wird übersprungen."
@ -2970,6 +2976,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdo
-- Load file -- Load file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Datei laden" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Datei laden"
-- Image View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2199753423"] = "Bildansicht"
-- See how we load your file. Review the content before we process it further. -- See how we load your file. Review the content before we process it further.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "So wird Ihre Datei geladen. Überprüfen Sie den Inhalt, bevor wir ihn weiterverarbeiten." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "So wird Ihre Datei geladen. Überprüfen Sie den Inhalt, bevor wir ihn weiterverarbeiten."
@ -2988,6 +2997,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T652739927"] = "Dies is
-- File Path -- File Path
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "Dateipfad" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "Dateipfad"
-- The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T973777830"] = "Die angegebene Datei konnte nicht gefunden werden. Die Datei wurde möglicherweise verschoben, gelöscht, umbenannt oder ist anderweitig nicht zugänglich."
-- Embedding Name -- Embedding Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Name der Einbettung" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Name der Einbettung"
@ -3438,12 +3450,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T1746160064"] = "Hi
-- There aren't any file attachments right now. -- There aren't any file attachments right now.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T2111340711"] = "Derzeit sind keine Dateianhänge vorhanden." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T2111340711"] = "Derzeit sind keine Dateianhänge vorhanden."
-- Document Preview
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T285154968"] = "Dokumentvorschau"
-- The file was deleted, renamed, or moved. -- The file was deleted, renamed, or moved.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3083729256"] = "Die Datei wurde gelöscht, umbenannt oder verschoben." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3083729256"] = "Die Datei wurde gelöscht, umbenannt oder verschoben."
-- Your attached file. -- Your attached file.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3154198222"] = "Ihre angehängte Datei." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3154198222"] = "Ihre angehängte Datei."
-- Preview what we send to the AI.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3160778981"] = "Vorschau dessen, was wir an die KI senden."
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3448155331"] = "Schließen"
-- Your attached files -- Your attached files
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Ihre angehängten Dateien" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Ihre angehängten Dateien"
@ -6060,9 +6081,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T29289275
-- Images are not supported yet -- Images are not supported yet
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T298062956"] = "Bilder werden derzeit nicht unterstützt." UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T298062956"] = "Bilder werden derzeit nicht unterstützt."
-- Images are not supported at this place
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T305247150"] = "Bilder werden an dieser Stelle nicht unterstützt."
-- Executables are not allowed -- Executables are not allowed
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T4167762413"] = "Ausführbare Dateien sind nicht erlaubt" UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T4167762413"] = "Ausführbare Dateien sind nicht erlaubt"
-- Images are not supported by the selected provider and model
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T999194030"] = "Bilder werden vom ausgewählten Anbieter und Modell nicht unterstützt."
-- The hostname is not a valid HTTP(S) URL. -- The hostname is not a valid HTTP(S) URL.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::PROVIDERVALIDATION::T1013354736"] = "Der Hostname ist keine gültige HTTP(S)-URL." UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::PROVIDERVALIDATION::T1013354736"] = "Der Hostname ist keine gültige HTTP(S)-URL."

View File

@ -1515,6 +1515,12 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T4188329028"] = "No, kee
-- Export Chat to Microsoft Word -- Export Chat to Microsoft Word
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T861873672"] = "Export Chat to Microsoft Word" UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T861873672"] = "Export Chat to Microsoft Word"
-- The local image file does not exist. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "The local image file does not exist. Skipping the image."
-- Failed to download the image from the URL. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T2996654916"] = "Failed to download the image from the URL. Skipping the image."
-- The local image file is too large (>10 MB). Skipping the image. -- The local image file is too large (>10 MB). Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T3219823625"] = "The local image file is too large (>10 MB). Skipping the image." UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T3219823625"] = "The local image file is too large (>10 MB). Skipping the image."
@ -2970,6 +2976,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdo
-- Load file -- Load file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Load file" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Load file"
-- Image View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2199753423"] = "Image View"
-- See how we load your file. Review the content before we process it further. -- See how we load your file. Review the content before we process it further.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "See how we load your file. Review the content before we process it further." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "See how we load your file. Review the content before we process it further."
@ -2988,6 +2997,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T652739927"] = "This is
-- File Path -- File Path
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "File Path" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "File Path"
-- The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T973777830"] = "The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible."
-- Embedding Name -- Embedding Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Embedding Name" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Embedding Name"
@ -3438,12 +3450,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T1746160064"] = "He
-- There aren't any file attachments right now. -- There aren't any file attachments right now.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T2111340711"] = "There aren't any file attachments right now." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T2111340711"] = "There aren't any file attachments right now."
-- Document Preview
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T285154968"] = "Document Preview"
-- The file was deleted, renamed, or moved. -- The file was deleted, renamed, or moved.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3083729256"] = "The file was deleted, renamed, or moved." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3083729256"] = "The file was deleted, renamed, or moved."
-- Your attached file. -- Your attached file.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3154198222"] = "Your attached file." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3154198222"] = "Your attached file."
-- Preview what we send to the AI.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3160778981"] = "Preview what we send to the AI."
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3448155331"] = "Close"
-- Your attached files -- Your attached files
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Your attached files" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Your attached files"
@ -6060,9 +6081,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T29289275
-- Images are not supported yet -- Images are not supported yet
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T298062956"] = "Images are not supported yet" UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T298062956"] = "Images are not supported yet"
-- Images are not supported at this place
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T305247150"] = "Images are not supported at this place"
-- Executables are not allowed -- Executables are not allowed
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T4167762413"] = "Executables are not allowed" UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T4167762413"] = "Executables are not allowed"
-- Images are not supported by the selected provider and model
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T999194030"] = "Images are not supported by the selected provider and model"
-- The hostname is not a valid HTTP(S) URL. -- The hostname is not a valid HTTP(S) URL.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::PROVIDERVALIDATION::T1013354736"] = "The hostname is not a valid HTTP(S) URL." UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::PROVIDERVALIDATION::T1013354736"] = "The hostname is not a valid HTTP(S) URL."

View File

@ -192,6 +192,7 @@ internal sealed class Program
programLogger.LogInformation("Initialize internal file system."); programLogger.LogInformation("Initialize internal file system.");
app.Use(Redirect.HandlerContentAsync); app.Use(Redirect.HandlerContentAsync);
app.Use(FileHandler.HandlerAsync);
#if DEBUG #if DEBUG
app.UseStaticFiles(); app.UseStaticFiles();

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.AlibabaCloud; namespace AIStudio.Provider.AlibabaCloud;
public sealed class ProviderAlibabaCloud() : BaseProvider("https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", LOGGER) public sealed class ProviderAlibabaCloud() : BaseProvider(LLMProviders.ALIBABA_CLOUD, "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderAlibabaCloud> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderAlibabaCloud>(); private static readonly ILogger<ProviderAlibabaCloud> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderAlibabaCloud>();
@ -40,24 +40,7 @@ public sealed class ProviderAlibabaCloud() : BaseProvider("https://dashscope-int
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the AlibabaCloud HTTP chat request: // Prepare the AlibabaCloud HTTP chat request:
var alibabaCloudChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest var alibabaCloudChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest

View File

@ -0,0 +1,9 @@
namespace AIStudio.Provider.Anthropic;
public interface ISubContentImageSource
{
/// <summary>
/// The type of the sub-content image.
/// </summary>
public SubContentImageType Type { get; }
}

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Anthropic; namespace AIStudio.Provider.Anthropic;
public sealed class ProviderAnthropic() : BaseProvider("https://api.anthropic.com/v1/", LOGGER) public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, "https://api.anthropic.com/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderAnthropic> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderAnthropic>(); private static readonly ILogger<ProviderAnthropic> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderAnthropic>();
@ -31,9 +31,11 @@ public sealed class ProviderAnthropic() : BaseProvider("https://api.anthropic.co
var apiParameters = this.ParseAdditionalApiParameters("system"); var apiParameters = this.ParseAdditionalApiParameters("system");
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesAsync(
{ this.Provider, chatModel,
Role = n.Role switch
// Anthropic-specific role mapping:
role => role switch
{ {
ChatRole.USER => "user", ChatRole.USER => "user",
ChatRole.AI => "assistant", ChatRole.AI => "assistant",
@ -42,12 +44,25 @@ public sealed class ProviderAnthropic() : BaseProvider("https://api.anthropic.co
_ => "user", _ => "user",
}, },
Content = n.Content switch // Anthropic uses the standard text sub-content:
text => new SubContentText
{ {
ContentText text => await text.PrepareTextContentForAI(), Text = text,
_ => string.Empty, },
// Anthropic-specific image sub-content:
async attachment => new SubContentImage
{
Source = new SubContentBase64Image
{
Data = await attachment.TryAsBase64(token: token) is (true, var base64Content)
? base64Content
: string.Empty,
MediaType = attachment.DetermineMimeType(),
} }
}); }
);
// Prepare the Anthropic HTTP chat request: // Prepare the Anthropic HTTP chat request:
var chatRequest = JsonSerializer.Serialize(new ChatRequest var chatRequest = JsonSerializer.Serialize(new ChatRequest

View File

@ -0,0 +1,10 @@
namespace AIStudio.Provider.Anthropic;
public record SubContentBase64Image : ISubContentImageSource
{
public SubContentImageType Type => SubContentImageType.BASE64;
public string MediaType { get; init; } = string.Empty;
public string Data { get; init; } = string.Empty;
}

View File

@ -0,0 +1,10 @@
using AIStudio.Provider.OpenAI;
namespace AIStudio.Provider.Anthropic;
public record SubContentImage(SubContentType Type, ISubContentImageSource Source) : ISubContent
{
public SubContentImage() : this(SubContentType.IMAGE, new SubContentImageUrl())
{
}
}

View File

@ -0,0 +1,32 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace AIStudio.Provider.Anthropic;
/// <summary>
/// Custom JSON converter for the ISubContentImageSource interface to handle polymorphic serialization.
/// </summary>
/// <remarks>
/// This converter ensures that when serializing ISubContentImageSource objects, all properties
/// of the concrete implementation (e.g., SubContentBase64Image, SubContentImageUrl) are serialized,
/// not just the properties defined in the ISubContentImageSource interface.
/// </remarks>
public sealed class SubContentImageSourceConverter : JsonConverter<ISubContentImageSource>
{
private static readonly ILogger<SubContentImageSourceConverter> LOGGER = Program.LOGGER_FACTORY.CreateLogger<SubContentImageSourceConverter>();
public override ISubContentImageSource? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
// Deserialization is not needed for request objects, as sub-content image sources are only serialized
// when sending requests to LLM providers.
LOGGER.LogError("Deserializing ISubContentImageSource is not supported. This converter is only used for serializing request messages.");
return null;
}
public override void Write(Utf8JsonWriter writer, ISubContentImageSource value, JsonSerializerOptions options)
{
// Serialize the actual concrete type (e.g., SubContentBase64Image, SubContentImageUrl) instead of just the ISubContentImageSource interface.
// This ensures all properties of the concrete type are included in the JSON output.
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
}

View File

@ -0,0 +1,7 @@
namespace AIStudio.Provider.Anthropic;
public enum SubContentImageType
{
URL,
BASE64
}

View File

@ -0,0 +1,8 @@
namespace AIStudio.Provider.Anthropic;
public record SubContentImageUrl : ISubContentImageSource
{
public SubContentImageType Type => SubContentImageType.URL;
public string Url { get; init; } = string.Empty;
}

View File

@ -1,8 +1,10 @@
using System.Net; using System.Net;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Serialization;
using AIStudio.Chat; using AIStudio.Chat;
using AIStudio.Provider.Anthropic;
using AIStudio.Provider.OpenAI; using AIStudio.Provider.OpenAI;
using AIStudio.Settings; using AIStudio.Settings;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
@ -40,18 +42,28 @@ public abstract class BaseProvider : IProvider, ISecretId
protected static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new() protected static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
{ {
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
Converters = { new AnnotationConverter(), new MessageBaseConverter() }, Converters =
{
new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower),
new AnnotationConverter(),
new MessageBaseConverter(),
new SubContentConverter(),
new SubContentImageSourceConverter(),
new SubContentImageUrlConverter(),
},
AllowTrailingCommas = false AllowTrailingCommas = false
}; };
/// <summary> /// <summary>
/// Constructor for the base provider. /// Constructor for the base provider.
/// </summary> /// </summary>
/// <param name="provider">The provider enum value.</param>
/// <param name="url">The base URL for the provider.</param> /// <param name="url">The base URL for the provider.</param>
/// <param name="logger">The logger to use.</param> /// <param name="logger">The logger to use.</param>
protected BaseProvider(string url, ILogger logger) protected BaseProvider(LLMProviders provider, string url, ILogger logger)
{ {
this.logger = logger; this.logger = logger;
this.Provider = provider;
// Set the base URL: // Set the base URL:
this.httpClient.BaseAddress = new(url); this.httpClient.BaseAddress = new(url);
@ -59,6 +71,9 @@ public abstract class BaseProvider : IProvider, ISecretId
#region Handling of IProvider, which all providers must implement #region Handling of IProvider, which all providers must implement
/// <inheritdoc />
public LLMProviders Provider { get; }
/// <inheritdoc /> /// <inheritdoc />
public abstract string Id { get; } public abstract string Id { get; }

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.DeepSeek; namespace AIStudio.Provider.DeepSeek;
public sealed class ProviderDeepSeek() : BaseProvider("https://api.deepseek.com/", LOGGER) public sealed class ProviderDeepSeek() : BaseProvider(LLMProviders.DEEP_SEEK, "https://api.deepseek.com/", LOGGER)
{ {
private static readonly ILogger<ProviderDeepSeek> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderDeepSeek>(); private static readonly ILogger<ProviderDeepSeek> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderDeepSeek>();
@ -40,24 +40,7 @@ public sealed class ProviderDeepSeek() : BaseProvider("https://api.deepseek.com/
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the DeepSeek HTTP chat request: // Prepare the DeepSeek HTTP chat request:
var deepSeekChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest var deepSeekChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Fireworks; namespace AIStudio.Provider.Fireworks;
public class ProviderFireworks() : BaseProvider("https://api.fireworks.ai/inference/v1/", LOGGER) public class ProviderFireworks() : BaseProvider(LLMProviders.FIREWORKS, "https://api.fireworks.ai/inference/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderFireworks> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderFireworks>(); private static readonly ILogger<ProviderFireworks> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderFireworks>();
@ -40,24 +40,7 @@ public class ProviderFireworks() : BaseProvider("https://api.fireworks.ai/infere
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the Fireworks HTTP chat request: // Prepare the Fireworks HTTP chat request:
var fireworksChatRequest = JsonSerializer.Serialize(new ChatRequest var fireworksChatRequest = JsonSerializer.Serialize(new ChatRequest

View File

@ -1,13 +0,0 @@
namespace AIStudio.Provider.Fireworks;
/// <summary>
/// Chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>
public record TextMessage(string Content, string Role) : IMessage<string>
{
public TextMessage() : this(string.Empty, string.Empty)
{
}
}

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.GWDG; namespace AIStudio.Provider.GWDG;
public sealed class ProviderGWDG() : BaseProvider("https://chat-ai.academiccloud.de/v1/", LOGGER) public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, "https://chat-ai.academiccloud.de/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderGWDG> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGWDG>(); private static readonly ILogger<ProviderGWDG> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGWDG>();
@ -40,24 +40,7 @@ public sealed class ProviderGWDG() : BaseProvider("https://chat-ai.academiccloud
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the GWDG HTTP chat request: // Prepare the GWDG HTTP chat request:
var gwdgChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest var gwdgChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Google; namespace AIStudio.Provider.Google;
public class ProviderGoogle() : BaseProvider("https://generativelanguage.googleapis.com/v1beta/", LOGGER) public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, "https://generativelanguage.googleapis.com/v1beta/", LOGGER)
{ {
private static readonly ILogger<ProviderGoogle> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGoogle>(); private static readonly ILogger<ProviderGoogle> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGoogle>();
@ -40,24 +40,7 @@ public class ProviderGoogle() : BaseProvider("https://generativelanguage.googlea
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the Google HTTP chat request: // Prepare the Google HTTP chat request:
var geminiChatRequest = JsonSerializer.Serialize(new ChatRequest var geminiChatRequest = JsonSerializer.Serialize(new ChatRequest

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Groq; namespace AIStudio.Provider.Groq;
public class ProviderGroq() : BaseProvider("https://api.groq.com/openai/v1/", LOGGER) public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, "https://api.groq.com/openai/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderGroq> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGroq>(); private static readonly ILogger<ProviderGroq> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderGroq>();
@ -40,24 +40,7 @@ public class ProviderGroq() : BaseProvider("https://api.groq.com/openai/v1/", LO
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the OpenAI HTTP chat request: // Prepare the OpenAI HTTP chat request:
var groqChatRequest = JsonSerializer.Serialize(new ChatRequest var groqChatRequest = JsonSerializer.Serialize(new ChatRequest

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Helmholtz; namespace AIStudio.Provider.Helmholtz;
public sealed class ProviderHelmholtz() : BaseProvider("https://api.helmholtz-blablador.fz-juelich.de/v1/", LOGGER) public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, "https://api.helmholtz-blablador.fz-juelich.de/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderHelmholtz> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderHelmholtz>(); private static readonly ILogger<ProviderHelmholtz> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderHelmholtz>();
@ -40,24 +40,7 @@ public sealed class ProviderHelmholtz() : BaseProvider("https://api.helmholtz-bl
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the Helmholtz HTTP chat request: // Prepare the Helmholtz HTTP chat request:
var helmholtzChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest var helmholtzChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest

View File

@ -13,9 +13,9 @@ public sealed class ProviderHuggingFace : BaseProvider
{ {
private static readonly ILogger<ProviderHuggingFace> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderHuggingFace>(); private static readonly ILogger<ProviderHuggingFace> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderHuggingFace>();
public ProviderHuggingFace(HFInferenceProvider hfProvider, Model model) : base($"https://router.huggingface.co/{hfProvider.Endpoints(model)}", LOGGER) public ProviderHuggingFace(HFInferenceProvider hfProvider, Model model) : base(LLMProviders.HUGGINGFACE, $"https://router.huggingface.co/{hfProvider.Endpoints(model)}", LOGGER)
{ {
LOGGER.LogInformation($"We use the inferende provider '{hfProvider}'. Thus we use the base URL 'https://router.huggingface.co/{hfProvider.Endpoints(model)}'."); LOGGER.LogInformation($"We use the inference provider '{hfProvider}'. Thus we use the base URL 'https://router.huggingface.co/{hfProvider.Endpoints(model)}'.");
} }
#region Implementation of IProvider #region Implementation of IProvider
@ -45,24 +45,7 @@ public sealed class ProviderHuggingFace : BaseProvider
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var message = await chatThread.Blocks.BuildMessages(async n => new TextMessage var message = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the HuggingFace HTTP chat request: // Prepare the HuggingFace HTTP chat request:
var huggingfaceChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest var huggingfaceChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest

View File

@ -8,6 +8,11 @@ namespace AIStudio.Provider;
/// </summary> /// </summary>
public interface IProvider public interface IProvider
{ {
/// <summary>
/// The provider type.
/// </summary>
public LLMProviders Provider { get; }
/// <summary> /// <summary>
/// The provider's ID. /// The provider's ID.
/// </summary> /// </summary>

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Mistral; namespace AIStudio.Provider.Mistral;
public sealed class ProviderMistral() : BaseProvider("https://api.mistral.ai/v1/", LOGGER) public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, "https://api.mistral.ai/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderMistral> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderMistral>(); private static readonly ILogger<ProviderMistral> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderMistral>();
@ -38,24 +38,7 @@ public sealed class ProviderMistral() : BaseProvider("https://api.mistral.ai/v1/
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the Mistral HTTP chat request: // Prepare the Mistral HTTP chat request:
var mistralChatRequest = JsonSerializer.Serialize(new ChatRequest var mistralChatRequest = JsonSerializer.Serialize(new ChatRequest

View File

@ -1,13 +0,0 @@
namespace AIStudio.Provider.Mistral;
/// <summary>
/// Text chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>
public record TextMessage(string Content, string Role) : IMessage<string>
{
public TextMessage() : this(string.Empty, string.Empty)
{
}
}

View File

@ -9,6 +9,8 @@ public class NoProvider : IProvider
{ {
#region Implementation of IProvider #region Implementation of IProvider
public LLMProviders Provider => LLMProviders.NONE;
public string Id => "none"; public string Id => "none";
public string InstanceName { get; set; } = "None"; public string InstanceName { get; set; } = "None";

View File

@ -0,0 +1,12 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Contract for sub-content in multimodal messages.
/// </summary>
public interface ISubContent
{
/// <summary>
/// The type of the sub-content.
/// </summary>
public SubContentType Type { get; init; }
}

View File

@ -0,0 +1,19 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Contract for nested image URL sub-content.
/// </summary>
/// <remarks>
/// Some providers use a nested object format for image URLs:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This interface represents the inner object with the "url" property.
/// </remarks>
public interface ISubContentImageUrl
{
/// <summary>
/// The URL or base64-encoded data URI of the image.
/// </summary>
public string Url { get; init; }
}

View File

@ -0,0 +1,13 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// A multimodal chat message model that can contain various types of content.
/// </summary>
/// <param name="Content">The list of sub-contents in the message.</param>
/// <param name="Role">The role of the message.</param>
public record MultimodalMessage(List<ISubContent> Content, string Role) : IMessage<List<ISubContent>>
{
public MultimodalMessage() : this([], string.Empty)
{
}
}

View File

@ -11,7 +11,7 @@ namespace AIStudio.Provider.OpenAI;
/// <summary> /// <summary>
/// The OpenAI provider. /// The OpenAI provider.
/// </summary> /// </summary>
public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/", LOGGER) public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, "https://api.openai.com/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderOpenAI> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderOpenAI>(); private static readonly ILogger<ProviderOpenAI> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderOpenAI>();
@ -59,7 +59,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
}; };
// Read the model capabilities: // Read the model capabilities:
var modelCapabilities = ProviderExtensions.GetModelCapabilitiesOpenAI(chatModel); var modelCapabilities = this.Provider.GetModelCapabilities(chatModel);
// Check if we are using the Responses API or the Chat Completion API: // Check if we are using the Responses API or the Chat Completion API:
var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API); var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API);
@ -90,9 +90,11 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
var apiParameters = this.ParseAdditionalApiParameters("input", "store", "tools"); var apiParameters = this.ParseAdditionalApiParameters("input", "store", "tools");
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesAsync(
{ this.Provider, chatModel,
Role = n.Role switch
// OpenAI-specific role mapping:
role => role switch
{ {
ChatRole.USER => "user", ChatRole.USER => "user",
ChatRole.AI => "assistant", ChatRole.AI => "assistant",
@ -102,10 +104,44 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
_ => "user", _ => "user",
}, },
Content = n.Content switch // OpenAI's text sub-content depends on the model, whether we are using
// the Responses API or the Chat Completion API:
text => usingResponsesAPI switch
{ {
ContentText text => await text.PrepareTextContentForAI(), // Responses API uses INPUT_TEXT:
_ => string.Empty, true => new SubContentInputText
{
Text = text,
},
// Chat Completion API uses TEXT:
false => new SubContentText
{
Text = text,
},
},
// OpenAI's image sub-content depends on the model as well,
// whether we are using the Responses API or the Chat Completion API:
async attachment => usingResponsesAPI switch
{
// Responses API uses INPUT_IMAGE:
true => new SubContentInputImage
{
ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content)
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
: string.Empty,
},
// Chat Completion API uses IMAGE_URL:
false => new SubContentImageUrlNested
{
ImageUrl = new SubContentImageUrlData
{
Url = await attachment.TryAsBase64(token: token) is (true, var base64Content)
? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
: string.Empty,
},
} }
}); });

View File

@ -0,0 +1,11 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image sub-content for multimodal messages.
/// </summary>
public record SubContentImageUrl(SubContentType Type, string ImageUrl) : ISubContent
{
public SubContentImageUrl() : this(SubContentType.IMAGE_URL, string.Empty)
{
}
}

View File

@ -0,0 +1,17 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Represents the inner object of a nested image URL sub-content.
/// </summary>
/// <remarks>
/// This record is used when the provider expects the format:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This class represents the inner <c>{ "url": "..." }</c> part.
/// </remarks>
public record SubContentImageUrlData : ISubContentImageUrl
{
/// <inheritdoc />
public string Url { get; init; } = string.Empty;
}

View File

@ -0,0 +1,18 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image sub-content for multimodal messages using nested URL format.
/// </summary>
/// <remarks>
/// This record is used when the provider expects the format:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// Used by LM Studio, VLLM, and other OpenAI-compatible providers.
/// </remarks>
public record SubContentImageUrlNested(SubContentType Type, ISubContentImageUrl ImageUrl) : ISubContent
{
public SubContentImageUrlNested() : this(SubContentType.IMAGE_URL, new SubContentImageUrlData())
{
}
}

View File

@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Image input sub-content for multimodal messages.
/// </summary>
/// <remarks>
/// Right now, this is used only by OpenAI in its responses API.
/// </remarks>
public record SubContentInputImage(SubContentType Type, string ImageUrl) : ISubContent
{
public SubContentInputImage() : this(SubContentType.INPUT_IMAGE, string.Empty)
{
}
}

View File

@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Text input sub-content for multimodal messages.
/// </summary>
/// <remarks>
/// Right now, this is used only by OpenAI in its responses API.
/// </remarks>
public record SubContentInputText(SubContentType Type, string Text) : ISubContent
{
public SubContentInputText() : this(SubContentType.INPUT_TEXT, string.Empty)
{
}
}

View File

@ -0,0 +1,11 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Text sub-content for multimodal messages.
/// </summary>
public record SubContentText(SubContentType Type, string Text) : ISubContent
{
public SubContentText() : this(SubContentType.TEXT, string.Empty)
{
}
}

View File

@ -1,7 +1,7 @@
namespace AIStudio.Provider.OpenAI; namespace AIStudio.Provider.OpenAI;
/// <summary> /// <summary>
/// Chat message model. /// Standard text-based chat message model.
/// </summary> /// </summary>
/// <param name="Content">The text content of the message.</param> /// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param> /// <param name="Role">The role of the message.</param>

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.OpenRouter; namespace AIStudio.Provider.OpenRouter;
public sealed class ProviderOpenRouter() : BaseProvider("https://openrouter.ai/api/v1/", LOGGER) public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER, "https://openrouter.ai/api/v1/", LOGGER)
{ {
private const string PROJECT_WEBSITE = "https://github.com/MindWorkAI/AI-Studio"; private const string PROJECT_WEBSITE = "https://github.com/MindWorkAI/AI-Studio";
private const string PROJECT_NAME = "MindWork AI Studio"; private const string PROJECT_NAME = "MindWork AI Studio";
@ -43,24 +43,7 @@ public sealed class ProviderOpenRouter() : BaseProvider("https://openrouter.ai/a
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the OpenRouter HTTP chat request: // Prepare the OpenRouter HTTP chat request:
var openRouterChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest var openRouterChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.Perplexity; namespace AIStudio.Provider.Perplexity;
public sealed class ProviderPerplexity() : BaseProvider("https://api.perplexity.ai/", LOGGER) public sealed class ProviderPerplexity() : BaseProvider(LLMProviders.PERPLEXITY, "https://api.perplexity.ai/", LOGGER)
{ {
private static readonly ILogger<ProviderPerplexity> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderPerplexity>(); private static readonly ILogger<ProviderPerplexity> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderPerplexity>();
@ -49,24 +49,7 @@ public sealed class ProviderPerplexity() : BaseProvider("https://api.perplexity.
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage() var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the Perplexity HTTP chat request: // Prepare the Perplexity HTTP chat request:
var perplexityChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest var perplexityChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.SelfHosted; namespace AIStudio.Provider.SelfHosted;
public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvider($"{hostname}{host.BaseURL()}", LOGGER) public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvider(LLMProviders.SELF_HOSTED, $"{hostname}{host.BaseURL()}", LOGGER)
{ {
private static readonly ILogger<ProviderSelfHosted> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderSelfHosted>(); private static readonly ILogger<ProviderSelfHosted> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderSelfHosted>();
@ -35,25 +35,14 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide
// Parse the API parameters: // Parse the API parameters:
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages. The image format depends on the host:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage // - Ollama uses the direct image URL format: { "type": "image_url", "image_url": "data:..." }
// - LM Studio, vLLM, and llama.cpp use the nested image URL format: { "type": "image_url", "image_url": { "url": "data:..." } }
var messages = host switch
{ {
Role = n.Role switch Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel),
{ _ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel),
ChatRole.USER => "user", };
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the OpenAI HTTP chat request: // Prepare the OpenAI HTTP chat request:
var providerChatRequest = JsonSerializer.Serialize(new ChatRequest var providerChatRequest = JsonSerializer.Serialize(new ChatRequest

View File

@ -1,13 +0,0 @@
namespace AIStudio.Provider.SelfHosted;
/// <summary>
/// Chat message model.
/// </summary>
/// <param name="Content">The text content of the message.</param>
/// <param name="Role">The role of the message.</param>
public record TextMessage(string Content, string Role) : IMessage<string>
{
public TextMessage() : this(string.Empty, string.Empty)
{
}
}

View File

@ -0,0 +1,34 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using AIStudio.Provider.OpenAI;
namespace AIStudio.Provider;
/// <summary>
/// Custom JSON converter for the ISubContent interface to handle polymorphic serialization.
/// </summary>
/// <remarks>
/// This converter ensures that when serializing ISubContent objects, all properties
/// of the concrete implementation (e.g., SubContentText, SubContentImageUrl) are serialized,
/// not just the properties defined in the ISubContent interface.
/// </remarks>
public sealed class SubContentConverter : JsonConverter<ISubContent>
{
private static readonly ILogger<SubContentConverter> LOGGER = Program.LOGGER_FACTORY.CreateLogger<SubContentConverter>();
public override ISubContent? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
// Deserialization is not needed for request objects, as sub-content is only serialized
// when sending requests to LLM providers.
LOGGER.LogError("Deserializing ISubContent is not supported. This converter is only used for serializing request messages.");
return null;
}
public override void Write(Utf8JsonWriter writer, ISubContent value, JsonSerializerOptions options)
{
// Serialize the actual concrete type (e.g., SubContentText, SubContentImageUrl) instead of just the ISubContent interface.
// This ensures all properties of the concrete type are included in the JSON output.
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
}

View File

@ -0,0 +1,34 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using AIStudio.Provider.OpenAI;
namespace AIStudio.Provider;
/// <summary>
/// Custom JSON converter for the ISubContentImageUrl interface to handle polymorphic serialization.
/// </summary>
/// <remarks>
/// This converter ensures that when serializing ISubContentImageUrl objects, all properties
/// of the concrete implementation (e.g., SubContentImageUrlData) are serialized,
/// not just the properties defined in the ISubContentImageUrl interface.
/// </remarks>
public sealed class SubContentImageUrlConverter : JsonConverter<ISubContentImageUrl>
{
private static readonly ILogger<SubContentImageUrlConverter> LOGGER = Program.LOGGER_FACTORY.CreateLogger<SubContentImageUrlConverter>();
public override ISubContentImageUrl? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
// Deserialization is not needed for request objects, as sub-content image URLs are only serialized
// when sending requests to LLM providers.
LOGGER.LogError("Deserializing ISubContentImageUrl is not supported. This converter is only used for serializing request messages.");
return null;
}
public override void Write(Utf8JsonWriter writer, ISubContentImageUrl value, JsonSerializerOptions options)
{
// Serialize the actual concrete type (e.g., SubContentImageUrlData) instead of just the ISubContentImageUrl interface.
// This ensures all properties of the concrete type are included in the JSON output.
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
}

View File

@ -0,0 +1,39 @@
namespace AIStudio.Provider;
/// <summary>
/// Sub content types for OpenAI-compatible API interactions when using multimodal messages.
/// </summary>
public enum SubContentType
{
/// <summary>
/// Default type for user prompts in multimodal messages. This type is supported across all providers.
/// </summary>
TEXT,
/// <summary>
/// Right now only supported by OpenAI and it's responses API. Even other providers that support multimodal messages
/// and the responses API do not support this type. They use TEXT instead.
/// </summary>
INPUT_TEXT,
/// <summary>
/// Right now only supported by OpenAI and it's responses API. Even other providers that support multimodal messages
/// and the responses API do not support this type. They use IMAGE_URL instead.
/// </summary>
INPUT_IMAGE,
/// <summary>
/// Default type for images in multimodal messages. This type is supported across all providers.
/// </summary>
IMAGE_URL,
/// <summary>
/// The image type is used exclusively by Anthropic's messages API.
/// </summary>
IMAGE,
/// <summary>
/// Right now only supported by OpenAI (responses & chat completion API), Google (chat completions API), and Mistral (chat completions API).
/// </summary>
INPUT_AUDIO,
}

View File

@ -9,7 +9,7 @@ using AIStudio.Settings;
namespace AIStudio.Provider.X; namespace AIStudio.Provider.X;
public sealed class ProviderX() : BaseProvider("https://api.x.ai/v1/", LOGGER) public sealed class ProviderX() : BaseProvider(LLMProviders.X, "https://api.x.ai/v1/", LOGGER)
{ {
private static readonly ILogger<ProviderX> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderX>(); private static readonly ILogger<ProviderX> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderX>();
@ -40,24 +40,7 @@ public sealed class ProviderX() : BaseProvider("https://api.x.ai/v1/", LOGGER)
var apiParameters = this.ParseAdditionalApiParameters(); var apiParameters = this.ParseAdditionalApiParameters();
// Build the list of messages: // Build the list of messages:
var messages = await chatThread.Blocks.BuildMessages(async n => new TextMessage var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel);
{
Role = n.Role switch
{
ChatRole.USER => "user",
ChatRole.AI => "assistant",
ChatRole.AGENT => "assistant",
ChatRole.SYSTEM => "system",
_ => "user",
},
Content = n.Content switch
{
ContentText text => await text.PrepareTextContentForAI(),
_ => string.Empty,
}
});
// Prepare the xAI HTTP chat request: // Prepare the xAI HTTP chat request:
var xChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest var xChatRequest = JsonSerializer.Serialize(new ChatCompletionAPIRequest

View File

@ -74,7 +74,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
LatestUserPrompt = lastUserPrompt switch LatestUserPrompt = lastUserPrompt switch
{ {
ContentText text => text.Text, ContentText text => text.Text,
ContentImage image => await image.AsBase64(token), ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
? base64Image
: string.Empty,
_ => string.Empty _ => string.Empty
}, },

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesAlibaba(Model model) private static List<Capability> GetModelCapabilitiesAlibaba(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesAnthropic(Model model) private static List<Capability> GetModelCapabilitiesAnthropic(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesDeepSeek(Model model) private static List<Capability> GetModelCapabilitiesDeepSeek(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesGoogle(Model model) private static List<Capability> GetModelCapabilitiesGoogle(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesMistral(Model model) private static List<Capability> GetModelCapabilitiesMistral(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesOpenAI(Model model) private static List<Capability> GetModelCapabilitiesOpenAI(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesOpenRouter(Model model) private static List<Capability> GetModelCapabilitiesOpenRouter(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesOpenSource(Model model) private static List<Capability> GetModelCapabilitiesOpenSource(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();
@ -102,6 +102,15 @@ public static partial class ProviderExtensions
Capability.CHAT_COMPLETION_API, Capability.CHAT_COMPLETION_API,
]; ];
if(modelName.IndexOf("-vl-") is not -1)
return [
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
return [ return [
Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, Capability.TEXT_INPUT, Capability.TEXT_OUTPUT,
Capability.CHAT_COMPLETION_API, Capability.CHAT_COMPLETION_API,
@ -159,7 +168,8 @@ public static partial class ProviderExtensions
Capability.CHAT_COMPLETION_API, Capability.CHAT_COMPLETION_API,
]; ];
if (modelName.IndexOf("3.1") is not -1) if (modelName.IndexOf("3.1") is not -1 ||
modelName.IndexOf("3.2") is not -1)
return return
[ [
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
@ -241,6 +251,43 @@ public static partial class ProviderExtensions
]; ];
} }
//
// Z AI / GLM models:
//
if (modelName.IndexOf("glm") is not -1)
{
if(modelName.IndexOf("v") is not -1)
return
[
Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT,
Capability.TEXT_OUTPUT,
Capability.OPTIONAL_REASONING,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
if (modelName.IndexOf("glm-4-") is not -1)
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.CHAT_COMPLETION_API,
];
return
[
Capability.TEXT_INPUT,
Capability.TEXT_OUTPUT,
Capability.FUNCTION_CALLING,
Capability.OPTIONAL_REASONING,
Capability.CHAT_COMPLETION_API,
];
}
// Default: // Default:
return [ return [
Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, Capability.TEXT_INPUT, Capability.TEXT_OUTPUT,

View File

@ -4,7 +4,7 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilitiesPerplexity(Model model) private static List<Capability> GetModelCapabilitiesPerplexity(Model model)
{ {
var modelName = model.Id.ToLowerInvariant().AsSpan(); var modelName = model.Id.ToLowerInvariant().AsSpan();

View File

@ -4,26 +4,39 @@ namespace AIStudio.Settings;
public static partial class ProviderExtensions public static partial class ProviderExtensions
{ {
public static List<Capability> GetModelCapabilities(this Provider provider) => provider.UsedLLMProvider switch /// <summary>
/// Get the capabilities of the model used by the configured provider.
/// </summary>
/// <param name="provider">The configured provider.</param>
/// <returns>The capabilities of the configured model.</returns>
public static List<Capability> GetModelCapabilities(this Provider provider) => provider.UsedLLMProvider.GetModelCapabilities(provider.Model);
/// <summary>
/// Get the capabilities of a model for a specific provider.
/// </summary>
/// <param name="provider">The LLM provider.</param>
/// <param name="model">The model to get the capabilities for.</param>
/// <returns>>The capabilities of the model.</returns>
public static List<Capability> GetModelCapabilities(this LLMProviders provider, Model model) => provider switch
{ {
LLMProviders.OPEN_AI => GetModelCapabilitiesOpenAI(provider.Model), LLMProviders.OPEN_AI => GetModelCapabilitiesOpenAI(model),
LLMProviders.MISTRAL => GetModelCapabilitiesMistral(provider.Model), LLMProviders.MISTRAL => GetModelCapabilitiesMistral(model),
LLMProviders.ANTHROPIC => GetModelCapabilitiesAnthropic(provider.Model), LLMProviders.ANTHROPIC => GetModelCapabilitiesAnthropic(model),
LLMProviders.GOOGLE => GetModelCapabilitiesGoogle(provider.Model), LLMProviders.GOOGLE => GetModelCapabilitiesGoogle(model),
LLMProviders.X => GetModelCapabilitiesOpenSource(provider.Model), LLMProviders.X => GetModelCapabilitiesOpenSource(model),
LLMProviders.DEEP_SEEK => GetModelCapabilitiesDeepSeek(provider.Model), LLMProviders.DEEP_SEEK => GetModelCapabilitiesDeepSeek(model),
LLMProviders.ALIBABA_CLOUD => GetModelCapabilitiesAlibaba(provider.Model), LLMProviders.ALIBABA_CLOUD => GetModelCapabilitiesAlibaba(model),
LLMProviders.PERPLEXITY => GetModelCapabilitiesPerplexity(provider.Model), LLMProviders.PERPLEXITY => GetModelCapabilitiesPerplexity(model),
LLMProviders.OPEN_ROUTER => GetModelCapabilitiesOpenRouter(provider.Model), LLMProviders.OPEN_ROUTER => GetModelCapabilitiesOpenRouter(model),
LLMProviders.GROQ => GetModelCapabilitiesOpenSource(provider.Model), LLMProviders.GROQ => GetModelCapabilitiesOpenSource(model),
LLMProviders.FIREWORKS => GetModelCapabilitiesOpenSource(provider.Model), LLMProviders.FIREWORKS => GetModelCapabilitiesOpenSource(model),
LLMProviders.HUGGINGFACE => GetModelCapabilitiesOpenSource(provider.Model), LLMProviders.HUGGINGFACE => GetModelCapabilitiesOpenSource(model),
LLMProviders.HELMHOLTZ => GetModelCapabilitiesOpenSource(provider.Model), LLMProviders.HELMHOLTZ => GetModelCapabilitiesOpenSource(model),
LLMProviders.GWDG => GetModelCapabilitiesOpenSource(provider.Model), LLMProviders.GWDG => GetModelCapabilitiesOpenSource(model),
LLMProviders.SELF_HOSTED => GetModelCapabilitiesOpenSource(provider.Model), LLMProviders.SELF_HOSTED => GetModelCapabilitiesOpenSource(model),
_ => [] _ => []
}; };

View File

@ -81,7 +81,9 @@ public static class IRetrievalContextExtensions
sb.AppendLine(); sb.AppendLine();
sb.AppendLine("Matched image content as base64-encoded data:"); sb.AppendLine("Matched image content as base64-encoded data:");
sb.AppendLine("````"); sb.AppendLine("````");
sb.AppendLine(await imageContext.AsBase64(token)); sb.AppendLine(await imageContext.TryAsBase64(token) is (success: true, { } base64Image)
? base64Image
: string.Empty);
sb.AppendLine("````"); sb.AppendLine("````");
break; break;

View File

@ -1,3 +1,5 @@
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Rust; using AIStudio.Tools.Rust;
@ -10,15 +12,38 @@ public static class FileExtensionValidation
{ {
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileExtensionValidation).Namespace, nameof(FileExtensionValidation)); private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileExtensionValidation).Namespace, nameof(FileExtensionValidation));
/// <summary>
/// Defines the use cases for file extension validation.
/// </summary>
public enum UseCase
{
/// <summary>
/// No specific use case; general validation.
/// </summary>
NONE,
/// <summary>
/// Validating for directly loading content into the UI. In this state, there might be no provider selected yet.
/// </summary>
DIRECTLY_LOADING_CONTENT,
/// <summary>
/// Validating for attaching content to a message or prompt.
/// </summary>
ATTACHING_CONTENT,
}
/// <summary> /// <summary>
/// Validates the file extension and sends appropriate MessageBus notifications when invalid. /// Validates the file extension and sends appropriate MessageBus notifications when invalid.
/// </summary> /// </summary>
/// <param name="useCae">The validation use case.</param>
/// <param name="filePath">The file path to validate.</param> /// <param name="filePath">The file path to validate.</param>
/// <param name="provider">The selected provider.</param>
/// <returns>True if valid, false if invalid (error/warning already sent via MessageBus).</returns> /// <returns>True if valid, false if invalid (error/warning already sent via MessageBus).</returns>
public static async Task<bool> IsExtensionValidWithNotifyAsync(string filePath) public static async Task<bool> IsExtensionValidWithNotifyAsync(UseCase useCae, string filePath, Settings.Provider? provider = null)
{ {
var ext = Path.GetExtension(filePath).TrimStart('.'); var ext = Path.GetExtension(filePath).TrimStart('.').ToLowerInvariant();
if (Array.Exists(FileTypeFilter.Executables.FilterExtensions, x => x.Equals(ext, StringComparison.OrdinalIgnoreCase))) if(FileTypeFilter.Executables.FilterExtensions.Contains(ext))
{ {
await MessageBus.INSTANCE.SendError(new( await MessageBus.INSTANCE.SendError(new(
Icons.Material.Filled.AppBlocking, Icons.Material.Filled.AppBlocking,
@ -26,15 +51,39 @@ public static class FileExtensionValidation
return false; return false;
} }
if (Array.Exists(FileTypeFilter.AllImages.FilterExtensions, x => x.Equals(ext, StringComparison.OrdinalIgnoreCase))) var capabilities = provider?.GetModelCapabilities() ?? new();
if (FileTypeFilter.AllImages.FilterExtensions.Contains(ext))
{ {
switch (useCae)
{
// In this use case, we cannot guarantee that a provider is selected yet:
case UseCase.DIRECTLY_LOADING_CONTENT:
await MessageBus.INSTANCE.SendWarning(new(
Icons.Material.Filled.ImageNotSupported,
TB("Images are not supported at this place")));
return false;
// In this use case, we can check the provider capabilities:
case UseCase.ATTACHING_CONTENT when capabilities.Contains(Capability.SINGLE_IMAGE_INPUT) ||
capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT):
return true;
// We know that images are not supported:
case UseCase.ATTACHING_CONTENT:
await MessageBus.INSTANCE.SendWarning(new(
Icons.Material.Filled.ImageNotSupported,
TB("Images are not supported by the selected provider and model")));
return false;
default:
await MessageBus.INSTANCE.SendWarning(new( await MessageBus.INSTANCE.SendWarning(new(
Icons.Material.Filled.ImageNotSupported, Icons.Material.Filled.ImageNotSupported,
TB("Images are not supported yet"))); TB("Images are not supported yet")));
return false; return false;
} }
}
if (Array.Exists(FileTypeFilter.AllVideos.FilterExtensions, x => x.Equals(ext, StringComparison.OrdinalIgnoreCase))) if(FileTypeFilter.AllVideos.FilterExtensions.Contains(ext))
{ {
await MessageBus.INSTANCE.SendWarning(new( await MessageBus.INSTANCE.SendWarning(new(
Icons.Material.Filled.FeaturedVideo, Icons.Material.Filled.FeaturedVideo,
@ -42,7 +91,7 @@ public static class FileExtensionValidation
return false; return false;
} }
if (Array.Exists(FileTypeFilter.AllAudio.FilterExtensions, x => x.Equals(ext, StringComparison.OrdinalIgnoreCase))) if(FileTypeFilter.AllAudio.FilterExtensions.Contains(ext))
{ {
await MessageBus.INSTANCE.SendWarning(new( await MessageBus.INSTANCE.SendWarning(new(
Icons.Material.Filled.AudioFile, Icons.Material.Filled.AudioFile,

View File

@ -2,16 +2,18 @@
- Added support for newer Mistral models (Mistral 3, Voxtral, and Magistral). - Added support for newer Mistral models (Mistral 3, Voxtral, and Magistral).
- Added support for the new OpenAI model GPT 5.2. - Added support for the new OpenAI model GPT 5.2.
- Added support for OpenRouter as LLM and embedding provider. - Added support for OpenRouter as LLM and embedding provider.
- Added support for multimodal processing (documents and images for now), when the selected LLM supports it.
- Added a description field to local data sources (preview feature) so that the data selection agent has more information about which data each local source contains when selecting data sources. - Added a description field to local data sources (preview feature) so that the data selection agent has more information about which data each local source contains when selecting data sources.
- Added the ability to use file attachments in chat. This is the initial implementation of this feature. We will continue to develop this feature and refine it further based on user feedback. Many thanks to Sabrina `Sabrina-devops` for this wonderful contribution. - Added the ability to use file attachments (including images) in chat. This is the initial implementation of this feature. We will continue to develop this feature and refine it further based on user feedback. Many thanks to Sabrina `Sabrina-devops` for this wonderful contribution.
- Improved the document analysis assistant (in preview) by adding descriptions to the different sections. - Improved the document analysis assistant (in preview) by adding descriptions to the different sections.
- Improved the document preview dialog for the document analysis assistant (in preview), providing Markdown and plain text views for attached files. - Improved the document analysis assistant (in preview) by allowing users to use images as input files in addition to documents.
- Improved the document preview dialog for the document analysis assistant (in preview), providing Markdown, image and plain text views for attached files.
- Improved the Pandoc handling for the document analysis assistant (in preview) and file attachments in chat. When Pandoc is not installed and users attempt to attach files, users are now prompted to install Pandoc first. - Improved the Pandoc handling for the document analysis assistant (in preview) and file attachments in chat. When Pandoc is not installed and users attempt to attach files, users are now prompted to install Pandoc first.
- Improved the ID handling for configuration plugins. - Improved the ID handling for configuration plugins.
- Improved error handling, logging, and code quality. - Improved error handling, logging, and code quality.
- Improved error handling for Microsoft Word export. - Improved error handling for the Microsoft Word export.
- Improved file reading, e.g. for the translation, summarization, and legal assistants, by performing the Pandoc validation in the first step. This prevents unnecessary selection of files that cannot be processed. - Improved the file reading, e.g. for the translation, summarization, and legal assistants, by performing the Pandoc validation in the first step. This prevents unnecessary selection of files that cannot be processed.
- Improved the file selection for file attachments in chat and assistant file loading by filtering out audio files. Audio attachments are not yet supported. - Improved the file selection for file attachments in chat and the assistant file loading by filtering out audio files. Audio attachments are not yet supported.
- Improved the developer experience by automating localization updates in the filesystem for the selected language in the localization assistant. - Improved the developer experience by automating localization updates in the filesystem for the selected language in the localization assistant.
- Improved the file selection so that users can now select multiple files at the same time. This is useful, for example, for document analysis (in preview) or adding file attachments to the chat. - Improved the file selection so that users can now select multiple files at the same time. This is useful, for example, for document analysis (in preview) or adding file attachments to the chat.
- Fixed a bug in the local data sources info dialog (preview feature) for data directories that could cause the app to crash. The error was caused by a background thread producing data while the frontend attempted to display it. - Fixed a bug in the local data sources info dialog (preview feature) for data directories that could cause the app to crash. The error was caused by a background thread producing data while the frontend attempted to display it.