Added support for images (#609)

app/MindWork AI Studio/Chat/ChatThread.cs

@@ -238,7 +238,7 @@ public sealed record ChatThread
         {
             var (contentData, contentType) = block.Content switch
             {
-                ContentImage image => (await image.AsBase64(token), Tools.ERIClient.DataModel.ContentType.IMAGE),
+                ContentImage image => (await image.TryAsBase64(token) is (success: true, { } base64Image) ? base64Image : string.Empty, Tools.ERIClient.DataModel.ContentType.IMAGE),
                 ContentText text => (text.Text, Tools.ERIClient.DataModel.ContentType.TEXT),
                 
                 _ => (string.Empty, Tools.ERIClient.DataModel.ContentType.UNKNOWN),

app/MindWork AI Studio/Chat/ContentImage.cs

@@ -47,6 +47,8 @@ public sealed class ContentImage : IContent, IImageSource
         InitialRemoteWait = this.InitialRemoteWait,
         IsStreaming = this.IsStreaming,
         SourceType = this.SourceType,
+        Sources = [..this.Sources],
+        FileAttachments = [..this.FileAttachments],
     };
 
     #endregion

app/MindWork AI Studio/Chat/ContentText.cs

@@ -195,6 +195,14 @@ public sealed class ContentText : IContent
                         sb.AppendLine(await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue));
                         sb.AppendLine("````");
                     }
+                    
+                    var numImages = this.FileAttachments.Count(x => x is { IsImage: true, Exists: true });
+                    if (numImages > 0)
+                    {
+                        sb.AppendLine();
+                        sb.AppendLine($"Additionally, there are {numImages} image file(s) attached to this message. ");
+                        sb.AppendLine("Please consider them as part of the message content and use them to answer accordingly.");
+                    }
                 }
             }
         }

app/MindWork AI Studio/Chat/FileAttachment.cs

@@ -1,3 +1,5 @@
+using System.Text.Json.Serialization;
+
 using AIStudio.Tools.Rust;
 
 namespace AIStudio.Chat;
@@ -9,22 +11,47 @@ namespace AIStudio.Chat;
 /// <param name="FileName">The name of the file, including extension.</param>
 /// <param name="FilePath">The full path to the file, including the filename and extension.</param>
 /// <param name="FileSizeBytes">The size of the file in bytes.</param>
-public readonly record struct FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes)
+[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
+[JsonDerivedType(typeof(FileAttachment), typeDiscriminator: "file")]
+[JsonDerivedType(typeof(FileAttachmentImage), typeDiscriminator: "image")]
+public record FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes)
 {
-    /// <summary>
-    /// Gets a value indicating whether the file still exists on the file system.
-    /// </summary>
-    public bool Exists => File.Exists(this.FilePath);
-
     /// <summary>
     /// Gets a value indicating whether the file type is forbidden and should not be attached.
     /// </summary>
-    public bool IsForbidden => this.Type == FileAttachmentType.FORBIDDEN;
+    /// <remarks>
+    /// The state is determined once during construction and does not change.
+    /// </remarks>
+    public bool IsForbidden { get; } = Type == FileAttachmentType.FORBIDDEN;
 
     /// <summary>
     /// Gets a value indicating whether the file type is valid and allowed to be attached.
     /// </summary>
-    public bool IsValid => this.Type != FileAttachmentType.FORBIDDEN;
+    /// <remarks>
+    /// The state is determined once during construction and does not change.
+    /// </remarks>
+    public bool IsValid { get; } = Type != FileAttachmentType.FORBIDDEN;
+
+    /// <summary>
+    /// Gets a value indicating whether the file type is an image.
+    /// </summary>
+    /// <remarks>
+    /// The state is determined once during construction and does not change.
+    /// </remarks>
+    public bool IsImage { get; } = Type == FileAttachmentType.IMAGE;
+
+    /// <summary>
+    /// Gets the file path for loading the file from the web browser-side (Blazor).
+    /// </summary>
+    public string FilePathAsUrl { get; } = FileHandler.CreateFileUrl(FilePath);
+    
+    /// <summary>
+    /// Gets a value indicating whether the file still exists on the file system.
+    /// </summary>
+    /// <remarks>
+    /// This property checks the file system each time it is accessed.
+    /// </remarks>
+    public bool Exists => File.Exists(this.FilePath);
 
     /// <summary>
     /// Creates a FileAttachment from a file path by automatically determining the type,
@@ -38,7 +65,13 @@ public readonly record struct FileAttachment(FileAttachmentType Type, string Fil
         var fileSize = File.Exists(filePath) ? new FileInfo(filePath).Length : 0;
         var type = DetermineFileType(filePath);
 
-        return new FileAttachment(type, fileName, filePath, fileSize);
+        return type switch
+        {
+            FileAttachmentType.DOCUMENT => new FileAttachment(type, fileName, filePath, fileSize),
+            FileAttachmentType.IMAGE => new FileAttachmentImage(fileName, filePath, fileSize),
+            
+            _ => new FileAttachment(type, fileName, filePath, fileSize),
+        };
     }
 
     /// <summary>

app/MindWork AI Studio/Chat/FileAttachmentImage.cs

@@ -0,0 +1,17 @@
+namespace AIStudio.Chat;
+
+public record FileAttachmentImage(string FileName, string FilePath, long FileSizeBytes) : FileAttachment(FileAttachmentType.IMAGE, FileName, FilePath, FileSizeBytes), IImageSource
+{
+    /// <summary>
+    /// The type of the image source.
+    /// </summary>
+    /// <remarks>
+    /// Is the image source a URL, a local file path, a base64 string, etc.?
+    /// </remarks>
+    public ContentImageSource SourceType { get; init; } = ContentImageSource.LOCAL_PATH;
+
+    /// <summary>
+    /// The image source.
+    /// </summary>
+    public string Source { get; set; } = FilePath;
+}

app/MindWork AI Studio/Chat/IImageSourceExtensions.cs

@@ -6,27 +6,89 @@ public static class IImageSourceExtensions
 {
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IImageSourceExtensions).Namespace, nameof(IImageSourceExtensions));
     
+    public static string DetermineMimeType(this IImageSource image)
+    {
+        switch (image.SourceType)
+        {
+            case ContentImageSource.BASE64:
+            {
+                // Try to detect the mime type from the base64 string:
+                var base64Data = image.Source;
+                if (base64Data.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
+                {
+                    var mimeEnd = base64Data.IndexOf(';');
+                    if (mimeEnd > 5)
+                    {
+                        return base64Data[5..mimeEnd];
+                    }
+                }
+
+                // Fallback:
+                return "application/octet-stream";
+            }
+
+            case ContentImageSource.URL:
+            {
+                // Try to detect the mime type from the URL extension:
+                var uri = new Uri(image.Source);
+                var extension = Path.GetExtension(uri.AbsolutePath).ToLowerInvariant();
+                return extension switch
+                {
+                    ".png" => "image/png",
+                    ".jpg" or ".jpeg" => "image/jpeg",
+                    ".gif" => "image/gif",
+                    ".bmp" => "image/bmp",
+                    ".webp" => "image/webp",
+
+                    _ => "application/octet-stream"
+                };
+            }
+
+            case ContentImageSource.LOCAL_PATH:
+            {
+                var extension = Path.GetExtension(image.Source).ToLowerInvariant();
+                return extension switch
+                {
+                    ".png" => "image/png",
+                    ".jpg" or ".jpeg" => "image/jpeg",
+                    ".gif" => "image/gif",
+                    ".bmp" => "image/bmp",
+                    ".webp" => "image/webp",
+
+                    _ => "application/octet-stream"
+                };
+            }
+
+            default:
+                return "application/octet-stream";
+        }
+    }
+    
     /// <summary>
     /// Read the image content as a base64 string.
     /// </summary>
     /// <remarks>
     /// The images are directly converted to base64 strings. The maximum
     /// size of the image is around 10 MB. If the image is larger, the method
-    /// returns an empty string.
-    /// 
+    /// returns an empty string.<br/>
+    /// <br/>
     /// As of now, this method does no sort of image processing. LLMs usually
     /// do not work with arbitrary image sizes. In the future, we might have
-    /// to resize the images before sending them to the model.
+    /// to resize the images before sending them to the model.<br/>
+    /// <br/>
+    /// Note as well that this method returns just the base64 string without
+    /// any data URI prefix (like "data:image/png;base64,"). The caller has
+    /// to take care of that if needed.
     /// </remarks>
     /// <param name="image">The image source.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>The image content as a base64 string; might be empty.</returns>
-    public static async Task<string> AsBase64(this IImageSource image, CancellationToken token = default)
+    public static async Task<(bool success, string base64Content)> TryAsBase64(this IImageSource image, CancellationToken token = default)
     {
         switch (image.SourceType)
         {
             case ContentImageSource.BASE64:
-                return image.Source;
+                return (success: true, image.Source);
             
             case ContentImageSource.URL:
             {
@@ -39,14 +101,15 @@ public static class IImageSourceExtensions
                     if(lengthBytes > 10_000_000)
                     {
                         await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The image at the URL is too large (>10 MB). Skipping the image.")));
-                        return string.Empty;
+                        return (success: false, string.Empty);
                     }
 
                     var bytes = await response.Content.ReadAsByteArrayAsync(token);
-                    return Convert.ToBase64String(bytes);
+                    return (success: true, Convert.ToBase64String(bytes));
                 }
 
-                return string.Empty;
+                await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("Failed to download the image from the URL. Skipping the image.")));
+                return (success: false, string.Empty);
             }
 
             case ContentImageSource.LOCAL_PATH:
@@ -57,17 +120,18 @@ public static class IImageSourceExtensions
                     if(length > 10_000_000)
                     {
                         await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file is too large (>10 MB). Skipping the image.")));
-                        return string.Empty;
+                        return (success: false, string.Empty);
                     }
 
                     var bytes = await File.ReadAllBytesAsync(image.Source, token);
-                    return Convert.ToBase64String(bytes);
+                    return (success: true, Convert.ToBase64String(bytes));
                 }
 
-                return string.Empty;
+                await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file does not exist. Skipping the image.")));
+                return (success: false, string.Empty);
             
             default:
-                return string.Empty;
+                return (success: false, string.Empty);
         }
     }
 }

app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs

@@ -1,4 +1,6 @@
 using AIStudio.Provider;
+using AIStudio.Provider.OpenAI;
+using AIStudio.Settings;
 
 namespace AIStudio.Chat;
 
@@ -8,19 +10,171 @@ public static class ListContentBlockExtensions
     /// Processes a list of content blocks by transforming them into a collection of message results asynchronously.
     /// </summary>
     /// <param name="blocks">The list of content blocks to process.</param>
-    /// <param name="transformer">A function that transforms each content block into a message result asynchronously.</param>
+    /// <param name="roleTransformer">A function that transforms each content block into a message result asynchronously.</param>
+    /// <param name="selectedProvider">The selected LLM provider.</param>
+    /// <param name="selectedModel">The selected model.</param>
+    /// <param name="textSubContentFactory">A factory function to create text sub-content.</param>
+    /// <param name="imageSubContentFactory">A factory function to create image sub-content.</param>
     /// <returns>An asynchronous task that resolves to a list of transformed results.</returns>
-    public static async Task<IList<IMessageBase>> BuildMessages(this List<ContentBlock> blocks, Func<ContentBlock, Task<IMessageBase>> transformer)
+    public static async Task<IList<IMessageBase>> BuildMessagesAsync(
+        this List<ContentBlock> blocks,
+        LLMProviders selectedProvider,
+        Model selectedModel,
+        Func<ChatRole, string> roleTransformer,
+        Func<string, ISubContent> textSubContentFactory,
+        Func<FileAttachmentImage, Task<ISubContent>> imageSubContentFactory)
     {
-        var messages = blocks
-            .Where(n => n.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace((n.Content as ContentText)?.Text))
-            .Select(transformer)
-            .ToList();
+        var capabilities = selectedProvider.GetModelCapabilities(selectedModel);
+        var canProcessImages = capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT) ||
+                               capabilities.Contains(Capability.SINGLE_IMAGE_INPUT);
         
-        // Await all messages:
-        await Task.WhenAll(messages);
+        var messageTaskList = new List<Task<IMessageBase>>(blocks.Count);
+        foreach (var block in blocks)
+        {
+            switch (block.Content)
+            {
+                // The prompt may or may not contain image(s), but the provider/model cannot process images.
+                // Thus, we treat it as a regular text message.
+                case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && !canProcessImages:
+                    messageTaskList.Add(CreateTextMessageAsync(block, text));
+                    break;
+                
+                // The regular case for text content without images:
+                case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && !text.FileAttachments.ContainsImages():
+                    messageTaskList.Add(CreateTextMessageAsync(block, text));
+                    break;
+                
+                // Text prompt with images as attachments, and the provider/model can process images:
+                case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && text.FileAttachments.ContainsImages():
+                    messageTaskList.Add(CreateMultimodalMessageAsync(block, text, textSubContentFactory, imageSubContentFactory));
+                    break;
+            }
+        }
 
+        // Await all messages:
+        await Task.WhenAll(messageTaskList);
+        
         // Select all results:
-        return messages.Select(n => n.Result).ToList();
+        return messageTaskList.Select(n => n.Result).ToList();
+        
+        // Local function to create a text message asynchronously.
+        Task<IMessageBase> CreateTextMessageAsync(ContentBlock block, ContentText text)
+        {
+            return Task.Run(async () => new TextMessage
+            {
+                Role = roleTransformer(block.Role),
+                Content = await text.PrepareTextContentForAI(),
+            } as IMessageBase);
+        }
+        
+        // Local function to create a multimodal message asynchronously.
+        Task<IMessageBase> CreateMultimodalMessageAsync(
+            ContentBlock block,
+            ContentText text,
+            Func<string, ISubContent> innerTextSubContentFactory,
+            Func<FileAttachmentImage, Task<ISubContent>> innerImageSubContentFactory)
+        {
+            return Task.Run(async () =>
+            {
+                var imagesTasks = text.FileAttachments
+                    .Where(x => x is { IsImage: true, Exists: true })
+                    .Cast<FileAttachmentImage>()
+                    .Select(innerImageSubContentFactory)
+                    .ToList();
+                
+                Task.WaitAll(imagesTasks);
+                var images = imagesTasks.Select(t => t.Result).ToList();
+                
+                return new MultimodalMessage
+                {
+                    Role = roleTransformer(block.Role),
+                    Content =
+                    [
+                        innerTextSubContentFactory(await text.PrepareTextContentForAI()),
+                        ..images,
+                    ]
+                } as IMessageBase;
+            });
+        }
     }
+
+    /// <summary>
+    /// Processes a list of content blocks using direct image URL format to create message results asynchronously.
+    /// </summary>
+    /// <param name="blocks">The list of content blocks to process.</param>
+    /// <param name="selectedProvider">The selected LLM provider.</param>
+    /// <param name="selectedModel">The selected model.</param>
+    /// <returns>An asynchronous task that resolves to a list of transformed message results.</returns>
+    /// <remarks>
+    /// Uses direct image URL format where the image data is placed directly in the image_url field:
+    /// <code>
+    /// { "type": "image_url", "image_url": "data:image/jpeg;base64,..." }
+    /// </code>
+    /// This format is used by OpenAI, Mistral, and Ollama.
+    /// </remarks>
+    public static async Task<IList<IMessageBase>> BuildMessagesUsingDirectImageUrlAsync(
+        this List<ContentBlock> blocks,
+        LLMProviders selectedProvider,
+        Model selectedModel) => await blocks.BuildMessagesAsync(
+            selectedProvider,
+            selectedModel,
+            StandardRoleTransformer,
+            StandardTextSubContentFactory,
+            DirectImageSubContentFactory);
+
+    /// <summary>
+    /// Processes a list of content blocks using nested image URL format to create message results asynchronously.
+    /// </summary>
+    /// <param name="blocks">The list of content blocks to process.</param>
+    /// <param name="selectedProvider">The selected LLM provider.</param>
+    /// <param name="selectedModel">The selected model.</param>
+    /// <returns>An asynchronous task that resolves to a list of transformed message results.</returns>
+    /// <remarks>
+    /// Uses nested image URL format where the image data is wrapped in an object:
+    /// <code>
+    /// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
+    /// </code>
+    /// This format is used by LM Studio, VLLM, llama.cpp, and other OpenAI-compatible providers.
+    /// </remarks>
+    public static async Task<IList<IMessageBase>> BuildMessagesUsingNestedImageUrlAsync(
+        this List<ContentBlock> blocks,
+        LLMProviders selectedProvider,
+        Model selectedModel) => await blocks.BuildMessagesAsync(
+            selectedProvider,
+            selectedModel,
+            StandardRoleTransformer,
+            StandardTextSubContentFactory,
+            NestedImageSubContentFactory);
+
+    private static ISubContent StandardTextSubContentFactory(string text) => new SubContentText
+    {
+        Text = text,
+    };
+
+    private static async Task<ISubContent> DirectImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrl
+    {
+        ImageUrl = await attachment.TryAsBase64() is (true, var base64Content)
+            ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
+            : string.Empty,
+    };
+
+    private static async Task<ISubContent> NestedImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrlNested
+    {
+        ImageUrl = new SubContentImageUrlData
+        {
+            Url = await attachment.TryAsBase64() is (true, var base64Content)
+                ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
+                : string.Empty,
+        },
+    };
+
+    private static string StandardRoleTransformer(ChatRole role) => role switch
+    {
+        ChatRole.USER => "user",
+        ChatRole.AI => "assistant",
+        ChatRole.AGENT => "assistant",
+        ChatRole.SYSTEM => "system",
+
+        _ => "user",
+    };
 }

app/MindWork AI Studio/Chat/ListFileAttachmentExtensions.cs

@@ -0,0 +1,6 @@
+namespace AIStudio.Chat;
+
+public static class ListFileAttachmentExtensions
+{
+    public static bool ContainsImages(this List<FileAttachment> attachments) => attachments.Any(attachment => attachment.IsImage);
+}