Scaled down and converted images before sending them to AI models (#1101)

app/MindWork AI Studio/Chat/ContentImage.cs

@@ -1,7 +1,6 @@
 using System.Text.Json.Serialization;
 
 using AIStudio.Provider;
-using AIStudio.Tools.Validation;
 
 namespace AIStudio.Chat;
 
@@ -53,23 +52,6 @@ public sealed class ContentImage : IContent, IImageSource
 
     #endregion
 
-    /// <summary>
-    /// Creates a ContentImage from a local file path.
-    /// </summary>
-    /// <param name="filePath">The path to the image file.</param>
-    /// <returns>A new ContentImage instance if the file is valid, null otherwise.</returns>
-    public static async Task<ContentImage?> CreateFromFileAsync(string filePath)
-    {
-        if (!await FileExtensionValidation.IsImageExtensionValidWithNotifyAsync(filePath))
-            return null;
-
-        return new ContentImage
-        {
-            SourceType = ContentImageSource.LOCAL_PATH,
-            Source = filePath,
-        };
-    }
-
     /// <summary>
     /// The type of the image source.
     /// </summary>

app/MindWork AI Studio/Chat/IImageSourceExtensions.cs

@@ -1,5 +1,5 @@
 using AIStudio.Provider;
-using AIStudio.Tools.MIME;
+using AIStudio.Provider.SelfHosted;
 using AIStudio.Tools.PluginSystem;
 
 namespace AIStudio.Chat;
@@ -9,7 +9,8 @@ public static class IImageSourceExtensions
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IImageSourceExtensions).Namespace, nameof(IImageSourceExtensions));
 
     /// <summary>
-    /// The largest image we send to a model. A larger image is skipped by TryAsBase64.
+    /// The largest image, in bytes, we send anywhere: no image prepared for a model is larger, and
+    /// TryAsBase64 skips every larger file.
     /// </summary>
     public const long MAX_IMAGE_SIZE_BYTES = 10_000_000;
 
@@ -17,79 +18,30 @@ public static class IImageSourceExtensions
     /// The largest image file, in bytes, which we send through the given provider.
     /// </summary>
     /// <remarks>
-    /// The limit of the provider where it states one, but never more than MAX_IMAGE_SIZE_BYTES:
-    /// TryAsBase64 skips every larger image, whatever the provider would accept.
+    /// The smallest of the limits we know: the one of a self-hosted engine, the one of the provider,
+    /// and MAX_IMAGE_SIZE_BYTES, our own. A provider or engine which states no limit gets ours.
     /// </remarks>
-    /// <param name="llmProvider">The provider the image is sent through.</param>
+    /// <param name="provider">The configured provider the image is sent through.</param>
     /// <returns>The size limit of one image file.</returns>
-    public static long MaxImageSizeBytesFor(LLMProviders llmProvider) => Math.Min(llmProvider.GetMaxImageSizeBytes() ?? MAX_IMAGE_SIZE_BYTES, MAX_IMAGE_SIZE_BYTES);
-
-    public static MIMEType DetermineMimeType(this IImageSource image)
+    public static long MaxImageSizeBytesFor(AIStudio.Settings.Provider provider)
     {
-        switch (image.SourceType)
-        {
-            case ContentImageSource.BASE64:
-            {
-                // Try to detect the mime type from the base64 string:
-                var base64Data = image.Source;
-                if (base64Data.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
-                {
-                    var mimeEnd = base64Data.IndexOf(';');
-                    if (mimeEnd > 5)
-                        return Builder.FromTextRepresentation(base64Data[5..mimeEnd]);
-                }
-
-                // Fallback:
-                return Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build();
-            }
-
-            case ContentImageSource.URL:
-            {
-                // Try to detect the mime type from the URL extension:
-                var uri = new Uri(image.Source);
-                var extension = Path.GetExtension(uri.AbsolutePath).ToLowerInvariant();
-                return DeriveMIMETypeFromExtension(extension);
-            }
-
-            case ContentImageSource.LOCAL_PATH:
-            {
-                var extension = Path.GetExtension(image.Source).ToLowerInvariant();
-                return DeriveMIMETypeFromExtension(extension);
-            }
-
-            default:
-                return Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build();
-        }
-    }
-
-    private static MIMEType DeriveMIMETypeFromExtension(string extension)
-    {
-        var imageBuilder = Builder.Create().UseImage();
-        return extension switch
-        {
-            ".png" => imageBuilder.UseSubtype(ImageSubtype.PNG).Build(),
-            ".jpg" or ".jpeg" => imageBuilder.UseSubtype(ImageSubtype.JPEG).Build(),
-            ".gif" => imageBuilder.UseSubtype(ImageSubtype.GIF).Build(),
-            ".webp" => imageBuilder.UseSubtype(ImageSubtype.WEBP).Build(),
-            ".tiff" or ".tif" => imageBuilder.UseSubtype(ImageSubtype.TIFF).Build(),
-            ".heic" or ".heif" => imageBuilder.UseSubtype(ImageSubtype.HEIC).Build(),
-
-            _ => Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build()
-        };
+        var engineLimit = provider.UsedLLMProvider is LLMProviders.SELF_HOSTED ? provider.Host.GetMaxImageSizeBytes() : null;
+        var providerLimit = provider.UsedLLMProvider.GetMaxImageSizeBytes();
+        return Math.Min(Math.Min(engineLimit ?? MAX_IMAGE_SIZE_BYTES, providerLimit ?? MAX_IMAGE_SIZE_BYTES), MAX_IMAGE_SIZE_BYTES);
     }
 
     /// <summary>
     /// Read the image content as a base64 string.
     /// </summary>
     /// <remarks>
+    /// This is for the images inside the content of a chat, which go to external data sources and
+    /// agents as they are. Attached images, which go to a model, take ModelImageService instead: it
+    /// scales them down and encodes them anew within the limit of the provider.<br/>
+    /// <br/>
     /// The images are directly converted to base64 strings. The maximum
     /// size of the image is around 10 MB. If the image is larger, the method
     /// returns an empty string.<br/>
     /// <br/>
-    /// As of now, this method does no sort of image processing. LLMs usually
-    /// do not work with arbitrary image sizes. In the future, we might have
-    /// to resize the images before sending them to the model.<br/>
-    /// <br/>
     /// Note as well that this method returns just the base64 string without
     /// any data URI prefix (like "data:image/png;base64,"). The caller has
     /// to take care of that if needed.

app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs

@@ -1,7 +1,9 @@
 using AIStudio.Provider;
 using AIStudio.Provider.OpenAI;
 using AIStudio.Settings;
+using AIStudio.Tools.Rust;
 using AIStudio.Tools.Security;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Chat;
 
@@ -14,14 +16,16 @@ public static class ListContentBlockExtensions
     /// <param name="roleTransformer">A function that transforms each content block into a message result asynchronously.</param>
     /// <param name="provider">The configured provider, whose model is being written to.</param>
     /// <param name="textSubContentFactory">A factory function to create text sub-content.</param>
-    /// <param name="imageSubContentFactory">A factory function to create image sub-content.</param>
+    /// <param name="imageSubContentFactory">A factory function to create image sub-content from an image prepared for the model.</param>
+    /// <param name="token">The cancellation token.</param>
     /// <returns>An asynchronous task that resolves to a list of transformed results.</returns>
     public static async Task<IList<IMessageBase>> BuildMessagesAsync(
         this List<ContentBlock> blocks,
         AIStudio.Settings.Provider provider,
         Func<ChatRole, string> roleTransformer,
         Func<string, ISubContent> textSubContentFactory,
-        Func<FileAttachmentImage, Task<ISubContent>> imageSubContentFactory)
+        Func<PreparedModelImage, ISubContent> imageSubContentFactory,
+        CancellationToken token = default)
     {
         //
         // Asked through the configured provider, so that what a person set in their expert settings
@@ -59,7 +63,7 @@ public static class ListContentBlockExtensions
                 
                 // Text prompt with images as attachments, and the provider/model can process images:
                 case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && text.FileAttachments.ContainsImages():
-                    messageTaskList.Add(CreateMultimodalMessageAsync(block, text, textSubContentFactory, imageSubContentFactory));
+                    messageTaskList.Add(CreateMultimodalMessageAsync(block, text));
                     break;
             }
         }
@@ -81,34 +85,41 @@ public static class ListContentBlockExtensions
         }
         
         // Local function to create a multimodal message asynchronously.
-        Task<IMessageBase> CreateMultimodalMessageAsync(
-            ContentBlock block,
-            ContentText text,
-            Func<string, ISubContent> innerTextSubContentFactory,
-            Func<FileAttachmentImage, Task<ISubContent>> innerImageSubContentFactory)
+        Task<IMessageBase> CreateMultimodalMessageAsync(ContentBlock block, ContentText text)
         {
             return Task.Run(async () =>
             {
-                var imagesTasks = text.FileAttachments
+                var images = await Task.WhenAll(text.FileAttachments
                     .Where(x => x is { IsImage: true, Exists: true })
                     .Cast<FileAttachmentImage>()
-                    .Select(innerImageSubContentFactory)
-                    .ToList();
-                
-                Task.WaitAll(imagesTasks);
-                var images = imagesTasks.Select(t => t.Result).ToList();
+                    .Select(CreateImageSubContentAsync));
                 
                 return new MultimodalMessage
                 {
                     Role = roleTransformer(block.Role),
                     Content =
                     [
-                        innerTextSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
-                        ..images,
+                        textSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
+                        ..images.OfType<ISubContent>(),
                     ]
                 } as IMessageBase;
             });
         }
+
+        //
+        // Local function to prepare one image for the model. An image which cannot be prepared
+        // stays out of the message, and the user learns why. It used to go along as an image
+        // without data, which the providers refuse together with the whole request.
+        //
+        async Task<ISubContent?> CreateImageSubContentAsync(FileAttachmentImage image)
+        {
+            var prepared = await Program.SERVICE_PROVIDER.GetRequiredService<ModelImageService>().PrepareAsync(image, provider, token);
+            if (prepared.IsPrepared)
+                return imageSubContentFactory(prepared);
+
+            await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, prepared.ErrorCode.ToUserMessage(image.FileName)));
+            return null;
+        }
     }
 
     /// <summary>
@@ -158,20 +169,16 @@ public static class ListContentBlockExtensions
         Text = text,
     };
 
-    private static async Task<ISubContent> DirectImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrl
+    private static ISubContent DirectImageSubContentFactory(PreparedModelImage image) => new SubContentImageUrl
     {
-        ImageUrl = await attachment.TryAsBase64() is (true, var base64Content)
-            ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-            : string.Empty,
+        ImageUrl = image.ToDataUrl(),
     };
 
-    private static async Task<ISubContent> NestedImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrlNested
+    private static ISubContent NestedImageSubContentFactory(PreparedModelImage image) => new SubContentImageUrlNested
     {
         ImageUrl = new SubContentImageUrlData
         {
-            Url = await attachment.TryAsBase64() is (true, var base64Content)
-                ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-                : string.Empty,
+            Url = image.ToDataUrl(),
         },
     };