Scaled down and converted images before sending them to AI models (#1101)

app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs

@@ -1,7 +1,9 @@
 using AIStudio.Provider;
 using AIStudio.Provider.OpenAI;
 using AIStudio.Settings;
+using AIStudio.Tools.Rust;
 using AIStudio.Tools.Security;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Chat;
 
@@ -14,14 +16,16 @@ public static class ListContentBlockExtensions
     /// <param name="roleTransformer">A function that transforms each content block into a message result asynchronously.</param>
     /// <param name="provider">The configured provider, whose model is being written to.</param>
     /// <param name="textSubContentFactory">A factory function to create text sub-content.</param>
-    /// <param name="imageSubContentFactory">A factory function to create image sub-content.</param>
+    /// <param name="imageSubContentFactory">A factory function to create image sub-content from an image prepared for the model.</param>
+    /// <param name="token">The cancellation token.</param>
     /// <returns>An asynchronous task that resolves to a list of transformed results.</returns>
     public static async Task<IList<IMessageBase>> BuildMessagesAsync(
         this List<ContentBlock> blocks,
         AIStudio.Settings.Provider provider,
         Func<ChatRole, string> roleTransformer,
         Func<string, ISubContent> textSubContentFactory,
-        Func<FileAttachmentImage, Task<ISubContent>> imageSubContentFactory)
+        Func<PreparedModelImage, ISubContent> imageSubContentFactory,
+        CancellationToken token = default)
     {
         //
         // Asked through the configured provider, so that what a person set in their expert settings
@@ -59,7 +63,7 @@ public static class ListContentBlockExtensions
                 
                 // Text prompt with images as attachments, and the provider/model can process images:
                 case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && text.FileAttachments.ContainsImages():
-                    messageTaskList.Add(CreateMultimodalMessageAsync(block, text, textSubContentFactory, imageSubContentFactory));
+                    messageTaskList.Add(CreateMultimodalMessageAsync(block, text));
                     break;
             }
         }
@@ -81,34 +85,41 @@ public static class ListContentBlockExtensions
         }
         
         // Local function to create a multimodal message asynchronously.
-        Task<IMessageBase> CreateMultimodalMessageAsync(
-            ContentBlock block,
-            ContentText text,
-            Func<string, ISubContent> innerTextSubContentFactory,
-            Func<FileAttachmentImage, Task<ISubContent>> innerImageSubContentFactory)
+        Task<IMessageBase> CreateMultimodalMessageAsync(ContentBlock block, ContentText text)
         {
             return Task.Run(async () =>
             {
-                var imagesTasks = text.FileAttachments
+                var images = await Task.WhenAll(text.FileAttachments
                     .Where(x => x is { IsImage: true, Exists: true })
                     .Cast<FileAttachmentImage>()
-                    .Select(innerImageSubContentFactory)
-                    .ToList();
-                
-                Task.WaitAll(imagesTasks);
-                var images = imagesTasks.Select(t => t.Result).ToList();
+                    .Select(CreateImageSubContentAsync));
                 
                 return new MultimodalMessage
                 {
                     Role = roleTransformer(block.Role),
                     Content =
                     [
-                        innerTextSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
-                        ..images,
+                        textSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
+                        ..images.OfType<ISubContent>(),
                     ]
                 } as IMessageBase;
             });
         }
+
+        //
+        // Local function to prepare one image for the model. An image which cannot be prepared
+        // stays out of the message, and the user learns why. It used to go along as an image
+        // without data, which the providers refuse together with the whole request.
+        //
+        async Task<ISubContent?> CreateImageSubContentAsync(FileAttachmentImage image)
+        {
+            var prepared = await Program.SERVICE_PROVIDER.GetRequiredService<ModelImageService>().PrepareAsync(image, provider, token);
+            if (prepared.IsPrepared)
+                return imageSubContentFactory(prepared);
+
+            await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, prepared.ErrorCode.ToUserMessage(image.FileName)));
+            return null;
+        }
     }
 
     /// <summary>
@@ -158,20 +169,16 @@ public static class ListContentBlockExtensions
         Text = text,
     };
 
-    private static async Task<ISubContent> DirectImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrl
+    private static ISubContent DirectImageSubContentFactory(PreparedModelImage image) => new SubContentImageUrl
     {
-        ImageUrl = await attachment.TryAsBase64() is (true, var base64Content)
-            ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-            : string.Empty,
+        ImageUrl = image.ToDataUrl(),
     };
 
-    private static async Task<ISubContent> NestedImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrlNested
+    private static ISubContent NestedImageSubContentFactory(PreparedModelImage image) => new SubContentImageUrlNested
     {
         ImageUrl = new SubContentImageUrlData
         {
-            Url = await attachment.TryAsBase64() is (true, var base64Content)
-                ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-                : string.Empty,
+            Url = image.ToDataUrl(),
         },
     };