Scaled down and converted images before sending them to AI models (#1101)

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor

@@ -26,7 +26,7 @@
 </MudJustifiedText>
 
 <MudJustifiedText Typo="Typo.body2" Class="mb-3">
-    @T("Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.")
+    @T("Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.")
 </MudJustifiedText>
 
 <MudStack Class="mb-3">

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs

@@ -20,7 +20,7 @@ public partial class AssistantBatchProcessing
     private async Task<BatchInput?> LoadInputContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
     {
         if (IsImage(fileResult.FilePath))
-            return this.LoadImage(fileResult) is { } image ? new BatchInput(string.Empty, image) : null;
+            return await this.LoadImageAsync(fileResult, token) is { } image ? new BatchInput(string.Empty, image) : null;
 
         var text = IsTranscribableMedia(fileResult.FilePath)
             ? await this.LoadMediaTranscriptAsync(fileResult, token)
@@ -30,13 +30,18 @@ public partial class AssistantBatchProcessing
     }
 
     /// <summary>
-    /// Checks an image before it goes to the model.
+    /// Prepares an image for the model before the request is built.
     /// </summary>
     /// <remarks>
     /// The request itself would only skip an image it cannot send, with a message, and ask the model
     /// anyway. The model would then answer about nothing, and the file would count as done, so that
-    /// continuing the run would skip it as well. A failed image is processed again when the run is
-    /// continued, e.g., with a model that can read images.
+    /// continuing the run would skip it as well. Preparing the image here lets it fail as a file
+    /// instead, with the reason, e.g., an image format AI Studio cannot decode. The request then finds
+    /// the prepared image in the cache of ModelImageService, so it is not prepared twice. A failed
+    /// image is processed again when the run is continued, e.g., with a model that can read images.
+    ///
+    /// How large an image may be needs no check of its own either: the preparation scales it down
+    /// until it fits the limit of the provider, and fails only when even that is not enough.
     ///
     /// How many images a model accepts needs no check of its own: every file gets a thread of its own,
     /// see CallAIAsync, so each request carries exactly one image, in every round with the model. A
@@ -49,7 +54,7 @@ public partial class AssistantBatchProcessing
     /// longer send data out, and without free address choice, its default, the Read Web Page tool
     /// refuses an address which appears only in the image.
     /// </remarks>
-    private FileAttachmentImage? LoadImage(BatchProcessingFileResult fileResult)
+    private async Task<FileAttachmentImage?> LoadImageAsync(BatchProcessingFileResult fileResult, CancellationToken token)
     {
         if (!this.ProviderSettings.SupportsImageInput())
         {
@@ -57,16 +62,17 @@ public partial class AssistantBatchProcessing
             return null;
         }
 
-        if (!FileTypes.IsAllowedPath(fileResult.FilePath, FileTypes.MODEL_IMAGE))
-        {
-            this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."));
-            return null;
-        }
-
-        long fileSize;
+        PreparedModelImage prepared;
+        FileAttachmentImage image;
         try
         {
-            fileSize = new FileInfo(fileResult.FilePath).Length;
+            image = new FileAttachmentImage(fileResult.FileName, fileResult.FilePath, new FileInfo(fileResult.FilePath).Length);
+            prepared = await this.ModelImageService.PrepareAsync(image, this.ProviderSettings, token);
+        }
+        catch (OperationCanceledException)
+        {
+            this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled."));
+            return null;
         }
         catch (Exception e)
         {
@@ -74,14 +80,11 @@ public partial class AssistantBatchProcessing
             return null;
         }
 
-        var maxImageSizeBytes = IImageSourceExtensions.MaxImageSizeBytesFor(this.ProviderSettings.UsedLLMProvider);
-        if (fileSize > maxImageSizeBytes)
-        {
-            this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("The image is larger than {0} MB, which is the most the selected provider accepts for one image."), (maxImageSizeBytes / 1_000_000d).ToString("0.#", Tools.PluginSystem.I18N.I.Culture)));
-            return null;
-        }
+        if (prepared.IsPrepared)
+            return image;
 
-        return new FileAttachmentImage(fileResult.FileName, fileResult.FilePath, fileSize);
+        this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, prepared.ErrorCode.ToUserMessage(fileResult.FileName));
+        return null;
     }
 
     private async Task<string?> LoadDocumentContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs

@@ -180,7 +180,7 @@ public partial class AssistantBatchProcessing
 
         //
         // A model that cannot read images would get the text of the request alone, without the
-        // image, and answer about nothing. Such images fail one by one, see LoadImage, while the
+        // image, and answer about nothing. Such images fail one by one, see LoadImageAsync, while the
         // documents next to them are processed. A run of nothing but images does not start at all:
         //
         if (files.All(IsImage) && !this.ProviderSettings.SupportsImageInput())

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs

@@ -15,6 +15,9 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
     [Inject]
     private PandocAvailabilityService PandocAvailability { get; init; } = null!;
 
+    [Inject]
+    private ModelImageService ModelImageService { get; init; } = null!;
+
     private const string DEFAULT_OUTPUT_DIRECTORY_NAME = "ai-results";
     private const string DEFAULT_RESULTS_FILENAME = "batch-results.csv";
     private const string CSV_EXTENSION = ".csv";

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -532,8 +532,8 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Was not able to write the result file: {0}
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Was not able to write the result file: {0}"
 
--- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
+-- Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3011193639"] = "Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
 
 -- The batch run finished, but one file could not be processed. See the progress table and log for details.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "The batch run finished, but one file could not be processed. See the progress table and log for details."
@@ -661,9 +661,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Start batch processing
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Start batch processing"
 
--- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."
-
 -- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again."
 
@@ -688,9 +685,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Include hidden files and folders?
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Include hidden files and folders?"
 
--- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "The image is larger than {0} MB, which is the most the selected provider accepts for one image."
-
 -- Restore default
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Restore default"
 
@@ -8590,9 +8584,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
 -- Default minimum pause between files
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Default minimum pause between files"
 
--- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format."
-
 -- Instructions
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Instructions"
 
@@ -8635,6 +8626,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T22
 -- Hidden files and folders are included
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2288334048"] = "Hidden files and folders are included"
 
+-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T232588799"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format."
+
 -- The lower end of the random pause interval. AI Studio never allows less than 6 seconds.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T237774509"] = "The lower end of the random pause interval. AI Studio never allows less than 6 seconds."
 
@@ -13441,6 +13435,30 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Image for AI
 -- Plugin archive
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin archive"
 
+-- AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1345506748"] = "AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent."
+
+-- The image '{0}' could not be made small enough for the selected provider, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1463083567"] = "The image '{0}' could not be made small enough for the selected provider, so it was not sent."
+
+-- The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2022724454"] = "The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable."
+
+-- The image '{0}' does not exist anymore and was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2330229240"] = "The image '{0}' does not exist anymore and was not sent."
+
+-- The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T3457568536"] = "The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension."
+
+-- The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T4029166607"] = "The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely."
+
+-- The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T567941882"] = "The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first."
+
+-- The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T568830141"] = "The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG."
+
 -- Attempt to override instructions
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONFINDINGCATEGORYEXTENSIONS::T161976090"] = "Attempt to override instructions"
 
@@ -14671,11 +14689,8 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T90753429"]
 -- Please enter the secret necessary for authentication.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T968385876"] = "Please enter the secret necessary for authentication."
 
--- Unsupported image format
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1398282880"] = "Unsupported image format"
-
--- File has no extension
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1555980031"] = "File has no extension"
+-- AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1578559061"] = "AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG."
 
 -- Audio files are not supported yet
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T2919730669"] = "Audio files are not supported yet"

app/MindWork AI Studio/Chat/ContentImage.cs

@@ -1,7 +1,6 @@
 using System.Text.Json.Serialization;
 
 using AIStudio.Provider;
-using AIStudio.Tools.Validation;
 
 namespace AIStudio.Chat;
 
@@ -53,23 +52,6 @@ public sealed class ContentImage : IContent, IImageSource
 
     #endregion
 
-    /// <summary>
-    /// Creates a ContentImage from a local file path.
-    /// </summary>
-    /// <param name="filePath">The path to the image file.</param>
-    /// <returns>A new ContentImage instance if the file is valid, null otherwise.</returns>
-    public static async Task<ContentImage?> CreateFromFileAsync(string filePath)
-    {
-        if (!await FileExtensionValidation.IsImageExtensionValidWithNotifyAsync(filePath))
-            return null;
-
-        return new ContentImage
-        {
-            SourceType = ContentImageSource.LOCAL_PATH,
-            Source = filePath,
-        };
-    }
-
     /// <summary>
     /// The type of the image source.
     /// </summary>

app/MindWork AI Studio/Chat/IImageSourceExtensions.cs

@@ -1,5 +1,5 @@
 using AIStudio.Provider;
-using AIStudio.Tools.MIME;
+using AIStudio.Provider.SelfHosted;
 using AIStudio.Tools.PluginSystem;
 
 namespace AIStudio.Chat;
@@ -9,7 +9,8 @@ public static class IImageSourceExtensions
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IImageSourceExtensions).Namespace, nameof(IImageSourceExtensions));
 
     /// <summary>
-    /// The largest image we send to a model. A larger image is skipped by TryAsBase64.
+    /// The largest image, in bytes, we send anywhere: no image prepared for a model is larger, and
+    /// TryAsBase64 skips every larger file.
     /// </summary>
     public const long MAX_IMAGE_SIZE_BYTES = 10_000_000;
 
@@ -17,79 +18,30 @@ public static class IImageSourceExtensions
     /// The largest image file, in bytes, which we send through the given provider.
     /// </summary>
     /// <remarks>
-    /// The limit of the provider where it states one, but never more than MAX_IMAGE_SIZE_BYTES:
-    /// TryAsBase64 skips every larger image, whatever the provider would accept.
+    /// The smallest of the limits we know: the one of a self-hosted engine, the one of the provider,
+    /// and MAX_IMAGE_SIZE_BYTES, our own. A provider or engine which states no limit gets ours.
     /// </remarks>
-    /// <param name="llmProvider">The provider the image is sent through.</param>
+    /// <param name="provider">The configured provider the image is sent through.</param>
     /// <returns>The size limit of one image file.</returns>
-    public static long MaxImageSizeBytesFor(LLMProviders llmProvider) => Math.Min(llmProvider.GetMaxImageSizeBytes() ?? MAX_IMAGE_SIZE_BYTES, MAX_IMAGE_SIZE_BYTES);
-
-    public static MIMEType DetermineMimeType(this IImageSource image)
+    public static long MaxImageSizeBytesFor(AIStudio.Settings.Provider provider)
     {
-        switch (image.SourceType)
-        {
-            case ContentImageSource.BASE64:
-            {
-                // Try to detect the mime type from the base64 string:
-                var base64Data = image.Source;
-                if (base64Data.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
-                {
-                    var mimeEnd = base64Data.IndexOf(';');
-                    if (mimeEnd > 5)
-                        return Builder.FromTextRepresentation(base64Data[5..mimeEnd]);
-                }
-
-                // Fallback:
-                return Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build();
-            }
-
-            case ContentImageSource.URL:
-            {
-                // Try to detect the mime type from the URL extension:
-                var uri = new Uri(image.Source);
-                var extension = Path.GetExtension(uri.AbsolutePath).ToLowerInvariant();
-                return DeriveMIMETypeFromExtension(extension);
-            }
-
-            case ContentImageSource.LOCAL_PATH:
-            {
-                var extension = Path.GetExtension(image.Source).ToLowerInvariant();
-                return DeriveMIMETypeFromExtension(extension);
-            }
-
-            default:
-                return Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build();
-        }
-    }
-
-    private static MIMEType DeriveMIMETypeFromExtension(string extension)
-    {
-        var imageBuilder = Builder.Create().UseImage();
-        return extension switch
-        {
-            ".png" => imageBuilder.UseSubtype(ImageSubtype.PNG).Build(),
-            ".jpg" or ".jpeg" => imageBuilder.UseSubtype(ImageSubtype.JPEG).Build(),
-            ".gif" => imageBuilder.UseSubtype(ImageSubtype.GIF).Build(),
-            ".webp" => imageBuilder.UseSubtype(ImageSubtype.WEBP).Build(),
-            ".tiff" or ".tif" => imageBuilder.UseSubtype(ImageSubtype.TIFF).Build(),
-            ".heic" or ".heif" => imageBuilder.UseSubtype(ImageSubtype.HEIC).Build(),
-
-            _ => Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build()
-        };
+        var engineLimit = provider.UsedLLMProvider is LLMProviders.SELF_HOSTED ? provider.Host.GetMaxImageSizeBytes() : null;
+        var providerLimit = provider.UsedLLMProvider.GetMaxImageSizeBytes();
+        return Math.Min(Math.Min(engineLimit ?? MAX_IMAGE_SIZE_BYTES, providerLimit ?? MAX_IMAGE_SIZE_BYTES), MAX_IMAGE_SIZE_BYTES);
     }
 
     /// <summary>
     /// Read the image content as a base64 string.
     /// </summary>
     /// <remarks>
+    /// This is for the images inside the content of a chat, which go to external data sources and
+    /// agents as they are. Attached images, which go to a model, take ModelImageService instead: it
+    /// scales them down and encodes them anew within the limit of the provider.<br/>
+    /// <br/>
     /// The images are directly converted to base64 strings. The maximum
     /// size of the image is around 10 MB. If the image is larger, the method
     /// returns an empty string.<br/>
     /// <br/>
-    /// As of now, this method does no sort of image processing. LLMs usually
-    /// do not work with arbitrary image sizes. In the future, we might have
-    /// to resize the images before sending them to the model.<br/>
-    /// <br/>
     /// Note as well that this method returns just the base64 string without
     /// any data URI prefix (like "data:image/png;base64,"). The caller has
     /// to take care of that if needed.

app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs

@@ -1,7 +1,9 @@
 using AIStudio.Provider;
 using AIStudio.Provider.OpenAI;
 using AIStudio.Settings;
+using AIStudio.Tools.Rust;
 using AIStudio.Tools.Security;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Chat;
 
@@ -14,14 +16,16 @@ public static class ListContentBlockExtensions
     /// <param name="roleTransformer">A function that transforms each content block into a message result asynchronously.</param>
     /// <param name="provider">The configured provider, whose model is being written to.</param>
     /// <param name="textSubContentFactory">A factory function to create text sub-content.</param>
-    /// <param name="imageSubContentFactory">A factory function to create image sub-content.</param>
+    /// <param name="imageSubContentFactory">A factory function to create image sub-content from an image prepared for the model.</param>
+    /// <param name="token">The cancellation token.</param>
     /// <returns>An asynchronous task that resolves to a list of transformed results.</returns>
     public static async Task<IList<IMessageBase>> BuildMessagesAsync(
         this List<ContentBlock> blocks,
         AIStudio.Settings.Provider provider,
         Func<ChatRole, string> roleTransformer,
         Func<string, ISubContent> textSubContentFactory,
-        Func<FileAttachmentImage, Task<ISubContent>> imageSubContentFactory)
+        Func<PreparedModelImage, ISubContent> imageSubContentFactory,
+        CancellationToken token = default)
     {
         //
         // Asked through the configured provider, so that what a person set in their expert settings
@@ -59,7 +63,7 @@ public static class ListContentBlockExtensions
                 
                 // Text prompt with images as attachments, and the provider/model can process images:
                 case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && text.FileAttachments.ContainsImages():
-                    messageTaskList.Add(CreateMultimodalMessageAsync(block, text, textSubContentFactory, imageSubContentFactory));
+                    messageTaskList.Add(CreateMultimodalMessageAsync(block, text));
                     break;
             }
         }
@@ -81,34 +85,41 @@ public static class ListContentBlockExtensions
         }
         
         // Local function to create a multimodal message asynchronously.
-        Task<IMessageBase> CreateMultimodalMessageAsync(
-            ContentBlock block,
-            ContentText text,
-            Func<string, ISubContent> innerTextSubContentFactory,
-            Func<FileAttachmentImage, Task<ISubContent>> innerImageSubContentFactory)
+        Task<IMessageBase> CreateMultimodalMessageAsync(ContentBlock block, ContentText text)
         {
             return Task.Run(async () =>
             {
-                var imagesTasks = text.FileAttachments
+                var images = await Task.WhenAll(text.FileAttachments
                     .Where(x => x is { IsImage: true, Exists: true })
                     .Cast<FileAttachmentImage>()
-                    .Select(innerImageSubContentFactory)
-                    .ToList();
-                
-                Task.WaitAll(imagesTasks);
-                var images = imagesTasks.Select(t => t.Result).ToList();
+                    .Select(CreateImageSubContentAsync));
                 
                 return new MultimodalMessage
                 {
                     Role = roleTransformer(block.Role),
                     Content =
                     [
-                        innerTextSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
-                        ..images,
+                        textSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
+                        ..images.OfType<ISubContent>(),
                     ]
                 } as IMessageBase;
             });
         }
+
+        //
+        // Local function to prepare one image for the model. An image which cannot be prepared
+        // stays out of the message, and the user learns why. It used to go along as an image
+        // without data, which the providers refuse together with the whole request.
+        //
+        async Task<ISubContent?> CreateImageSubContentAsync(FileAttachmentImage image)
+        {
+            var prepared = await Program.SERVICE_PROVIDER.GetRequiredService<ModelImageService>().PrepareAsync(image, provider, token);
+            if (prepared.IsPrepared)
+                return imageSubContentFactory(prepared);
+
+            await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, prepared.ErrorCode.ToUserMessage(image.FileName)));
+            return null;
+        }
     }
 
     /// <summary>
@@ -158,20 +169,16 @@ public static class ListContentBlockExtensions
         Text = text,
     };
 
-    private static async Task<ISubContent> DirectImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrl
+    private static ISubContent DirectImageSubContentFactory(PreparedModelImage image) => new SubContentImageUrl
     {
-        ImageUrl = await attachment.TryAsBase64() is (true, var base64Content)
-            ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-            : string.Empty,
+        ImageUrl = image.ToDataUrl(),
     };
 
-    private static async Task<ISubContent> NestedImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrlNested
+    private static ISubContent NestedImageSubContentFactory(PreparedModelImage image) => new SubContentImageUrlNested
     {
         ImageUrl = new SubContentImageUrlData
         {
-            Url = await attachment.TryAsBase64() is (true, var base64Content)
-                ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-                : string.Empty,
+            Url = image.ToDataUrl(),
         },
     };
 

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor

@@ -21,7 +21,7 @@
             <ConfigurationDirectory OptionDescription="@T("Default input folder")" Disabled="@this.DefaultsDisabled" DirectoryDialogTitle="@T("Select the default input folder")" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.InputDirectory)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.InputDirectory = value)" OptionHelp="@T("Leave empty when an input folder should be selected for every batch run.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.InputDirectory, out var meta) && meta.IsLocked"/>
             <ConfigurationOption OptionDescription="@T("Include subfolders by default?")" Disabled="@this.DefaultsDisabled" LabelOn="@T("Subfolders are included")" LabelOff="@T("Only the selected folder is processed")" State="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeSubdirectories)" StateUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeSubdirectories = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.IncludeSubdirectories, out var meta) && meta.IsLocked"/>
             <ConfigurationOption OptionDescription="@T("Include hidden files and folders by default?")" Disabled="@this.DefaultsDisabled" LabelOn="@T("Hidden files and folders are included")" LabelOff="@T("Hidden files and folders are ignored")" State="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeHiddenFilesAndFolders)" StateUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeHiddenFilesAndFolders = value)" OptionHelp="@T("Hidden are files and folders whose names start with a dot, such as .git or .env, and those hidden by the operating system. A file pattern which starts with a dot, such as .gitignore, selects such files anyway.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.IncludeHiddenFilesAndFolders, out var meta) && meta.IsLocked"/>
-            <ConfigurationText OptionDescription="@T("Default file patterns")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAlt" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns = value)" ResetValue="@(() => DataBatchProcessing.DEFAULT_FILE_PATTERNS)" ResetButtonText="@T("Restore default patterns")" OptionHelp="@T("Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.FilePatterns, out var meta) && meta.IsLocked"/>
+            <ConfigurationText OptionDescription="@T("Default file patterns")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAlt" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns = value)" ResetValue="@(() => DataBatchProcessing.DEFAULT_FILE_PATTERNS)" ResetButtonText="@T("Restore default patterns")" OptionHelp="@T("Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.FilePatterns, out var meta) && meta.IsLocked"/>
             <ConfigurationText OptionDescription="@T("Default excluded files and folders")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAltOff" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.ExcludePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.ExcludePatterns = value)" NumLines="@FileExclusions.DEFAULT_LINE_COUNT" MaxLines="@FileExclusions.MAX_LINE_COUNT" ResetValue="@(() => FileExclusions.DEFAULT_PATTERNS)" ResetButtonText="@T("Restore default")" OptionHelp="@T("These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.ExcludePatterns, out var meta) && meta.IsLocked"/>
             <ConfigurationNumber T="int" OptionDescription="@T("Ignore files larger than")" Min="@FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES" Unit="MB" Disabled="@this.DefaultsDisabled" Value="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.MaxFileSizeMegabytes)" ValueUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.MaxFileSizeMegabytes = value)" OptionHelp="@T("Matching files above this size are left out of a batch run, e.g., images of virtual machines.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.MaxFileSizeMegabytes, out var meta) && meta.IsLocked"/>
 

app/MindWork AI Studio/Plugins/configuration/plugin.lua

@@ -650,7 +650,7 @@ CONFIG["SETTINGS"] = {}
 --
 -- Configure the default file patterns and whether subfolders as well as hidden files and folders are included.
 -- Separate multiple patterns with semicolons.
--- CONFIG["SETTINGS"]["DataBatchProcessing.FilePatterns"] = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.jpg;*.jpeg;*.png;*.webp;*.gif;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm"
+-- CONFIG["SETTINGS"]["DataBatchProcessing.FilePatterns"] = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.jpg;*.jpeg;*.png;*.webp;*.gif;*.bmp;*.tif;*.tiff;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm"
 -- CONFIG["SETTINGS"]["DataBatchProcessing.IncludeSubdirectories"] = false
 -- CONFIG["SETTINGS"]["DataBatchProcessing.IncludeHiddenFilesAndFolders"] = false
 --

app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua

@@ -534,8 +534,8 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Was not able to write the result file: {0}
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Die Ergebnisdatei konnte nicht geschrieben werden: {0}"
 
--- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Bilder im JPG-, PNG-, WebP- oder GIF-Format werden direkt an das Modell übergeben, daher benötigen Sie ein Modell, das Bilder versteht. Falls das ausgewählte Modell keine Bilder lesen kann, entfernen Sie die Dateimuster für Bilder, z. B. *.jpg oder *.png, bevor Sie starten."
+-- Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3011193639"] = "Bilder im JPG-, PNG-, WebP-, GIF-, BMP- oder TIFF-Format werden als Bilder an das Modell übergeben, daher benötigen Sie ein Modell, das Bilder versteht. Falls das ausgewählte Modell keine Bilder lesen kann, entfernen Sie die Dateimuster für Bilder, z. B. *.jpg oder *.png, bevor Sie starten."
 
 -- The batch run finished, but one file could not be processed. See the progress table and log for details.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "Die Stapelverarbeitung ist abgeschlossen, aber eine Datei konnte nicht verarbeitet werden. Weitere Informationen finden Sie in der Fortschrittstabelle und im Protokoll."
@@ -663,9 +663,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Start batch processing
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Stapelverarbeitung starten"
 
--- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "Dieses Bildformat kann nicht an das Modell übergeben werden. Bitte konvertieren Sie das Bild in JPG, PNG, WebP oder GIF."
-
 -- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Die Ergebnistabelle des vorherigen Durchlaufs konnte nicht gelesen werden. Die bereits abgeschlossenen Dokumente können nicht wiederhergestellt werden und werden erneut verarbeitet."
 
@@ -690,9 +687,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Include hidden files and folders?
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Versteckte Dateien und Ordner einbeziehen?"
 
--- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "Das Bild überschreitet die maximale Größe von {0} MB, die der ausgewählte Anbieter pro Bild akzeptiert."
-
 -- Restore default
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Standard wiederherstellen"
 
@@ -8592,9 +8586,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
 -- Default minimum pause between files
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Standardmäßige Mindestpause zwischen Dateien"
 
--- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Trennen Sie mehrere Dateimuster durch ein Semikolon, z. B. *.pdf;*.docx. Die Standardmuster umfassen alle unterstützten Audio- und Videoformate sowie Bilder im JPG-, PNG-, WebP- und GIF-Format."
-
 -- Instructions
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Anweisungen"
 
@@ -8637,6 +8628,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T22
 -- Hidden files and folders are included
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2288334048"] = "Versteckte Dateien und Ordner werden einbezogen"
 
+-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T232588799"] = "Trennen Sie mehrere Dateimuster durch ein Semikolon, z. B. *.pdf;*.docx. Die Standardmuster umfassen alle unterstützten Audio- und Videoformate sowie Bilder im JPG-, PNG-, WebP-, GIF-, BMP- und TIFF-Format."
+
 -- The lower end of the random pause interval. AI Studio never allows less than 6 seconds.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T237774509"] = "Das untere Ende des Intervalls für die zufällige Pause. AI Studio erlaubt niemals weniger als 6 Sekunden."
 
@@ -13443,6 +13437,30 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Bild für KI-
 -- Plugin archive
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin-Archiv"
 
+-- AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1345506748"] = "AI Studio konnte das Bild „{0}“ nicht für die KI vorbereiten und hat es daher nicht gesendet."
+
+-- The image '{0}' could not be made small enough for the selected provider, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1463083567"] = "Das Bild „{0}“ konnte für den ausgewählten Anbieter nicht ausreichend verkleinert werden und wurde daher nicht gesendet."
+
+-- The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2022724454"] = "Das Bild „{0}“ konnte nicht geöffnet und daher nicht gesendet werden. Wenn das Bild auf einem Netzlaufwerk gespeichert ist, ist das Laufwerk möglicherweise nicht verfügbar."
+
+-- The image '{0}' does not exist anymore and was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2330229240"] = "Das Bild „{0}“ existiert nicht mehr und wurde nicht gesendet."
+
+-- The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T3457568536"] = "Das Bild „{0}“ hat nicht das Format, das seine Dateiendung angibt, und wurde daher nicht gesendet. Bitte korrigieren Sie die Dateiendung."
+
+-- The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T4029166607"] = "Das Bild „{0}“ konnte nicht gelesen und daher nicht gesendet werden. Möglicherweise ist es beschädigt oder unvollständig übertragen worden."
+
+-- The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T567941882"] = "Das Bild „{0}“ hat zu viele Pixel, als dass AI Studio es vorbereiten könnte, und wurde daher nicht gesendet. Bitte verkleinern Sie das Bild zuerst."
+
+-- The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T568830141"] = "Das Bild „{0}“ liegt in einem Format vor, das AI Studio nicht an ein KI-Modell übergeben kann, und wurde daher nicht gesendet. Bitte konvertieren Sie das Bild in JPG oder PNG."
+
 -- Attempt to override instructions
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONFINDINGCATEGORYEXTENSIONS::T161976090"] = "Versuch, Anweisungen zu überschreiben"
 
@@ -14673,11 +14691,8 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T90753429"]
 -- Please enter the secret necessary for authentication.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T968385876"] = "Bitte geben Sie das für die Authentifizierung benötigte Geheimnis ein."
 
--- Unsupported image format
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1398282880"] = "Nicht unterstütztes Bildformat"
-
--- File has no extension
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1555980031"] = "Datei hat keine Erweiterung"
+-- AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1578559061"] = "AI Studio kann keine Bilder im Format {0} an ein KI-Modell übergeben. Bitte konvertieren Sie das Bild in JPG oder PNG."
 
 -- Audio files are not supported yet
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T2919730669"] = "Audio-Dateien werden noch nicht unterstützt."

app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua

@@ -534,8 +534,8 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Was not able to write the result file: {0}
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Was not able to write the result file: {0}"
 
--- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
+-- Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3011193639"] = "Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
 
 -- The batch run finished, but one file could not be processed. See the progress table and log for details.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "The batch run finished, but one file could not be processed. See the progress table and log for details."
@@ -663,9 +663,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Start batch processing
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Start batch processing"
 
--- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."
-
 -- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again."
 
@@ -690,9 +687,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Include hidden files and folders?
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Include hidden files and folders?"
 
--- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "The image is larger than {0} MB, which is the most the selected provider accepts for one image."
-
 -- Restore default
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Restore default"
 
@@ -8592,9 +8586,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
 -- Default minimum pause between files
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Default minimum pause between files"
 
--- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format."
-
 -- Instructions
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Instructions"
 
@@ -8637,6 +8628,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T22
 -- Hidden files and folders are included
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2288334048"] = "Hidden files and folders are included"
 
+-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T232588799"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format."
+
 -- The lower end of the random pause interval. AI Studio never allows less than 6 seconds.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T237774509"] = "The lower end of the random pause interval. AI Studio never allows less than 6 seconds."
 
@@ -13443,6 +13437,30 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Image for AI
 -- Plugin archive
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin archive"
 
+-- AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1345506748"] = "AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent."
+
+-- The image '{0}' could not be made small enough for the selected provider, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1463083567"] = "The image '{0}' could not be made small enough for the selected provider, so it was not sent."
+
+-- The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2022724454"] = "The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable."
+
+-- The image '{0}' does not exist anymore and was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2330229240"] = "The image '{0}' does not exist anymore and was not sent."
+
+-- The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T3457568536"] = "The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension."
+
+-- The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T4029166607"] = "The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely."
+
+-- The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T567941882"] = "The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first."
+
+-- The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T568830141"] = "The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG."
+
 -- Attempt to override instructions
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONFINDINGCATEGORYEXTENSIONS::T161976090"] = "Attempt to override instructions"
 
@@ -14673,11 +14691,8 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T90753429"]
 -- Please enter the secret necessary for authentication.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T968385876"] = "Please enter the secret necessary for authentication."
 
--- Unsupported image format
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1398282880"] = "Unsupported image format"
-
--- File has no extension
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1555980031"] = "File has no extension"
+-- AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1578559061"] = "AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG."
 
 -- Audio files are not supported yet
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T2919730669"] = "Audio files are not supported yet"

app/MindWork AI Studio/Program.cs

@@ -175,6 +175,7 @@ internal sealed class Program
         builder.Services.AddSingleton<SettingsManager>();
         builder.Services.AddSingleton<AIGeneratedContentDisclosureService>();
         builder.Services.AddSingleton<PromptInjectionGuardService>();
+        builder.Services.AddSingleton<ModelImageService>();
         builder.Services.AddSingleton<ToolSettingsService>();
         builder.Services.AddSingleton<WebPageRetrievalService>();
         builder.Services.AddSingleton<IToolImplementation, ReadWebPageTool>();

app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs

@@ -59,18 +59,16 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
                 Text = text,
             },
             
-            // Anthropic-specific image sub-content:
-            async attachment => new SubContentImage
+            // Anthropic-specific image sub-content, which takes the base64 and its type apart:
+            image => new SubContentImage
             {
                 Source = new SubContentBase64Image
                 {
-                    Data = await attachment.TryAsBase64(token: token) is (true, var base64Content)
-                        ? base64Content
-                        : string.Empty,
-                    
-                    MediaType = attachment.DetermineMimeType(),
+                    Data = image.Base64,
+                    MediaType = image.MimeType,
                 }
-            }
+            },
+            token
         );
 
         //

app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs

@@ -113,15 +113,14 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
                                        {
                                            Text = text,
                                        },
-                                       async attachment => new SubContentImageUrlNested
+                                       image => new SubContentImageUrlNested
                                        {
                                            ImageUrl = new SubContentImageUrlData
                                            {
-                                               Url = await attachment.TryAsBase64(token: token) is (true, var base64Content)
-                                                   ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-                                                   : string.Empty,
+                                               Url = image.ToDataUrl(),
                                            },
-                                       });
+                                       },
+                                       token);
 
                                    return new ChatCompletionAPIRequest
                                    {
@@ -192,12 +191,11 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
             {
                 Text = text,
             },
-            async attachment => new SubContentInputImage
+            image => new SubContentInputImage
             {
-                ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content)
-                    ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-                    : string.Empty,
-            });
+                ImageUrl = image.ToDataUrl(),
+            },
+            token);
 
         var baseInput = new List<object> { systemPrompt };
         baseInput.AddRange(messages);

app/MindWork AI Studio/Provider/SelfHosted/HostExtensions.cs

@@ -36,6 +36,29 @@ public static class HostExtensions
         _ => "embeddings",
     };
     
+    /// <summary>
+    /// The largest image file, in bytes, which the engine accepts in a chat request, where we know of one.
+    /// </summary>
+    /// <remarks>
+    /// Like the limit of a cloud provider, see LLMProvidersExtensions.GetMaxImageSizeBytes, this
+    /// belongs to the API of the engine, not to the model. What we actually send is asked through
+    /// IImageSourceExtensions.MaxImageSizeBytesFor.
+    /// </remarks>
+    /// <param name="host">The engine.</param>
+    /// <returns>The size limit of one image file, or null when the engine states none we know of.</returns>
+    public static long? GetMaxImageSizeBytes(this Host host) => host switch
+    {
+        //
+        // llmman answers a chat request larger than 2 MiB with 413: its route /v1/chat/completions
+        // keeps the default limit of its web framework, read on 2026-10-22 at
+        // https://github.com/llmmanorg/llmman/blob/main/src/cmd/serve.rs. An image of 1 MiB takes
+        // about 1.33 MiB as base64, which leaves room for the text of the conversation:
+        //
+        Host.LLMMAN => 1_048_576,
+
+        _ => null,
+    };
+
     public static bool IsChatSupported(this Host host)
     {
         switch (host)

app/MindWork AI Studio/Settings/DataModel/DataBatchProcessing.cs

@@ -11,7 +11,7 @@ namespace AIStudio.Settings.DataModel;
 /// <param name="configSelection">The managed-configuration selector.</param>
 public sealed class DataBatchProcessing(Expression<Func<Data, DataBatchProcessing>>? configSelection = null)
 {
-    public const string DEFAULT_FILE_PATTERNS = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.jpg;*.jpeg;*.png;*.webp;*.gif;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm";
+    public const string DEFAULT_FILE_PATTERNS = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.jpg;*.jpeg;*.png;*.webp;*.gif;*.bmp;*.tif;*.tiff;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm";
     public const int MIN_DELAY_SECONDS = 6;
     public const int MAX_DELAY_SECONDS = 300;
     public const int DEFAULT_MIN_DELAY_SECONDS = 6;

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -98,18 +98,19 @@ public static class FileTypes
 
     // Media hierarchy
     public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
-        "jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic", "avif");
+        "jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "svg", "webp", "heic", "avif");
 
     /// <summary>
-    /// The image formats we hand to a model as they are.
+    /// The image formats AI Studio can prepare for a model.
     /// </summary>
     /// <remarks>
-    /// The providers accept these, while other formats, such as TIFF or HEIC, would end in an
-    /// error of the provider which tells the user nothing. The default file patterns of the batch
-    /// processing assistant list the same formats, see DataBatchProcessing.DEFAULT_FILE_PATTERNS.
+    /// These are the formats the runtime decodes before it sends an image to a model as JPEG or
+    /// PNG, see prepare_image_for_model in runtime/src/image.rs. Other images, such as HEIC, AVIF,
+    /// or SVG, cannot go to a model at all. The default file patterns of the batch processing
+    /// assistant list the same formats, see DataBatchProcessing.DEFAULT_FILE_PATTERNS.
     /// </remarks>
     public static readonly FileTypeFilter MODEL_IMAGE = FileTypeFilter.Leaf(TB("Image for AI models"),
-        "jpg", "jpeg", "png", "webp", "gif");
+        "jpg", "jpeg", "png", "webp", "gif", "bmp", "tif", "tiff");
 
     /// <summary>
     /// Gets the prototype visual-asset image formats.

app/MindWork AI Studio/Tools/Rust/ModelImageErrorCode.cs

@@ -0,0 +1,63 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Why the Rust runtime could not prepare an image for an AI model.
+/// </summary>
+/// <remarks>
+/// UNKNOWN comes first on purpose. The converter for the enums of the runtime falls back to the
+/// default value for a code it does not know, e.g., from a newer runtime, and such a code must
+/// never read as a prepared image.
+/// </remarks>
+public enum ModelImageErrorCode
+{
+    /// <summary>
+    /// The runtime answered with a code this version does not know, or it could not be asked at all.
+    /// </summary>
+    UNKNOWN,
+
+    /// <summary>
+    /// No failure happened: the image was prepared.
+    /// </summary>
+    NONE,
+
+    //
+    // Codes reported by the Rust runtime:
+    //
+
+    FILE_NOT_FOUND,
+
+    /// <summary>
+    /// The file exists, but could not be opened, e.g., for lack of permission.
+    /// </summary>
+    FILE_NOT_READABLE,
+
+    /// <summary>
+    /// AI Studio cannot decode this image format, e.g., HEIC, AVIF, or SVG.
+    /// </summary>
+    UNSUPPORTED_FORMAT,
+
+    /// <summary>
+    /// The content of the file does not match its file extension.
+    /// </summary>
+    CONTENT_MISMATCH,
+
+    /// <summary>
+    /// The file is damaged or uses a variant of its format which cannot be decoded.
+    /// </summary>
+    NOT_DECODABLE,
+
+    /// <summary>
+    /// Decoding the image would need more memory than the runtime allows.
+    /// </summary>
+    TOO_MANY_PIXELS,
+
+    /// <summary>
+    /// Not even the smallest size and lowest quality the runtime accepts fit into the byte budget.
+    /// </summary>
+    BUDGET_NOT_REACHABLE,
+
+    /// <summary>
+    /// Any failure in the runtime which does not carry a code of its own.
+    /// </summary>
+    INTERNAL,
+}

app/MindWork AI Studio/Tools/Rust/ModelImageErrorCodeExtensions.cs

@@ -0,0 +1,36 @@
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Translates why an image could not be prepared for an AI model into user-facing text.
+/// </summary>
+/// <remarks>
+/// The runtime writes the technical details into the log. The texts here name what the user can
+/// act on, such as converting the image into another format.
+/// </remarks>
+public static class ModelImageErrorCodeExtensions
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ModelImageErrorCodeExtensions).Namespace, nameof(ModelImageErrorCodeExtensions));
+
+    /// <summary>
+    /// Gets the localized message which explains why an image was not sent to the AI.
+    /// </summary>
+    /// <param name="code">Why the image could not be prepared.</param>
+    /// <param name="fileName">The name of the image file, as shown to the user.</param>
+    /// <returns>The localized message.</returns>
+    public static string ToUserMessage(this ModelImageErrorCode code, string fileName) => string.Format(ToUserMessageFormat(code), fileName);
+
+    private static string ToUserMessageFormat(ModelImageErrorCode code) => code switch
+    {
+        ModelImageErrorCode.FILE_NOT_FOUND => TB("The image '{0}' does not exist anymore and was not sent."),
+        ModelImageErrorCode.FILE_NOT_READABLE => TB("The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable."),
+        ModelImageErrorCode.UNSUPPORTED_FORMAT => TB("The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG."),
+        ModelImageErrorCode.CONTENT_MISMATCH => TB("The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension."),
+        ModelImageErrorCode.NOT_DECODABLE => TB("The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely."),
+        ModelImageErrorCode.TOO_MANY_PIXELS => TB("The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first."),
+        ModelImageErrorCode.BUDGET_NOT_REACHABLE => TB("The image '{0}' could not be made small enough for the selected provider, so it was not sent."),
+
+        _ => TB("AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent."),
+    };
+}

app/MindWork AI Studio/Tools/Rust/PreparedModelImage.cs

@@ -0,0 +1,39 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// An image as the Rust runtime prepared it for an AI model, or why it could not.
+/// </summary>
+/// <remarks>
+/// The base64 and its MIME type travel side by side, so a provider which wants a data URL builds
+/// one and a provider which wants them apart, such as Anthropic, never parses one.
+/// </remarks>
+/// <param name="ErrorCode">NONE when the image was prepared; then the other values describe it.</param>
+/// <param name="Base64">The encoded image as base64, without a data URI prefix.</param>
+/// <param name="MimeType">The MIME type of the encoded image: image/jpeg or image/png.</param>
+/// <param name="Width">The width of the prepared image in pixels.</param>
+/// <param name="Height">The height of the prepared image in pixels.</param>
+/// <param name="WasResized">Whether the runtime scaled the image down.</param>
+public sealed record PreparedModelImage(ModelImageErrorCode ErrorCode, string Base64, string MimeType, uint Width, uint Height, bool WasResized)
+{
+    /// <summary>
+    /// Whether the image can go to the model.
+    /// </summary>
+    /// <remarks>
+    /// The image itself has to be there as well: a success without one would become an empty image
+    /// in the request, which providers refuse along with the whole request.
+    /// </remarks>
+    public bool IsPrepared => this.ErrorCode is ModelImageErrorCode.NONE && !string.IsNullOrEmpty(this.Base64);
+
+    /// <summary>
+    /// The image as a data URL, as the OpenAI-compatible APIs take it.
+    /// </summary>
+    /// <returns>The data URL.</returns>
+    public string ToDataUrl() => $"data:{this.MimeType};base64,{this.Base64}";
+
+    /// <summary>
+    /// An image which could not be prepared.
+    /// </summary>
+    /// <param name="errorCode">Why it could not be prepared.</param>
+    /// <returns>The failed image.</returns>
+    public static PreparedModelImage Failed(ModelImageErrorCode errorCode) => new(errorCode, string.Empty, string.Empty, 0, 0, false);
+}

app/MindWork AI Studio/Tools/Services/ModelImageService.cs

@@ -0,0 +1,93 @@
+using AIStudio.Chat;
+using AIStudio.Tools.Rust;
+
+using Microsoft.Extensions.Caching.Memory;
+// ReSharper disable NotAccessedPositionalProperty.Local
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Prepares the images which go to an AI model, and keeps the latest results for a while.
+/// </summary>
+/// <remarks>
+/// Every message builds the whole conversation anew, with every image in it. Without the cache,
+/// each photo of a chat would be decoded and encoded again for each message, which a small device
+/// such as a Raspberry Pi notices. The cache also lets the Batch Processing assistant check an
+/// image ahead of its request without preparing it twice.
+///
+/// The cache is a MemoryCache of its own: the shared one, which the Markdown library uses, has no
+/// size limit, and its entries state no size.
+/// </remarks>
+/// <param name="rustService">The runtime which prepares the images.</param>
+/// <param name="logger">The logger.</param>
+public sealed class ModelImageService(RustService rustService, ILogger<ModelImageService> logger) : IDisposable
+{
+    /// <summary>
+    /// How much memory the cached images may take: a 1 MB photo takes about 2.7 MB as base64 in a string.
+    /// </summary>
+    private const long CACHE_SIZE_LIMIT_BYTES = 32 * 1024 * 1024;
+
+    /// <summary>
+    /// How long a cached image waits for its next message. A chat paused for longer prepares its images again.
+    /// </summary>
+    private static readonly TimeSpan CACHE_SLIDING_EXPIRATION = TimeSpan.FromMinutes(10);
+
+    private readonly MemoryCache cache = new(new MemoryCacheOptions { SizeLimit = CACHE_SIZE_LIMIT_BYTES });
+
+    /// <summary>
+    /// Prepares an attached image for the model of the given provider.
+    /// </summary>
+    /// <remarks>
+    /// A changed file and another byte budget each make for a new preparation. An image which could
+    /// not be prepared is not cached, so the next message tries it again, e.g., once the network
+    /// drive holding it is back.
+    /// </remarks>
+    /// <param name="image">The attached image.</param>
+    /// <param name="provider">The configured provider the image goes to.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The prepared image, or why it could not be prepared.</returns>
+    public async Task<PreparedModelImage> PrepareAsync(FileAttachmentImage image, AIStudio.Settings.Provider provider, CancellationToken token = default)
+    {
+        var file = new FileInfo(image.FilePath);
+        if (!file.Exists)
+        {
+            logger.LogWarning("The image '{ImagePath}' does not exist anymore and cannot go to the model.", image.FilePath);
+            return PreparedModelImage.Failed(ModelImageErrorCode.FILE_NOT_FOUND);
+        }
+
+        var maxBytes = IImageSourceExtensions.MaxImageSizeBytesFor(provider);
+        var key = new CacheKey(file.FullName, file.LastWriteTimeUtc, file.Length, maxBytes);
+        if (this.cache.TryGetValue(key, out PreparedModelImage? cached) && cached is not null)
+        {
+            logger.LogInformation("Took the image '{ImagePath}' for a budget of {MaxBytes} bytes from the cache.", file.FullName, maxBytes);
+            return cached;
+        }
+
+        var prepared = await rustService.PrepareImageForModelAsync(file.FullName, maxBytes, token);
+        if (!prepared.IsPrepared)
+        {
+            logger.LogWarning("The image '{ImagePath}' could not be prepared for the model: {ErrorCode}.", file.FullName, prepared.ErrorCode);
+            return prepared;
+        }
+
+        //
+        // The runtime logs the sizes before and after, so this line only says where it came from:
+        //
+        logger.LogInformation("The runtime prepared the image '{ImagePath}' for a budget of {MaxBytes} bytes; it is cached now.", file.FullName, maxBytes);
+        this.cache.Set(key, prepared, new MemoryCacheEntryOptions
+        {
+            Size = prepared.Base64.Length * (long)sizeof(char),
+            SlidingExpiration = CACHE_SLIDING_EXPIRATION,
+        });
+
+        return prepared;
+    }
+
+    /// <inheritdoc />
+    public void Dispose() => this.cache.Dispose();
+
+    /// <summary>
+    /// What makes a prepared image valid: the file as it is now, and the budget it was made for.
+    /// </summary>
+    private readonly record struct CacheKey(string FullPath, DateTime LastWriteTimeUtc, long Length, long MaxBytes);
+}

app/MindWork AI Studio/Tools/Services/RustService.Image.cs

@@ -26,4 +26,34 @@ public sealed partial class RustService
         return await response.Content.ReadFromJsonAsync<ImagePrepareResponse>(this.jsonRustSerializerOptions, token)
             ?? throw new InvalidDataException("The Rust image preparation returned an empty response.");
     }
+
+    /// <summary>
+    /// Prepares a local image for an AI model in the Rust runtime.
+    /// </summary>
+    /// <remarks>
+    /// The runtime turns the image upright, scales it down, and encodes it anew as JPEG or PNG,
+    /// within the given number of bytes. An image it cannot prepare, e.g., in an unsupported format,
+    /// comes back with an error code. So does a failed call to the runtime, as UNKNOWN, and never as
+    /// an empty image.
+    /// </remarks>
+    /// <param name="path">The absolute path of the image file.</param>
+    /// <param name="maxBytes">The largest number of bytes the encoded image may have, before base64.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The prepared image, or why it could not be prepared.</returns>
+    public async Task<PreparedModelImage> PrepareImageForModelAsync(string path, long maxBytes, CancellationToken token = default)
+    {
+        try
+        {
+            using var response = await this.http.PostAsJsonAsync("/image/prepare-for-model", new { path, maxBytes }, this.jsonRustSerializerOptions, token);
+            response.EnsureSuccessStatusCode();
+
+            return await response.Content.ReadFromJsonAsync<PreparedModelImage>(this.jsonRustSerializerOptions, token)
+                ?? PreparedModelImage.Failed(ModelImageErrorCode.UNKNOWN);
+        }
+        catch (Exception e) when (e is not OperationCanceledException)
+        {
+            this.logger?.LogError(e, "Preparing the image '{ImagePath}' for an AI model failed in the runtime.", path);
+            return PreparedModelImage.Failed(ModelImageErrorCode.UNKNOWN);
+        }
+    }
 }

app/MindWork AI Studio/Tools/Validation/FileExtensionValidation.cs

@@ -69,6 +69,17 @@ public static class FileExtensionValidation
                         TB("Images are not supported at this place")));
                     return false;
                 
+                //
+                // Whether AI Studio can prepare an image for a model does not depend on the
+                // provider. This holds for templates as well, therefore: a HEIC image attached
+                // there could never be sent.
+                //
+                case UseCase.ATTACHING_CONTENT when !FileTypes.IsAllowedPath(filePath, FileTypes.MODEL_IMAGE):
+                    await MessageBus.INSTANCE.SendWarning(new(
+                        Icons.Material.Filled.ImageNotSupported,
+                        string.Format(TB("AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG."), Path.GetExtension(filePath).TrimStart('.').ToUpperInvariant())));
+                    return false;
+
                 // In this use case, we don't validate the provider capabilities:
                 case UseCase.ATTACHING_CONTENT when !validateMediaFileTypes:
                     return true;
@@ -116,31 +127,4 @@ public static class FileExtensionValidation
             TB("Unsupported file type")));
         return false;
     }
-
-    /// <summary>
-    /// Validates that the file is a supported image format and sends appropriate MessageBus notifications when invalid.
-    /// </summary>
-    /// <param name="filePath">The file path to validate.</param>
-    /// <returns>True if valid image, false if invalid (error already sent via MessageBus).</returns>
-    public static async Task<bool> IsImageExtensionValidWithNotifyAsync(string filePath)
-    {
-        var ext = Path.GetExtension(filePath).TrimStart('.');
-        if (string.IsNullOrWhiteSpace(ext))
-        {
-            await MessageBus.INSTANCE.SendError(new(
-                Icons.Material.Filled.ImageNotSupported,
-                TB("File has no extension")));
-            return false;
-        }
-
-        if (FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE))
-        {
-            await MessageBus.INSTANCE.SendError(new(
-                Icons.Material.Filled.ImageNotSupported,
-                TB("Unsupported image format")));
-            return false;
-        }
-
-        return true;
-    }
 }

app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md

@@ -38,7 +38,7 @@
 - Improved local data sources for folders. You can now choose which files AI Studio indexes, for example, only PDF files, and whether it includes subfolders as well as hidden files and folders. By default, AI Studio now also indexes text files it did not recognize before, such as program code of any kind or a LICENSE file, so a Git repository makes a useful data source. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution.
 - Improved local data sources for folders, which now leave out hidden files and folders as well as build output and downloaded dependencies, such as the folders `node_modules`, `bin`, and `obj`. You can change this in the expert settings of the data source. When an existing data source contains such files, AI Studio removes them from its index during the next update.
 - Improved the background embeddings page, which now lists the files of a local data source that AI Studio left out, for example, images or files that are too large. It also shows which file types and folders lie outside what the data source selects.
-- Improved the Batch Processing assistant, which now also processes images in JPG, PNG, WebP, and GIF format when the selected model can read images. If you used the assistant before, your saved file patterns stay as they are: click "Restore default patterns" or add patterns such as `*.jpg;*.png` yourself.
+- Improved the Batch Processing assistant, which now also processes images in JPG, PNG, WebP, GIF, BMP, and TIFF format when the selected model can read images. If you used the assistant before, your saved file patterns stay as they are: click "Restore default patterns" or add patterns such as `*.jpg;*.png` yourself.
 - Improved the Batch Processing assistant, which now leaves out hidden files and folders as well as build output and downloaded dependencies, such as the folder `node_modules`. You can change both in the expert settings of the assistant.
 - Improved local data sources for folders and the Batch Processing assistant, which now leave out files larger than 1 GB, such as images of virtual machines. You can change this limit.
 - Improved local data sources for files and folders. When you move or rename the file or folder, for example, to another drive or a new computer, you can now select its new location when you edit the data source. AI Studio keeps what it has already prepared for every unchanged file, so these files do not go to your embedding provider again. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution.
@@ -49,6 +49,8 @@
 - Improved the dialogs for managing your data sources, profiles, and chat templates, which you open, for example, from a chat. The buttons for adding and importing now stay at the bottom of the dialog, so you no longer need to scroll past a long list to reach them. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution.
 - Improved how often AI Studio signs in to the server of an external data source (ERI server). AI Studio now keeps the server's answer to its security check for a minute, so a server which limits sign-ins no longer refuses the searches of the AI.
 - Improved the search results from external data sources (ERI servers), which now tell the AI what kind of content a result comes from, for example, a dictionary or a book chapter, when the server names it.
+- Improved images in chats and assistants. AI Studio now scales large photos down before it sends them to the AI, so they no longer fail at providers with size limits, such as Anthropic. Photos also no longer carry their location data along.
+- Improved attaching images to chats and assistants. AI Studio now converts BMP and TIFF images into a format the AI understands, and it tells you right away when it cannot give an image to the AI, for example, in HEIC or SVG format.
 - Changed the banner on the start page to a fresh design that shows what AI Studio can do today.
 - Changed when a chat you add to a workspace in the workspace list appears there. Like any other new chat, it now appears once there is something to keep, for example, your first message, instead of right away.
 - Fixed a warning about the confidence level that stayed in the dialog for adding a local data source, even after you chose an embedding provider that meets it. The dialog now checks again whenever you change either of them, so a warning also appears when you switch to a provider that falls short.
@@ -111,6 +113,8 @@
 - Fixed sharing plugins when the share dialog of Windows or macOS does not open or stops responding. AI Studio now notices this and offers to save the plugin archive to a location of your choice instead. Until you restart AI Studio, the plugin page then offers this export instead of sharing.
 - Fixed the Batch Processing assistant finding no documents when you chose an output folder that contains the input folder, such as its parent folder.
 - Fixed the Batch Processing assistant processing its own result files again as new documents when you chose the input folder as the output folder.
+- Fixed chats failing with an error from the provider when an attached image could not be read. AI Studio now leaves out such an image and tells you why.
+- Fixed photos taken with a phone appearing rotated in the Visual Briefing assistant when it optimizes images.
 - Updated the code contributions on the supporters page, which now thank everyone who has contributed code to AI Studio so far.
 - Upgraded several libraries to improve security.
 - Upgraded to Rust v1.99.1

app/Tests/Chat/ListContentBlockExtensionsTests.cs

@@ -1,6 +1,11 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Provider.OpenAI;
+using AIStudio.Tests.Tools;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+using Microsoft.Extensions.DependencyInjection;
 
 namespace AIStudio.Tests.Chat;
 
@@ -13,10 +18,47 @@ namespace AIStudio.Tests.Chat;
 /// Somebody who switched image input on for their own installation watched the picture attach and
 /// then watched it disappear on the way to the model -- every chat round and every tool round, with
 /// nothing anywhere saying why.
+///
+/// Each picture is prepared for the model on its way, by the runtime, which a test plays here.
+/// A picture which cannot be prepared has to stay out of the message entirely.
 /// </remarks>
 [TestFixture]
 public sealed class ListContentBlockExtensionsTests
 {
+    private string picturePath = string.Empty;
+    private FakeImageRuntime runtime = null!;
+    private RustService rustService = null!;
+    private ServiceProvider serviceProvider = null!;
+    private IServiceProvider previousServiceProvider = null!;
+
+    [SetUp]
+    public void PrepareServices()
+    {
+        this.picturePath = Path.Combine(Path.GetTempPath(), $"picture-{Guid.NewGuid():N}.webp");
+        File.WriteAllBytes(this.picturePath, [1, 2, 3]);
+
+        this.runtime = new FakeImageRuntime();
+        this.rustService = new RustService(this.runtime);
+        this.previousServiceProvider = Program.SERVICE_PROVIDER;
+        this.serviceProvider = new ServiceCollection()
+            .AddSingleton(this.rustService)
+            .AddSingleton<ModelImageService>()
+            .AddLogging()
+            .BuildServiceProvider();
+
+        Program.SERVICE_PROVIDER = this.serviceProvider;
+    }
+
+    [TearDown]
+    public void RestoreApplicationState()
+    {
+        Program.SERVICE_PROVIDER = this.previousServiceProvider;
+        this.serviceProvider.Dispose();
+        this.rustService.Dispose();
+        this.runtime.Dispose();
+        File.Delete(this.picturePath);
+    }
+
     [Test]
     public async Task ImageInputSwitchedOnByHandReachesTheMessageAsWell()
     {
@@ -31,6 +73,32 @@ public sealed class ListContentBlockExtensionsTests
         Assert.That(messages.Single(), Is.InstanceOf<MultimodalMessage>(), "The picture is part of the message because the person said this model can read one.");
     }
 
+    [Test]
+    public async Task APictureWhichCannotBePreparedLeavesNoEmptyPart()
+    {
+        this.runtime.ErrorCode = "NOT_DECODABLE";
+
+        var messages = await BlocksWithAPicture(this.picturePath).BuildMessagesUsingNestedImageUrlAsync(PictureReadingProvider());
+
+        var message = messages.Single() as MultimodalMessage;
+        Assert.That(message, Is.Not.Null, "The text still goes to the model.");
+        Assert.That(message!.Content, Has.Count.EqualTo(1).And.All.InstanceOf<SubContentText>(), "A picture without data makes the provider refuse the whole request.");
+    }
+
+    [Test]
+    public async Task TheTypeOfAPictureComesFromItsPreparation()
+    {
+        // The file is a WebP, which the runtime prepared as JPEG:
+        var messages = await BlocksWithAPicture(this.picturePath).BuildMessagesUsingDirectImageUrlAsync(PictureReadingProvider());
+
+        var picture = (messages.Single() as MultimodalMessage)!.Content.OfType<SubContentImageUrl>().Single();
+        Assert.Multiple(() =>
+        {
+            Assert.That(picture.ImageUrl, Is.EqualTo($"data:image/jpeg;base64,{FakeImageRuntime.IMAGE_BASE64}"));
+            Assert.That(this.runtime.RequestedPaths, Is.EqualTo(new[] { new FileInfo(this.picturePath).FullName }));
+        });
+    }
+
     [Test]
     public async Task WithoutThatSwitchThePictureStaysOut()
     {
@@ -41,11 +109,20 @@ public sealed class ListContentBlockExtensionsTests
         Assert.That(messages.Single(), Is.InstanceOf<TextMessage>(), "Nothing says this model reads pictures, so the text goes on its own.");
     }
 
+    /// <summary>
+    /// A model which reads text only, according to the rules, but which a person said reads pictures.
+    /// </summary>
+    private static AIStudio.Settings.Provider PictureReadingProvider() => new(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
+    {
+        CapabilityOverrides = new() { MultipleImageInput = true },
+    };
+
     /// <summary>
     /// One block of text with a picture hanging on it.
     /// </summary>
+    /// <param name="picturePath">Where the picture lies.</param>
     /// <returns>The blocks.</returns>
-    private static List<ContentBlock> BlocksWithAPicture() =>
+    private static List<ContentBlock> BlocksWithAPicture(string picturePath = "/tmp/picture.png") =>
     [
         new()
         {
@@ -54,12 +131,12 @@ public sealed class ListContentBlockExtensionsTests
             Content = new ContentText
             {
                 Text = "What is in this picture?",
-                FileAttachments = [new FileAttachmentImage("picture.png", "/tmp/picture.png", 1_024)],
+                FileAttachments = [new FileAttachmentImage(Path.GetFileName(picturePath), picturePath, 1_024)],
             },
         },
     ];
 
     private static ISubContent Text(string text) => new SubContentText { Text = text };
 
-    private static Task<ISubContent> Picture(FileAttachmentImage image) => Task.FromResult<ISubContent>(new SubContentText { Text = image.FileName });
+    private static ISubContent Picture(PreparedModelImage image) => new SubContentText { Text = image.MimeType };
 }

app/Tests/Provider/ImageSizeLimitTests.cs

@@ -1,5 +1,6 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
+using AIStudio.Provider.SelfHosted;
 
 namespace AIStudio.Tests.Provider;
 
@@ -7,8 +8,8 @@ namespace AIStudio.Tests.Provider;
 /// Checks how large an image may be when we send it through a provider.
 /// </summary>
 /// <remarks>
-/// A provider states its limit or says nothing. Either way, TryAsBase64 skips every image above our
-/// own limit, so a larger limit of a provider must never reach the checks done before sending.
+/// A provider or a self-hosted engine states its limit or says nothing. Either way, we never send
+/// more than our own limit, so a larger limit of a provider must never reach the image preparation.
 /// </remarks>
 [TestFixture]
 public sealed class ImageSizeLimitTests
@@ -19,7 +20,7 @@ public sealed class ImageSizeLimitTests
         Assert.Multiple(() =>
         {
             Assert.That(LLMProviders.ANTHROPIC.GetMaxImageSizeBytes(), Is.EqualTo(3_932_160));
-            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(LLMProviders.ANTHROPIC), Is.EqualTo(3_932_160));
+            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.ANTHROPIC)), Is.EqualTo(3_932_160));
         });
     }
 
@@ -29,17 +30,41 @@ public sealed class ImageSizeLimitTests
         Assert.Multiple(() =>
         {
             Assert.That(LLMProviders.OPEN_AI.GetMaxImageSizeBytes(), Is.Null, "Nobody stated a limit, and no number stands in for that.");
-            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(LLMProviders.OPEN_AI), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
+            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.OPEN_AI)), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
         });
     }
 
     [Test]
-    public void NoProviderGetsMoreThanTryAsBase64Sends()
+    public void LlmmanLeavesRoomForTheTextOfTheConversation()
     {
         Assert.Multiple(() =>
         {
-            foreach (var provider in Enum.GetValues<LLMProviders>())
-                Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(provider), Is.LessThanOrEqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), provider.ToString());
+            Assert.That(Host.LLMMAN.GetMaxImageSizeBytes(), Is.EqualTo(1_048_576));
+            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.SELF_HOSTED, Host.LLMMAN)), Is.EqualTo(1_048_576));
+            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.SELF_HOSTED, Host.OLLAMA)), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), "Ollama states no limit we know of.");
         });
     }
+
+    [Test]
+    public void AnEngineCountsOnlyForSelfHostedProviders()
+    {
+        // A provider keeps the host it was once configured with, even after somebody switched it
+        // to a cloud provider in the dialog:
+        var cloudProviderWithAnOldHost = ProviderOf(LLMProviders.OPEN_AI, Host.LLMMAN);
+
+        Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(cloudProviderWithAnOldHost), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
+    }
+
+    [Test]
+    public void NoProviderGetsMoreThanOurOwnLimit()
+    {
+        Assert.Multiple(() =>
+        {
+            foreach (var llmProvider in Enum.GetValues<LLMProviders>())
+                foreach (var host in Enum.GetValues<Host>())
+                    Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(llmProvider, host)), Is.LessThanOrEqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), $"{llmProvider} with {host}");
+        });
+    }
+
+    private static AIStudio.Settings.Provider ProviderOf(LLMProviders llmProvider, Host host = Host.NONE) => new(0, "test", "Test", llmProvider, new AIStudio.Provider.Model("model", null), Host: host);
 }

app/Tests/Tools/FakeImageRuntime.cs

@@ -0,0 +1,60 @@
+using System.Collections.Concurrent;
+using System.Net;
+using System.Text;
+using System.Text.Json;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Plays the image route of the runtime for a test, and notes what the app asked for.
+/// </summary>
+/// <remarks>
+/// It prepares nothing: every image comes back as the same three bytes, with the code and type the
+/// test sets. The answer is spelled out in the wire format of the runtime, so a test fails as soon
+/// as the app stops understanding what the runtime actually sends.
+/// </remarks>
+internal sealed class FakeImageRuntime : HttpMessageHandler
+{
+    /// <summary>
+    /// The three bytes every prepared image consists of, as base64.
+    /// </summary>
+    public const string IMAGE_BASE64 = "AQID";
+
+    private readonly ConcurrentQueue<string> requestedPaths = new();
+    private readonly ConcurrentQueue<long> requestedBudgets = new();
+
+    /// <summary>
+    /// The code the runtime answers with, as the runtime spells it.
+    /// </summary>
+    public string ErrorCode { get; set; } = "NONE";
+
+    /// <summary>
+    /// The MIME type the runtime reports for every prepared image.
+    /// </summary>
+    public string MimeType { get; set; } = "image/jpeg";
+
+    /// <summary>
+    /// The paths of the images the app asked to prepare, in order.
+    /// </summary>
+    public IReadOnlyList<string> RequestedPaths => [..this.requestedPaths];
+
+    /// <summary>
+    /// The byte budgets the app asked for, in order.
+    /// </summary>
+    public IReadOnlyList<long> RequestedBudgets => [..this.requestedBudgets];
+
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        if (request.RequestUri!.AbsolutePath is not "/image/prepare-for-model")
+            return new HttpResponseMessage(HttpStatusCode.NotFound);
+
+        using (var body = JsonDocument.Parse(await request.Content!.ReadAsStringAsync(cancellationToken)))
+        {
+            this.requestedPaths.Enqueue(body.RootElement.GetProperty("path").GetString() ?? string.Empty);
+            this.requestedBudgets.Enqueue(body.RootElement.GetProperty("max_bytes").GetInt64());
+        }
+
+        var answer = JsonSerializer.Serialize(new { error_code = this.ErrorCode, base64 = IMAGE_BASE64, mime_type = this.MimeType, width = 1, height = 1, was_resized = false });
+        return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(answer, Encoding.UTF8, "application/json") };
+    }
+}

app/Tests/Tools/ModelImageFormatTests.cs

@@ -0,0 +1,46 @@
+using AIStudio.Chat;
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which image formats may be attached for an AI model.
+/// </summary>
+/// <remarks>
+/// The runtime decides which formats it can prepare for a model, see prepare_image_for_model in
+/// runtime/src/image.rs. An image the app lets through but the runtime cannot prepare would only
+/// fail once the message is sent; one the app refuses although the runtime could prepare it would
+/// be refused for nothing.
+/// </remarks>
+[TestFixture]
+public sealed class ModelImageFormatTests
+{
+    /// <summary>
+    /// The extensions model_input_format in runtime/src/image.rs accepts.
+    /// </summary>
+    private static readonly string[] RUNTIME_MODEL_FORMATS = ["jpg", "jpeg", "png", "webp", "gif", "bmp", "tif", "tiff"];
+
+    [Test]
+    public void EveryFormatTheRuntimePreparesMayBeAttached()
+    {
+        Assert.That(FileTypes.MODEL_IMAGE.FlattenExtensions(), Is.EquivalentTo(RUNTIME_MODEL_FORMATS));
+    }
+
+    [Test]
+    public void OnlyFormatsWithoutADecoderAreRefused()
+    {
+        var refused = FileTypes.IMAGE.FlattenExtensions().Except(FileTypes.MODEL_IMAGE.FlattenExtensions());
+
+        Assert.That(refused, Is.EquivalentTo(new[] { "svg", "heic", "avif" }));
+    }
+
+    [Test]
+    public void ATifFileIsAnImage()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(FileAttachment.FromPath("/tmp/scan.tif").IsImage, Is.True);
+            Assert.That(FileTypes.IsAllowedPath("/tmp/scan.tif", FileTypes.MODEL_IMAGE), Is.True);
+        });
+    }
+}

app/Tests/Tools/ModelImageServiceTests.cs

@@ -0,0 +1,132 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks when the image preparation asks the runtime again, and when it answers from its cache.
+/// </summary>
+/// <remarks>
+/// Every message prepares all images of the conversation anew. Asking the runtime each time would
+/// decode every photo again; answering from the cache after the file changed would send the old
+/// picture. The runtime is played in its wire format, so these tests also fail as soon as the app
+/// stops understanding what the runtime sends.
+/// </remarks>
+[TestFixture]
+public sealed class ModelImageServiceTests
+{
+    private string imagePath = string.Empty;
+    private FakeImageRuntime runtime = null!;
+    private RustService rustService = null!;
+    private ModelImageService service = null!;
+
+    [SetUp]
+    public void PrepareService()
+    {
+        this.imagePath = Path.Combine(Path.GetTempPath(), $"model-image-{Guid.NewGuid():N}.png");
+        File.WriteAllBytes(this.imagePath, [1, 2, 3]);
+
+        this.runtime = new FakeImageRuntime();
+        this.rustService = new RustService(this.runtime);
+        this.service = new ModelImageService(this.rustService, NullLogger<ModelImageService>.Instance);
+    }
+
+    [TearDown]
+    public void CleanUp()
+    {
+        this.service.Dispose();
+        this.rustService.Dispose();
+        this.runtime.Dispose();
+        File.Delete(this.imagePath);
+    }
+
+    [Test]
+    public async Task TheSameFileForTheSameProviderIsPreparedOnce()
+    {
+        var first = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+        var second = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(first.IsPrepared, Is.True);
+            Assert.That(first.MimeType, Is.EqualTo("image/jpeg"));
+            Assert.That(second, Is.SameAs(first));
+            Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(1), "The second message decoded the photo again.");
+        });
+    }
+
+    [Test]
+    public async Task AChangedFileIsPreparedAgain()
+    {
+        await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+        await File.WriteAllBytesAsync(this.imagePath, [1, 2, 3, 4]);
+        await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(2), "The model would get the picture as it was before.");
+    }
+
+    [Test]
+    public async Task AnotherBudgetIsPreparedAgainWithThatBudget()
+    {
+        await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+        await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.OPEN_AI));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(this.runtime.RequestedBudgets, Is.EqualTo(new[] { 3_932_160, IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES }));
+            Assert.That(this.runtime.RequestedPaths, Is.All.EqualTo(new FileInfo(this.imagePath).FullName));
+        });
+    }
+
+    [Test]
+    public async Task AFailureIsTriedAgainWithTheNextMessage()
+    {
+        this.runtime.ErrorCode = "NOT_DECODABLE";
+        var failed = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        this.runtime.ErrorCode = "NONE";
+        var prepared = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(failed.ErrorCode, Is.EqualTo(ModelImageErrorCode.NOT_DECODABLE));
+            Assert.That(failed.IsPrepared, Is.False);
+            Assert.That(prepared.IsPrepared, Is.True);
+            Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(2), "A failure must not stick to the image.");
+        });
+    }
+
+    [Test]
+    public async Task ACodeFromANewerRuntimeNeverCountsAsAPreparedImage()
+    {
+        this.runtime.ErrorCode = "SOMETHING_NEW";
+        var image = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(image.ErrorCode, Is.EqualTo(ModelImageErrorCode.UNKNOWN));
+            Assert.That(image.IsPrepared, Is.False, "The base64 of the answer must not go to the model when the code says nothing we know.");
+        });
+    }
+
+    [Test]
+    public async Task AMissingFileDoesNotReachTheRuntime()
+    {
+        File.Delete(this.imagePath);
+        var image = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(image.ErrorCode, Is.EqualTo(ModelImageErrorCode.FILE_NOT_FOUND));
+            Assert.That(this.runtime.RequestedPaths, Is.Empty);
+        });
+    }
+
+    private FileAttachmentImage Image => new("picture.png", this.imagePath, 3);
+
+    private static AIStudio.Settings.Provider ProviderOf(LLMProviders llmProvider) => new(0, "test", "Test", llmProvider, new Model("model", null));
+}

runtime/Cargo.toml

@@ -77,7 +77,7 @@ qdrant-edge = "0.8.0"
 regex = "1.13.1"
 aho-corasick = "1.1.5"
 toml = "1.1.4"
-image = { version = "0.25.10", default-features = false, features = ["jpeg", "png", "webp"] }
+image = { version = "0.25.10", default-features = false, features = ["jpeg", "png", "webp", "gif", "bmp", "tiff"] }
 tokenizers = "0.23.1"
 
 [dev-dependencies]

runtime/src/image.rs

@@ -3,6 +3,9 @@
 //! This module is deliberately free of any feature-specific behavior so that every part of
 //! AI Studio that needs an embeddable image can use it. The size policy is a single maximum edge
 //! length; callers that want the original bytes pass `optimize = false`.
+//!
+//! Images for AI models take a route of their own, see [`prepare_image_for_model`]: they always
+//! leave as JPEG or PNG and have to fit into the byte budget of the provider.
 
 use std::io::Cursor;
 use std::path::Path;
@@ -12,8 +15,10 @@ use axum::http::StatusCode;
 use base64::{Engine as _, engine::general_purpose};
 use image::codecs::jpeg::JpegEncoder;
 use image::imageops::FilterType;
-use image::{DynamicImage, ImageFormat, ImageReader};
+use image::{DynamicImage, ImageDecoder, ImageError, ImageFormat, ImageReader, Limits, Rgb, RgbImage};
+use log::{info, warn};
 use serde::{Deserialize, Serialize};
+use tokio::sync::Semaphore;
 
 /// The longest edge an optimized image may have. Larger images are scaled down proportionally.
 const MAX_EDGE_PIXELS: u32 = 2_560;
@@ -86,29 +91,14 @@ fn prepare_image_sync(
     }
 
     let format = supported_format(path)?;
-    let reader = ImageReader::open(path)
-        .and_then(|reader| reader.with_guessed_format())
-        .map_err(|error| {
-            (
-                StatusCode::BAD_REQUEST,
-                format!("The image could not be opened: {error}"),
-            )
-        })?;
-        
-    if reader.format() != Some(format) {
-        return Err((
-            StatusCode::BAD_REQUEST,
-            "The image content does not match its file extension.".to_string(),
-        ));
-    }
 
-    let decoded = reader.decode().map_err(|error| {
-        (
-            StatusCode::BAD_REQUEST,
-            format!("The image could not be decoded: {error}"),
-        )
-    })?;
-    
+    //
+    // Upright as the photo was taken: re-encoding drops the EXIF data which told a viewer how to
+    // turn it. The original bytes keep that note, and browsers apply it, so the dimensions we
+    // report fit both ways:
+    //
+    let decoded = decode_oriented(path, format).map_err(|failure| (StatusCode::BAD_REQUEST, failure.message))?;
+
     let original_width = decoded.width();
     let original_height = decoded.height();
     let should_resize = request.optimize && original_width.max(original_height) > MAX_EDGE_PIXELS;
@@ -179,9 +169,16 @@ fn supported_format(path: &Path) -> Result<ImageFormat, (StatusCode, String)> {
 ///
 /// The aspect ratio is preserved, and both edges stay at least one pixel wide.
 fn resize_to_max_edge(image: DynamicImage) -> DynamicImage {
+    resize_to_longest_edge(image, MAX_EDGE_PIXELS)
+}
+
+/// Scales an image so that its longest edge equals the given length.
+///
+/// The aspect ratio is preserved, and both edges stay at least one pixel wide.
+fn resize_to_longest_edge(image: DynamicImage, longest_edge: u32) -> DynamicImage {
     let width = image.width();
     let height = image.height();
-    let scale = MAX_EDGE_PIXELS as f64 / width.max(height) as f64;
+    let scale = longest_edge as f64 / width.max(height) as f64;
     let target_width = (width as f64 * scale).round().max(1.0) as u32;
     let target_height = (height as f64 * scale).round().max(1.0) as u32;
     image.resize_exact(target_width, target_height, FilterType::Lanczos3)
@@ -218,6 +215,426 @@ fn encode(image: &DynamicImage, format: ImageFormat) -> Result<Vec<u8>, (StatusC
     Ok(bytes)
 }
 
+//
+// Preparation for AI models
+//
+
+/// The JPEG qualities tried, in this order, while an image does not fit into its byte budget.
+///
+/// Below the last one, compression artifacts start to blur small text such as the labels of a
+/// diagram. A smaller image keeps such text more legible than a still lower quality does, which
+/// is why the budget shrinks the image instead from there on.
+const BUDGET_JPEG_QUALITIES: [u8; 3] = [JPEG_QUALITY, 70, 55];
+
+/// How much the longest edge shrinks in each round, once no quality fits into the budget.
+const BUDGET_EDGE_FACTOR: f64 = 0.75;
+
+/// The longest edge the budget may shrink an image to. Models no longer read the details of a
+/// photo or a document below that, so such an image is refused instead of sent unreadable.
+const MIN_BUDGET_EDGE_PIXELS: u32 = 1_024;
+
+/// How many images are prepared for models at the same time.
+///
+/// A request with several photos would otherwise decode all of them at once, and every decoded
+/// photo holds a pixel buffer of up to the decoder's limit of 512 MiB. That is too much for small
+/// devices such as a Raspberry Pi, while two at a time still keep a multi-core computer busy.
+const MAX_PARALLEL_MODEL_PREPARATIONS: usize = 2;
+
+/// Lets no more than [`MAX_PARALLEL_MODEL_PREPARATIONS`] images be prepared at once.
+static MODEL_PREPARATION_SLOTS: Semaphore = Semaphore::const_new(MAX_PARALLEL_MODEL_PREPARATIONS);
+
+/// Why an image could not be prepared, so the .NET app can tell the user in their language.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub enum ImageErrorCode {
+    /// The image was prepared.
+    None,
+
+    /// The path is not absolute, or no file exists there.
+    FileNotFound,
+
+    /// The file exists, but could not be opened, e.g., for lack of permission.
+    FileNotReadable,
+
+    /// AI Studio cannot decode this image format, e.g., HEIC, AVIF, or SVG.
+    UnsupportedFormat,
+
+    /// The content of the file does not match its extension.
+    ContentMismatch,
+
+    /// The file claims the right format but is damaged or uses a variant we cannot decode.
+    NotDecodable,
+
+    /// Decoding the image would need more memory than the decoder allows.
+    TooManyPixels,
+
+    /// Not even the smallest image and lowest quality we accept fit into the byte budget.
+    BudgetNotReachable,
+
+    /// Any failure which does not carry a code of its own, e.g., a failed encoder.
+    Internal,
+}
+
+/// A failed preparation: the code for the app and a message for the log.
+#[derive(Debug)]
+struct ImageFailure {
+    code: ImageErrorCode,
+    message: String,
+}
+
+impl ImageFailure {
+    fn new(code: ImageErrorCode, message: impl Into<String>) -> Self {
+        Self { code, message: message.into() }
+    }
+}
+
+/// The request to prepare one local image file for an AI model.
+#[derive(Debug, Deserialize)]
+pub struct PrepareModelImageRequest {
+    /// The absolute path of the image file to read.
+    path: String,
+
+    /// The largest number of bytes the encoded image may have, before base64.
+    max_bytes: u64,
+}
+
+/// The image as it goes to the model, or why it cannot go.
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub struct PrepareModelImageResponse {
+    /// `None` when the image was prepared; then all other fields describe it.
+    error_code: ImageErrorCode,
+
+    /// The encoded image as base64, without a `data:` prefix. The MIME type travels next to it.
+    base64: String,
+
+    /// Either `image/jpeg` or `image/png`, describing the encoded bytes.
+    mime_type: String,
+
+    /// The width of the prepared image in pixels.
+    width: u32,
+
+    /// The height of the prepared image in pixels.
+    height: u32,
+
+    /// Whether the image was scaled down, by the size policy or to fit into the byte budget.
+    was_resized: bool,
+}
+
+impl PrepareModelImageResponse {
+    fn failure(code: ImageErrorCode) -> Self {
+        Self {
+            error_code: code,
+            base64: String::new(),
+            mime_type: String::new(),
+            width: 0,
+            height: 0,
+            was_resized: false,
+        }
+    }
+}
+
+/// The encoding an image leaves in for a model.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ModelEncoding {
+    Jpeg,
+    Png,
+}
+
+/// An image encoded for a model, before base64.
+struct EncodedModelImage {
+    bytes: Vec<u8>,
+    encoding: ModelEncoding,
+
+    /// The JPEG quality which fit into the budget; PNG has none.
+    jpeg_quality: Option<u8>,
+
+    width: u32,
+    height: u32,
+    was_resized: bool,
+}
+
+/// Prepares one local image for an AI model.
+///
+/// Every image is decoded, turned upright according to its EXIF orientation, scaled down to
+/// [`MAX_EDGE_PIXELS`], and encoded anew as JPEG or PNG, the two formats every provider and every
+/// self-hosted engine accepts. Encoding anew also drops the metadata of the file, among them the
+/// location where a photo was taken. When the result exceeds the byte budget, the quality and then
+/// the size go down until it fits.
+///
+/// The answer is always a success at the HTTP level; `error_code` tells whether the image was
+/// prepared. Decoding runs on a blocking worker, and only a few images at a time, see
+/// [`MAX_PARALLEL_MODEL_PREPARATIONS`].
+pub async fn prepare_image_for_model(
+    Json(request): Json<PrepareModelImageRequest>,
+) -> Json<PrepareModelImageResponse> {
+    //
+    // The semaphore is never closed, so acquiring it cannot fail. Should it fail anyway, the
+    // image is prepared without a slot rather than not at all:
+    //
+    let _slot = MODEL_PREPARATION_SLOTS.acquire().await.ok();
+    let response = tokio::task::spawn_blocking(move || prepare_image_for_model_sync(&request))
+        .await
+        .unwrap_or_else(|error| {
+            warn!("The image worker for an AI model failed: {error}");
+            PrepareModelImageResponse::failure(ImageErrorCode::Internal)
+        });
+
+    Json(response)
+}
+
+/// Performs the blocking part of [`prepare_image_for_model`].
+fn prepare_image_for_model_sync(request: &PrepareModelImageRequest) -> PrepareModelImageResponse {
+    match prepare_for_model(Path::new(&request.path), request.max_bytes) {
+        Ok(encoded) => PrepareModelImageResponse {
+            error_code: ImageErrorCode::None,
+            base64: general_purpose::STANDARD.encode(&encoded.bytes),
+            mime_type: match encoded.encoding {
+                ModelEncoding::Jpeg => "image/jpeg",
+                ModelEncoding::Png => "image/png",
+            }
+            .to_string(),
+            width: encoded.width,
+            height: encoded.height,
+            was_resized: encoded.was_resized,
+        },
+
+        Err(failure) => {
+            warn!("Preparing the image '{}' for an AI model failed ({:?}): {}", request.path, failure.code, failure.message);
+            PrepareModelImageResponse::failure(failure.code)
+        }
+    }
+}
+
+/// Decodes, orients, scales, and encodes one image for a model.
+fn prepare_for_model(path: &Path, max_bytes: u64) -> Result<EncodedModelImage, ImageFailure> {
+    if !path.is_absolute() || !path.is_file() {
+        return Err(ImageFailure::new(ImageErrorCode::FileNotFound, "The image path is not an accessible absolute file path."));
+    }
+
+    let format = model_input_format(path)?;
+    let source_bytes = std::fs::metadata(path).map(|metadata| metadata.len()).unwrap_or(0);
+    let decoded = decode_oriented(path, format)?;
+    let (source_width, source_height) = (decoded.width(), decoded.height());
+    let was_resized = source_width.max(source_height) > MAX_EDGE_PIXELS;
+    let image = if was_resized {
+        resize_to_max_edge(decoded)
+    } else {
+        decoded
+    };
+
+    let encoding = model_encoding(format, &image);
+    let encoded = fit_into_budget(image, encoding, max_bytes, was_resized)?;
+    let encoded_bytes = encoded.bytes.len() as u64;
+    info!(
+        "Image prepared for model: path='{}', source={format:?} {source_width}x{source_height} {}, result={:?} {}x{} {} ({} of the original), quality={}, budget={}, resized={}",
+        path.display(),
+        megabytes(source_bytes),
+        encoded.encoding,
+        encoded.width,
+        encoded.height,
+        megabytes(encoded_bytes),
+        percentage(encoded_bytes, source_bytes),
+        encoded.jpeg_quality.map_or_else(|| "lossless".to_string(), |quality| quality.to_string()),
+        megabytes(max_bytes),
+        encoded.was_resized,
+    );
+
+    Ok(encoded)
+}
+
+/// Writes a number of bytes as megabytes, for the log.
+fn megabytes(bytes: u64) -> String {
+    format!("{:.2} MB", bytes as f64 / 1_000_000.0)
+}
+
+/// Writes how large a part is compared to the whole, for the log.
+fn percentage(part: u64, whole: u64) -> String {
+    if whole == 0 {
+        return "n/a".to_string();
+    }
+
+    format!("{:.0} %", part as f64 * 100.0 / whole as f64)
+}
+
+/// Maps a file extension to the image format we expect for a model input.
+///
+/// The result is only the expected format; [`decode_oriented`] still verifies it against the
+/// actual file content.
+fn model_input_format(path: &Path) -> Result<ImageFormat, ImageFailure> {
+    match path
+        .extension()
+        .and_then(|extension| extension.to_str())
+        .map(str::to_ascii_lowercase)
+        .as_deref()
+    {
+        Some("jpg" | "jpeg") => Ok(ImageFormat::Jpeg),
+        Some("png") => Ok(ImageFormat::Png),
+        Some("webp") => Ok(ImageFormat::WebP),
+        Some("gif") => Ok(ImageFormat::Gif),
+        Some("bmp") => Ok(ImageFormat::Bmp),
+        Some("tif" | "tiff") => Ok(ImageFormat::Tiff),
+
+        _ => Err(ImageFailure::new(ImageErrorCode::UnsupportedFormat, "Images for AI models must be JPEG, PNG, WebP, GIF, BMP, or TIFF files.")),
+    }
+}
+
+/// Decodes an image of the expected format and turns it upright according to its EXIF orientation.
+///
+/// Phones store a photo as the sensor saw it and only note in the EXIF data how to turn it.
+/// Without applying that note, a photo taken upright would lie on its side once encoded anew,
+/// because the new file no longer carries the note. The content has to match the expected format,
+/// so that a file which merely claims to be an image is not decoded as one.
+fn decode_oriented(path: &Path, format: ImageFormat) -> Result<DynamicImage, ImageFailure> {
+    let reader = ImageReader::open(path)
+        .and_then(|reader| reader.with_guessed_format())
+        .map_err(|error| ImageFailure::new(ImageErrorCode::FileNotReadable, format!("The image could not be opened: {error}")))?;
+
+    if reader.format() != Some(format) {
+        return Err(ImageFailure::new(ImageErrorCode::ContentMismatch, "The image content does not match its file extension."));
+    }
+
+    let mut decoder = reader.into_decoder().map_err(decoding_failure)?;
+    let orientation = decoder.orientation().map_err(decoding_failure)?;
+
+    //
+    // The same check ImageReader::decode makes: the pixel buffer must fit into the decoder's
+    // memory limit. into_decoder only checks the dimensions.
+    //
+    Limits::default().reserve(decoder.total_bytes()).map_err(decoding_failure)?;
+
+    let mut image = DynamicImage::from_decoder(decoder).map_err(decoding_failure)?;
+    image.apply_orientation(orientation);
+    Ok(image)
+}
+
+/// Tells a decoder which hit its memory limit apart from any other decoding failure.
+fn decoding_failure(error: ImageError) -> ImageFailure {
+    match error {
+        ImageError::Limits(_) => ImageFailure::new(ImageErrorCode::TooManyPixels, format!("The image is too large to decode: {error}")),
+        _ => ImageFailure::new(ImageErrorCode::NotDecodable, format!("The image could not be decoded: {error}")),
+    }
+}
+
+/// Chooses the encoding of an image for a model.
+///
+/// JPEG and PNG keep their format: PNG holds screenshots and diagrams without loss, and JPEG
+/// holds photos. Any other format becomes PNG when it uses transparency, which JPEG cannot hold,
+/// and JPEG otherwise.
+fn model_encoding(source: ImageFormat, image: &DynamicImage) -> ModelEncoding {
+    match source {
+        ImageFormat::Jpeg => ModelEncoding::Jpeg,
+        ImageFormat::Png => ModelEncoding::Png,
+        _ if uses_transparency(image) => ModelEncoding::Png,
+        _ => ModelEncoding::Jpeg,
+    }
+}
+
+/// Whether any pixel of the image is not fully opaque.
+///
+/// An alpha channel alone says nothing: GIF decodes into RGBA, and many files carry a channel in
+/// which every pixel is opaque.
+fn uses_transparency(image: &DynamicImage) -> bool {
+    match image {
+        DynamicImage::ImageLumaA8(buffer) => buffer.pixels().any(|pixel| pixel.0[1] < u8::MAX),
+        DynamicImage::ImageRgba8(buffer) => buffer.pixels().any(|pixel| pixel.0[3] < u8::MAX),
+        DynamicImage::ImageLumaA16(buffer) => buffer.pixels().any(|pixel| pixel.0[1] < u16::MAX),
+        DynamicImage::ImageRgba16(buffer) => buffer.pixels().any(|pixel| pixel.0[3] < u16::MAX),
+        DynamicImage::ImageRgba32F(buffer) => buffer.pixels().any(|pixel| pixel.0[3] < 1.0),
+        _ => image.color().has_alpha() && image.to_rgba8().pixels().any(|pixel| pixel.0[3] < u8::MAX),
+    }
+}
+
+/// Encodes the image and lowers quality and size until it fits into the byte budget.
+fn fit_into_budget(image: DynamicImage, encoding: ModelEncoding, max_bytes: u64, was_resized: bool) -> Result<EncodedModelImage, ImageFailure> {
+    if encoding == ModelEncoding::Png {
+        let png = to_png_pixels(&image);
+        let bytes = encode_model_image(&png, ModelEncoding::Png, JPEG_QUALITY)?;
+        if bytes.len() as u64 <= max_bytes {
+            return Ok(EncodedModelImage { bytes, encoding, jpeg_quality: None, width: png.width(), height: png.height(), was_resized });
+        }
+    }
+
+    //
+    // From here on, the image goes as JPEG. A PNG which does not fit loses its transparency,
+    // which is laid onto white, the background most documents and chats show it on:
+    //
+    let mut current = to_jpeg_pixels(&image);
+    drop(image);
+
+    let mut was_resized = was_resized;
+    loop {
+        for quality in BUDGET_JPEG_QUALITIES {
+            let bytes = encode_model_image(&current, ModelEncoding::Jpeg, quality)?;
+            if bytes.len() as u64 <= max_bytes {
+                return Ok(EncodedModelImage { bytes, encoding: ModelEncoding::Jpeg, jpeg_quality: Some(quality), width: current.width(), height: current.height(), was_resized });
+            }
+        }
+
+        let longest_edge = current.width().max(current.height());
+        if longest_edge <= MIN_BUDGET_EDGE_PIXELS {
+            return Err(ImageFailure::new(ImageErrorCode::BudgetNotReachable, format!("The image does not fit into {max_bytes} bytes, not even with {longest_edge} pixels on its longest edge.")));
+        }
+
+        let next_edge = ((longest_edge as f64 * BUDGET_EDGE_FACTOR).round() as u32).max(MIN_BUDGET_EDGE_PIXELS);
+        current = resize_to_longest_edge(current, next_edge);
+        was_resized = true;
+    }
+}
+
+/// Brings an image into 8 bits per channel for PNG, keeping the transparency where it is used.
+fn to_png_pixels(image: &DynamicImage) -> DynamicImage {
+    let is_gray = !image.color().has_color();
+    match (is_gray, uses_transparency(image)) {
+        (true, true) => DynamicImage::ImageLumaA8(image.to_luma_alpha8()),
+        (true, false) => DynamicImage::ImageLuma8(image.to_luma8()),
+        (false, true) => DynamicImage::ImageRgba8(image.to_rgba8()),
+        (false, false) => DynamicImage::ImageRgb8(image.to_rgb8()),
+    }
+}
+
+/// Brings an image into 8 bits per channel for JPEG, which has no alpha channel.
+fn to_jpeg_pixels(image: &DynamicImage) -> DynamicImage {
+    if uses_transparency(image) {
+        return flatten_onto_white(image);
+    }
+
+    if image.color().has_color() {
+        DynamicImage::ImageRgb8(image.to_rgb8())
+    } else {
+        DynamicImage::ImageLuma8(image.to_luma8())
+    }
+}
+
+/// Lays a partly transparent image onto a white background.
+///
+/// Dropping the alpha channel alone would keep whatever color the transparent pixels happen to
+/// hold, which is often black.
+fn flatten_onto_white(image: &DynamicImage) -> DynamicImage {
+    let rgba = image.to_rgba8();
+    let mut flattened = RgbImage::new(rgba.width(), rgba.height());
+    for (source, target) in rgba.pixels().zip(flattened.pixels_mut()) {
+        let [red, green, blue, alpha] = source.0;
+        let alpha = u16::from(alpha);
+        let blend = |channel: u8| ((u16::from(channel) * alpha + 255 * (255 - alpha) + 127) / 255) as u8;
+        *target = Rgb([blend(red), blend(green), blend(blue)]);
+    }
+
+    DynamicImage::ImageRgb8(flattened)
+}
+
+/// Encodes an image whose pixels already suit the encoding.
+fn encode_model_image(image: &DynamicImage, encoding: ModelEncoding, jpeg_quality: u8) -> Result<Vec<u8>, ImageFailure> {
+    let mut bytes = Vec::new();
+    let result = match encoding {
+        ModelEncoding::Jpeg => JpegEncoder::new_with_quality(&mut bytes, jpeg_quality).encode_image(image),
+        ModelEncoding::Png => image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png),
+    };
+
+    result.map_err(|error| ImageFailure::new(ImageErrorCode::Internal, format!("The image could not be encoded: {error}")))?;
+    Ok(bytes)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -391,4 +808,270 @@ mod tests {
         assert_eq!(issue.0, StatusCode::BAD_REQUEST);
         std::fs::remove_file(path).unwrap();
     }
+
+    /// Noise which compresses badly, the same on every run.
+    fn noise_image(width: u32, height: u32) -> RgbImage {
+        let mut state: u32 = 0x1234_5678;
+        RgbImage::from_fn(width, height, |_, _| {
+            let mut channel = || {
+                state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
+                (state >> 24) as u8
+            };
+
+            Rgb([channel(), channel(), channel()])
+        })
+    }
+
+    /// Writes a black JPEG whose EXIF data says how to turn it.
+    ///
+    /// The encoder writes no EXIF data, so the segment is put in by hand, right after the start
+    /// of the image: a big-endian TIFF header with one entry, the orientation.
+    fn write_jpeg_with_orientation(path: &Path, width: u32, height: u32, orientation: u8) {
+        let mut jpeg = Vec::new();
+        JpegEncoder::new_with_quality(&mut jpeg, JPEG_QUALITY)
+            .encode_image(&DynamicImage::new_rgb8(width, height))
+            .unwrap();
+
+        let exif: [u8; 36] = [
+            0xFF, 0xE1, 0x00, 0x22, // APP1 and its length, which counts itself but not the marker
+            b'E', b'x', b'i', b'f', 0x00, 0x00,
+            b'M', b'M', 0x00, 0x2A, 0x00, 0x00, 0x00, 0x08, // big-endian, first IFD at offset 8
+            0x00, 0x01, // one entry
+            0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, orientation, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, // no further IFD
+        ];
+
+        let mut file = jpeg[..2].to_vec();
+        file.extend_from_slice(&exif);
+        file.extend_from_slice(&jpeg[2..]);
+        std::fs::write(path, file).unwrap();
+    }
+
+    #[test]
+    fn model_preparation_scales_down_only_above_the_maximum_edge() {
+        let larger = temporary_image_path("png");
+        DynamicImage::new_rgb8(MAX_EDGE_PIXELS + 1, 1).save_with_format(&larger, ImageFormat::Png).unwrap();
+        let prepared = prepare_for_model(&larger, u64::MAX).unwrap();
+        assert_eq!((prepared.width, prepared.height), (MAX_EDGE_PIXELS, 1));
+        assert!(prepared.was_resized);
+        std::fs::remove_file(larger).unwrap();
+
+        let at_the_edge = temporary_image_path("png");
+        DynamicImage::new_rgb8(MAX_EDGE_PIXELS, 2).save_with_format(&at_the_edge, ImageFormat::Png).unwrap();
+        let prepared = prepare_for_model(&at_the_edge, u64::MAX).unwrap();
+        assert_eq!((prepared.width, prepared.height), (MAX_EDGE_PIXELS, 2));
+        assert!(!prepared.was_resized);
+        std::fs::remove_file(at_the_edge).unwrap();
+    }
+
+    #[test]
+    fn model_preparation_keeps_jpeg_and_png_and_turns_other_formats_into_one_of_them() {
+        let opaque = DynamicImage::ImageRgb8(RgbImage::from_pixel(9, 7, Rgb([200, 10, 10])));
+        let mut transparent = opaque.to_rgba8();
+        transparent.get_pixel_mut(0, 0).0[3] = 0;
+        let transparent = DynamicImage::ImageRgba8(transparent);
+
+        for (extension, source_format, image, expected) in [
+            ("jpg", ImageFormat::Jpeg, &opaque, ImageFormat::Jpeg),
+            ("png", ImageFormat::Png, &opaque, ImageFormat::Png),
+            ("webp", ImageFormat::WebP, &opaque, ImageFormat::Jpeg),
+            ("webp", ImageFormat::WebP, &transparent, ImageFormat::Png),
+            ("gif", ImageFormat::Gif, &opaque, ImageFormat::Jpeg),
+            ("gif", ImageFormat::Gif, &transparent, ImageFormat::Png),
+            ("bmp", ImageFormat::Bmp, &opaque, ImageFormat::Jpeg),
+            ("tif", ImageFormat::Tiff, &opaque, ImageFormat::Jpeg),
+            ("tiff", ImageFormat::Tiff, &transparent, ImageFormat::Png),
+        ] {
+            let path = temporary_image_path(extension);
+            image.save_with_format(&path, source_format).unwrap();
+            let prepared = prepare_for_model(&path, u64::MAX).unwrap();
+            assert_eq!(image::guess_format(&prepared.bytes).unwrap(), expected, "{extension} with alpha={}", image.color().has_alpha());
+            assert_eq!(prepared.encoding == ModelEncoding::Png, expected == ImageFormat::Png, "{extension}");
+            assert_eq!((prepared.width, prepared.height), (9, 7), "{extension}");
+            std::fs::remove_file(path).unwrap();
+        }
+    }
+
+    #[test]
+    fn model_preparation_refuses_formats_it_cannot_decode() {
+        for extension in ["heic", "heif", "avif", "svg", "ico"] {
+            let path = temporary_image_path(extension);
+            std::fs::write(&path, b"whatever").unwrap();
+            let failure = prepare_for_model(&path, u64::MAX).err().unwrap();
+            assert_eq!(failure.code, ImageErrorCode::UnsupportedFormat, "{extension}");
+            std::fs::remove_file(path).unwrap();
+        }
+    }
+
+    #[test]
+    fn model_preparation_tells_missing_mismatched_and_damaged_files_apart() {
+        let missing = prepare_for_model(&temporary_image_path("png"), u64::MAX).err().unwrap();
+        assert_eq!(missing.code, ImageErrorCode::FileNotFound);
+
+        let relative = prepare_for_model(Path::new("picture.png"), u64::MAX).err().unwrap();
+        assert_eq!(relative.code, ImageErrorCode::FileNotFound);
+
+        let jpeg_named_png = temporary_image_path("png");
+        DynamicImage::new_rgb8(4, 3).save_with_format(&jpeg_named_png, ImageFormat::Jpeg).unwrap();
+        let mismatch = prepare_for_model(&jpeg_named_png, u64::MAX).err().unwrap();
+        assert_eq!(mismatch.code, ImageErrorCode::ContentMismatch);
+        std::fs::remove_file(jpeg_named_png).unwrap();
+
+        let damaged = temporary_image_path("png");
+        std::fs::write(&damaged, b"not an image").unwrap();
+        let failure = prepare_for_model(&damaged, u64::MAX).err().unwrap();
+        assert_eq!(failure.code, ImageErrorCode::NotDecodable);
+        std::fs::remove_file(damaged).unwrap();
+    }
+
+    #[test]
+    fn model_preparation_turns_photos_upright() {
+        let path = temporary_image_path("jpg");
+        write_jpeg_with_orientation(&path, 20, 10, 6);
+        let prepared = prepare_for_model(&path, u64::MAX).unwrap();
+        assert_eq!((prepared.width, prepared.height), (10, 20), "Orientation 6 turns the image by 90 degrees.");
+        std::fs::remove_file(path).unwrap();
+    }
+
+    #[test]
+    fn optimization_turns_photos_upright() {
+        let path = temporary_image_path("jpg");
+        write_jpeg_with_orientation(&path, 20, 10, 6);
+        let response = prepare_image_sync(&PrepareImageRequest {
+            path: path.to_string_lossy().into_owned(),
+            optimize: true,
+        })
+        .unwrap();
+
+        let bytes = general_purpose::STANDARD
+            .decode(response.data_url.split_once(',').unwrap().1)
+            .unwrap();
+        let encoded = image::load_from_memory(&bytes).unwrap();
+        assert_eq!((response.width, response.height), (10, 20), "Orientation 6 turns the image by 90 degrees.");
+        assert_eq!((encoded.width(), encoded.height()), (10, 20), "The encoded image carries no orientation of its own any longer.");
+        std::fs::remove_file(path).unwrap();
+    }
+
+    #[test]
+    fn model_preparation_gives_the_same_bytes_for_the_same_image() {
+        let path = temporary_image_path("jpg");
+        DynamicImage::ImageRgb8(noise_image(64, 48)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+        let first = prepare_for_model(&path, u64::MAX).unwrap();
+        let second = prepare_for_model(&path, u64::MAX).unwrap();
+        assert_eq!(first.bytes, second.bytes);
+        std::fs::remove_file(path).unwrap();
+    }
+
+    #[test]
+    fn budget_lowers_the_quality_before_the_size() {
+        let path = temporary_image_path("jpg");
+        DynamicImage::ImageRgb8(noise_image(320, 200)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+
+        //
+        // The sizes the budget will meet, measured on the very pixels it works with:
+        //
+        let decoded = to_jpeg_pixels(&decode_oriented(&path, ImageFormat::Jpeg).unwrap());
+        let at_first_quality = encode_model_image(&decoded, ModelEncoding::Jpeg, BUDGET_JPEG_QUALITIES[0]).unwrap();
+        let at_second_quality = encode_model_image(&decoded, ModelEncoding::Jpeg, BUDGET_JPEG_QUALITIES[1]).unwrap();
+        assert!(at_second_quality.len() < at_first_quality.len());
+
+        let prepared = prepare_for_model(&path, at_first_quality.len() as u64 - 1).unwrap();
+        assert_eq!(prepared.bytes, at_second_quality);
+        assert_eq!((prepared.width, prepared.height), (320, 200));
+        assert!(!prepared.was_resized);
+        std::fs::remove_file(path).unwrap();
+    }
+
+    #[test]
+    fn budget_shrinks_the_image_once_no_quality_fits() {
+        let path = temporary_image_path("jpg");
+        DynamicImage::ImageRgb8(noise_image(1_200, 600)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+
+        let decoded = to_jpeg_pixels(&decode_oriented(&path, ImageFormat::Jpeg).unwrap());
+        let at_last_quality = encode_model_image(&decoded, ModelEncoding::Jpeg, *BUDGET_JPEG_QUALITIES.last().unwrap()).unwrap();
+        let budget = at_last_quality.len() as u64 - 1;
+
+        let prepared = prepare_for_model(&path, budget).unwrap();
+        assert!(prepared.bytes.len() as u64 <= budget);
+        assert_eq!((prepared.width, prepared.height), (MIN_BUDGET_EDGE_PIXELS, 512));
+        assert!(prepared.was_resized);
+        assert_eq!(prepared.encoding, ModelEncoding::Jpeg);
+        std::fs::remove_file(path).unwrap();
+    }
+
+    #[test]
+    fn budget_lays_a_transparent_png_onto_white_when_it_has_to_become_jpeg() {
+        let mut pixels = DynamicImage::ImageRgb8(noise_image(300, 200)).to_rgba8();
+        for (x, _, pixel) in pixels.enumerate_pixels_mut() {
+            if x < 150 {
+                pixel.0 = [0, 0, 0, 0];
+            }
+        }
+
+        let path = temporary_image_path("png");
+        DynamicImage::ImageRgba8(pixels).save_with_format(&path, ImageFormat::Png).unwrap();
+        let unlimited = prepare_for_model(&path, u64::MAX).unwrap();
+        assert_eq!(unlimited.encoding, ModelEncoding::Png);
+
+        let prepared = prepare_for_model(&path, unlimited.bytes.len() as u64 - 1).unwrap();
+        assert_eq!(prepared.encoding, ModelEncoding::Jpeg);
+
+        let corner = image::load_from_memory(&prepared.bytes).unwrap().to_rgb8().get_pixel(5, 5).0;
+        assert!(corner.iter().all(|&channel| channel > 240), "The transparent half has to be white, not black: {corner:?}");
+        std::fs::remove_file(path).unwrap();
+    }
+
+    #[test]
+    fn budget_reports_the_quality_which_fit() {
+        let path = temporary_image_path("jpg");
+        DynamicImage::ImageRgb8(noise_image(64, 48)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+        assert_eq!(prepare_for_model(&path, u64::MAX).unwrap().jpeg_quality, Some(JPEG_QUALITY));
+        std::fs::remove_file(path).unwrap();
+
+        let path = temporary_image_path("png");
+        DynamicImage::new_rgb8(4, 3).save_with_format(&path, ImageFormat::Png).unwrap();
+        assert_eq!(prepare_for_model(&path, u64::MAX).unwrap().jpeg_quality, None, "PNG has no quality.");
+        std::fs::remove_file(path).unwrap();
+    }
+
+    #[test]
+    fn log_writes_sizes_in_megabytes_and_the_ratio_in_percent() {
+        assert_eq!(megabytes(5_812_345), "5.81 MB");
+        assert_eq!(megabytes(0), "0.00 MB");
+        assert_eq!(percentage(912_000, 5_812_345), "16 %");
+        assert_eq!(percentage(1, 0), "n/a", "An empty source has no ratio.");
+    }
+
+    #[test]
+    fn budget_refuses_an_image_it_cannot_make_fit() {
+        let path = temporary_image_path("jpg");
+        DynamicImage::ImageRgb8(noise_image(64, 48)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+        let failure = prepare_for_model(&path, 100).err().unwrap();
+        assert_eq!(failure.code, ImageErrorCode::BudgetNotReachable);
+        std::fs::remove_file(path).unwrap();
+    }
+
+    #[test]
+    fn serializes_the_model_response_for_the_rust_service_contract() {
+        let failure = serde_json::to_value(PrepareModelImageResponse::failure(ImageErrorCode::BudgetNotReachable)).unwrap();
+        assert_eq!(failure["error_code"], "BUDGET_NOT_REACHABLE");
+        assert_eq!(failure["base64"], "");
+
+        let path = temporary_image_path("png");
+        DynamicImage::new_rgb8(4, 3).save_with_format(&path, ImageFormat::Png).unwrap();
+        let success = serde_json::to_value(prepare_image_for_model_sync(&PrepareModelImageRequest {
+            path: path.to_string_lossy().into_owned(),
+            max_bytes: u64::MAX,
+        }))
+        .unwrap();
+
+        assert_eq!(success["error_code"], "NONE");
+        assert_eq!(success["mime_type"], "image/png");
+        assert_eq!(success["width"], 4);
+        assert_eq!(success["height"], 3);
+        assert_eq!(success["was_resized"], false);
+        assert!(!success["base64"].as_str().unwrap().starts_with("data:"));
+        assert!(success.get("mimeType").is_none());
+        std::fs::remove_file(path).unwrap();
+    }
 }

runtime/src/runtime_api.rs

@@ -86,6 +86,7 @@ fn create_router() -> Router {
         .route("/media/jobs/{id}/events", get(crate::media::get_job_events))
         .route("/media/jobs/{id}", delete(crate::media::cancel_job))
         .route("/image/prepare", post(crate::image::prepare_image))
+        .route("/image/prepare-for-model", post(crate::image::prepare_image_for_model))
         .route("/log/paths", get(crate::log::get_log_paths))
         .route("/log/event", post(crate::log::log_event))
         .route("/tokenizer/count", post(crate::tokenizer::token_count))