Scaled down and converted images before sending them to AI models (#1101)

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -98,18 +98,19 @@ public static class FileTypes
 
     // Media hierarchy
     public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
-        "jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic", "avif");
+        "jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "svg", "webp", "heic", "avif");
 
     /// <summary>
-    /// The image formats we hand to a model as they are.
+    /// The image formats AI Studio can prepare for a model.
     /// </summary>
     /// <remarks>
-    /// The providers accept these, while other formats, such as TIFF or HEIC, would end in an
-    /// error of the provider which tells the user nothing. The default file patterns of the batch
-    /// processing assistant list the same formats, see DataBatchProcessing.DEFAULT_FILE_PATTERNS.
+    /// These are the formats the runtime decodes before it sends an image to a model as JPEG or
+    /// PNG, see prepare_image_for_model in runtime/src/image.rs. Other images, such as HEIC, AVIF,
+    /// or SVG, cannot go to a model at all. The default file patterns of the batch processing
+    /// assistant list the same formats, see DataBatchProcessing.DEFAULT_FILE_PATTERNS.
     /// </remarks>
     public static readonly FileTypeFilter MODEL_IMAGE = FileTypeFilter.Leaf(TB("Image for AI models"),
-        "jpg", "jpeg", "png", "webp", "gif");
+        "jpg", "jpeg", "png", "webp", "gif", "bmp", "tif", "tiff");
 
     /// <summary>
     /// Gets the prototype visual-asset image formats.

app/MindWork AI Studio/Tools/Rust/ModelImageErrorCode.cs

@@ -0,0 +1,63 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Why the Rust runtime could not prepare an image for an AI model.
+/// </summary>
+/// <remarks>
+/// UNKNOWN comes first on purpose. The converter for the enums of the runtime falls back to the
+/// default value for a code it does not know, e.g., from a newer runtime, and such a code must
+/// never read as a prepared image.
+/// </remarks>
+public enum ModelImageErrorCode
+{
+    /// <summary>
+    /// The runtime answered with a code this version does not know, or it could not be asked at all.
+    /// </summary>
+    UNKNOWN,
+
+    /// <summary>
+    /// No failure happened: the image was prepared.
+    /// </summary>
+    NONE,
+
+    //
+    // Codes reported by the Rust runtime:
+    //
+
+    FILE_NOT_FOUND,
+
+    /// <summary>
+    /// The file exists, but could not be opened, e.g., for lack of permission.
+    /// </summary>
+    FILE_NOT_READABLE,
+
+    /// <summary>
+    /// AI Studio cannot decode this image format, e.g., HEIC, AVIF, or SVG.
+    /// </summary>
+    UNSUPPORTED_FORMAT,
+
+    /// <summary>
+    /// The content of the file does not match its file extension.
+    /// </summary>
+    CONTENT_MISMATCH,
+
+    /// <summary>
+    /// The file is damaged or uses a variant of its format which cannot be decoded.
+    /// </summary>
+    NOT_DECODABLE,
+
+    /// <summary>
+    /// Decoding the image would need more memory than the runtime allows.
+    /// </summary>
+    TOO_MANY_PIXELS,
+
+    /// <summary>
+    /// Not even the smallest size and lowest quality the runtime accepts fit into the byte budget.
+    /// </summary>
+    BUDGET_NOT_REACHABLE,
+
+    /// <summary>
+    /// Any failure in the runtime which does not carry a code of its own.
+    /// </summary>
+    INTERNAL,
+}

app/MindWork AI Studio/Tools/Rust/ModelImageErrorCodeExtensions.cs

@@ -0,0 +1,36 @@
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Translates why an image could not be prepared for an AI model into user-facing text.
+/// </summary>
+/// <remarks>
+/// The runtime writes the technical details into the log. The texts here name what the user can
+/// act on, such as converting the image into another format.
+/// </remarks>
+public static class ModelImageErrorCodeExtensions
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ModelImageErrorCodeExtensions).Namespace, nameof(ModelImageErrorCodeExtensions));
+
+    /// <summary>
+    /// Gets the localized message which explains why an image was not sent to the AI.
+    /// </summary>
+    /// <param name="code">Why the image could not be prepared.</param>
+    /// <param name="fileName">The name of the image file, as shown to the user.</param>
+    /// <returns>The localized message.</returns>
+    public static string ToUserMessage(this ModelImageErrorCode code, string fileName) => string.Format(ToUserMessageFormat(code), fileName);
+
+    private static string ToUserMessageFormat(ModelImageErrorCode code) => code switch
+    {
+        ModelImageErrorCode.FILE_NOT_FOUND => TB("The image '{0}' does not exist anymore and was not sent."),
+        ModelImageErrorCode.FILE_NOT_READABLE => TB("The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable."),
+        ModelImageErrorCode.UNSUPPORTED_FORMAT => TB("The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG."),
+        ModelImageErrorCode.CONTENT_MISMATCH => TB("The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension."),
+        ModelImageErrorCode.NOT_DECODABLE => TB("The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely."),
+        ModelImageErrorCode.TOO_MANY_PIXELS => TB("The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first."),
+        ModelImageErrorCode.BUDGET_NOT_REACHABLE => TB("The image '{0}' could not be made small enough for the selected provider, so it was not sent."),
+
+        _ => TB("AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent."),
+    };
+}

app/MindWork AI Studio/Tools/Rust/PreparedModelImage.cs

@@ -0,0 +1,39 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// An image as the Rust runtime prepared it for an AI model, or why it could not.
+/// </summary>
+/// <remarks>
+/// The base64 and its MIME type travel side by side, so a provider which wants a data URL builds
+/// one and a provider which wants them apart, such as Anthropic, never parses one.
+/// </remarks>
+/// <param name="ErrorCode">NONE when the image was prepared; then the other values describe it.</param>
+/// <param name="Base64">The encoded image as base64, without a data URI prefix.</param>
+/// <param name="MimeType">The MIME type of the encoded image: image/jpeg or image/png.</param>
+/// <param name="Width">The width of the prepared image in pixels.</param>
+/// <param name="Height">The height of the prepared image in pixels.</param>
+/// <param name="WasResized">Whether the runtime scaled the image down.</param>
+public sealed record PreparedModelImage(ModelImageErrorCode ErrorCode, string Base64, string MimeType, uint Width, uint Height, bool WasResized)
+{
+    /// <summary>
+    /// Whether the image can go to the model.
+    /// </summary>
+    /// <remarks>
+    /// The image itself has to be there as well: a success without one would become an empty image
+    /// in the request, which providers refuse along with the whole request.
+    /// </remarks>
+    public bool IsPrepared => this.ErrorCode is ModelImageErrorCode.NONE && !string.IsNullOrEmpty(this.Base64);
+
+    /// <summary>
+    /// The image as a data URL, as the OpenAI-compatible APIs take it.
+    /// </summary>
+    /// <returns>The data URL.</returns>
+    public string ToDataUrl() => $"data:{this.MimeType};base64,{this.Base64}";
+
+    /// <summary>
+    /// An image which could not be prepared.
+    /// </summary>
+    /// <param name="errorCode">Why it could not be prepared.</param>
+    /// <returns>The failed image.</returns>
+    public static PreparedModelImage Failed(ModelImageErrorCode errorCode) => new(errorCode, string.Empty, string.Empty, 0, 0, false);
+}

app/MindWork AI Studio/Tools/Services/ModelImageService.cs

@@ -0,0 +1,93 @@
+using AIStudio.Chat;
+using AIStudio.Tools.Rust;
+
+using Microsoft.Extensions.Caching.Memory;
+// ReSharper disable NotAccessedPositionalProperty.Local
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Prepares the images which go to an AI model, and keeps the latest results for a while.
+/// </summary>
+/// <remarks>
+/// Every message builds the whole conversation anew, with every image in it. Without the cache,
+/// each photo of a chat would be decoded and encoded again for each message, which a small device
+/// such as a Raspberry Pi notices. The cache also lets the Batch Processing assistant check an
+/// image ahead of its request without preparing it twice.
+///
+/// The cache is a MemoryCache of its own: the shared one, which the Markdown library uses, has no
+/// size limit, and its entries state no size.
+/// </remarks>
+/// <param name="rustService">The runtime which prepares the images.</param>
+/// <param name="logger">The logger.</param>
+public sealed class ModelImageService(RustService rustService, ILogger<ModelImageService> logger) : IDisposable
+{
+    /// <summary>
+    /// How much memory the cached images may take: a 1 MB photo takes about 2.7 MB as base64 in a string.
+    /// </summary>
+    private const long CACHE_SIZE_LIMIT_BYTES = 32 * 1024 * 1024;
+
+    /// <summary>
+    /// How long a cached image waits for its next message. A chat paused for longer prepares its images again.
+    /// </summary>
+    private static readonly TimeSpan CACHE_SLIDING_EXPIRATION = TimeSpan.FromMinutes(10);
+
+    private readonly MemoryCache cache = new(new MemoryCacheOptions { SizeLimit = CACHE_SIZE_LIMIT_BYTES });
+
+    /// <summary>
+    /// Prepares an attached image for the model of the given provider.
+    /// </summary>
+    /// <remarks>
+    /// A changed file and another byte budget each make for a new preparation. An image which could
+    /// not be prepared is not cached, so the next message tries it again, e.g., once the network
+    /// drive holding it is back.
+    /// </remarks>
+    /// <param name="image">The attached image.</param>
+    /// <param name="provider">The configured provider the image goes to.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The prepared image, or why it could not be prepared.</returns>
+    public async Task<PreparedModelImage> PrepareAsync(FileAttachmentImage image, AIStudio.Settings.Provider provider, CancellationToken token = default)
+    {
+        var file = new FileInfo(image.FilePath);
+        if (!file.Exists)
+        {
+            logger.LogWarning("The image '{ImagePath}' does not exist anymore and cannot go to the model.", image.FilePath);
+            return PreparedModelImage.Failed(ModelImageErrorCode.FILE_NOT_FOUND);
+        }
+
+        var maxBytes = IImageSourceExtensions.MaxImageSizeBytesFor(provider);
+        var key = new CacheKey(file.FullName, file.LastWriteTimeUtc, file.Length, maxBytes);
+        if (this.cache.TryGetValue(key, out PreparedModelImage? cached) && cached is not null)
+        {
+            logger.LogInformation("Took the image '{ImagePath}' for a budget of {MaxBytes} bytes from the cache.", file.FullName, maxBytes);
+            return cached;
+        }
+
+        var prepared = await rustService.PrepareImageForModelAsync(file.FullName, maxBytes, token);
+        if (!prepared.IsPrepared)
+        {
+            logger.LogWarning("The image '{ImagePath}' could not be prepared for the model: {ErrorCode}.", file.FullName, prepared.ErrorCode);
+            return prepared;
+        }
+
+        //
+        // The runtime logs the sizes before and after, so this line only says where it came from:
+        //
+        logger.LogInformation("The runtime prepared the image '{ImagePath}' for a budget of {MaxBytes} bytes; it is cached now.", file.FullName, maxBytes);
+        this.cache.Set(key, prepared, new MemoryCacheEntryOptions
+        {
+            Size = prepared.Base64.Length * (long)sizeof(char),
+            SlidingExpiration = CACHE_SLIDING_EXPIRATION,
+        });
+
+        return prepared;
+    }
+
+    /// <inheritdoc />
+    public void Dispose() => this.cache.Dispose();
+
+    /// <summary>
+    /// What makes a prepared image valid: the file as it is now, and the budget it was made for.
+    /// </summary>
+    private readonly record struct CacheKey(string FullPath, DateTime LastWriteTimeUtc, long Length, long MaxBytes);
+}

app/MindWork AI Studio/Tools/Services/RustService.Image.cs

@@ -26,4 +26,34 @@ public sealed partial class RustService
         return await response.Content.ReadFromJsonAsync<ImagePrepareResponse>(this.jsonRustSerializerOptions, token)
             ?? throw new InvalidDataException("The Rust image preparation returned an empty response.");
     }
+
+    /// <summary>
+    /// Prepares a local image for an AI model in the Rust runtime.
+    /// </summary>
+    /// <remarks>
+    /// The runtime turns the image upright, scales it down, and encodes it anew as JPEG or PNG,
+    /// within the given number of bytes. An image it cannot prepare, e.g., in an unsupported format,
+    /// comes back with an error code. So does a failed call to the runtime, as UNKNOWN, and never as
+    /// an empty image.
+    /// </remarks>
+    /// <param name="path">The absolute path of the image file.</param>
+    /// <param name="maxBytes">The largest number of bytes the encoded image may have, before base64.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The prepared image, or why it could not be prepared.</returns>
+    public async Task<PreparedModelImage> PrepareImageForModelAsync(string path, long maxBytes, CancellationToken token = default)
+    {
+        try
+        {
+            using var response = await this.http.PostAsJsonAsync("/image/prepare-for-model", new { path, maxBytes }, this.jsonRustSerializerOptions, token);
+            response.EnsureSuccessStatusCode();
+
+            return await response.Content.ReadFromJsonAsync<PreparedModelImage>(this.jsonRustSerializerOptions, token)
+                ?? PreparedModelImage.Failed(ModelImageErrorCode.UNKNOWN);
+        }
+        catch (Exception e) when (e is not OperationCanceledException)
+        {
+            this.logger?.LogError(e, "Preparing the image '{ImagePath}' for an AI model failed in the runtime.", path);
+            return PreparedModelImage.Failed(ModelImageErrorCode.UNKNOWN);
+        }
+    }
 }

app/MindWork AI Studio/Tools/Validation/FileExtensionValidation.cs

@@ -69,6 +69,17 @@ public static class FileExtensionValidation
                         TB("Images are not supported at this place")));
                     return false;
                 
+                //
+                // Whether AI Studio can prepare an image for a model does not depend on the
+                // provider. This holds for templates as well, therefore: a HEIC image attached
+                // there could never be sent.
+                //
+                case UseCase.ATTACHING_CONTENT when !FileTypes.IsAllowedPath(filePath, FileTypes.MODEL_IMAGE):
+                    await MessageBus.INSTANCE.SendWarning(new(
+                        Icons.Material.Filled.ImageNotSupported,
+                        string.Format(TB("AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG."), Path.GetExtension(filePath).TrimStart('.').ToUpperInvariant())));
+                    return false;
+
                 // In this use case, we don't validate the provider capabilities:
                 case UseCase.ATTACHING_CONTENT when !validateMediaFileTypes:
                     return true;
@@ -116,31 +127,4 @@ public static class FileExtensionValidation
             TB("Unsupported file type")));
         return false;
     }
-
-    /// <summary>
-    /// Validates that the file is a supported image format and sends appropriate MessageBus notifications when invalid.
-    /// </summary>
-    /// <param name="filePath">The file path to validate.</param>
-    /// <returns>True if valid image, false if invalid (error already sent via MessageBus).</returns>
-    public static async Task<bool> IsImageExtensionValidWithNotifyAsync(string filePath)
-    {
-        var ext = Path.GetExtension(filePath).TrimStart('.');
-        if (string.IsNullOrWhiteSpace(ext))
-        {
-            await MessageBus.INSTANCE.SendError(new(
-                Icons.Material.Filled.ImageNotSupported,
-                TB("File has no extension")));
-            return false;
-        }
-
-        if (FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE))
-        {
-            await MessageBus.INSTANCE.SendError(new(
-                Icons.Material.Filled.ImageNotSupported,
-                TB("Unsupported image format")));
-            return false;
-        }
-
-        return true;
-    }
 }