Scaled down and converted images before sending them to AI models (#1101)

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -98,18 +98,19 @@ public static class FileTypes
 
     // Media hierarchy
     public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
-        "jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic", "avif");
+        "jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "svg", "webp", "heic", "avif");
 
     /// <summary>
-    /// The image formats we hand to a model as they are.
+    /// The image formats AI Studio can prepare for a model.
     /// </summary>
     /// <remarks>
-    /// The providers accept these, while other formats, such as TIFF or HEIC, would end in an
-    /// error of the provider which tells the user nothing. The default file patterns of the batch
-    /// processing assistant list the same formats, see DataBatchProcessing.DEFAULT_FILE_PATTERNS.
+    /// These are the formats the runtime decodes before it sends an image to a model as JPEG or
+    /// PNG, see prepare_image_for_model in runtime/src/image.rs. Other images, such as HEIC, AVIF,
+    /// or SVG, cannot go to a model at all. The default file patterns of the batch processing
+    /// assistant list the same formats, see DataBatchProcessing.DEFAULT_FILE_PATTERNS.
     /// </remarks>
     public static readonly FileTypeFilter MODEL_IMAGE = FileTypeFilter.Leaf(TB("Image for AI models"),
-        "jpg", "jpeg", "png", "webp", "gif");
+        "jpg", "jpeg", "png", "webp", "gif", "bmp", "tif", "tiff");
 
     /// <summary>
     /// Gets the prototype visual-asset image formats.

app/MindWork AI Studio/Tools/Rust/ModelImageErrorCode.cs

@@ -0,0 +1,63 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Why the Rust runtime could not prepare an image for an AI model.
+/// </summary>
+/// <remarks>
+/// UNKNOWN comes first on purpose. The converter for the enums of the runtime falls back to the
+/// default value for a code it does not know, e.g., from a newer runtime, and such a code must
+/// never read as a prepared image.
+/// </remarks>
+public enum ModelImageErrorCode
+{
+    /// <summary>
+    /// The runtime answered with a code this version does not know, or it could not be asked at all.
+    /// </summary>
+    UNKNOWN,
+
+    /// <summary>
+    /// No failure happened: the image was prepared.
+    /// </summary>
+    NONE,
+
+    //
+    // Codes reported by the Rust runtime:
+    //
+
+    FILE_NOT_FOUND,
+
+    /// <summary>
+    /// The file exists, but could not be opened, e.g., for lack of permission.
+    /// </summary>
+    FILE_NOT_READABLE,
+
+    /// <summary>
+    /// AI Studio cannot decode this image format, e.g., HEIC, AVIF, or SVG.
+    /// </summary>
+    UNSUPPORTED_FORMAT,
+
+    /// <summary>
+    /// The content of the file does not match its file extension.
+    /// </summary>
+    CONTENT_MISMATCH,
+
+    /// <summary>
+    /// The file is damaged or uses a variant of its format which cannot be decoded.
+    /// </summary>
+    NOT_DECODABLE,
+
+    /// <summary>
+    /// Decoding the image would need more memory than the runtime allows.
+    /// </summary>
+    TOO_MANY_PIXELS,
+
+    /// <summary>
+    /// Not even the smallest size and lowest quality the runtime accepts fit into the byte budget.
+    /// </summary>
+    BUDGET_NOT_REACHABLE,
+
+    /// <summary>
+    /// Any failure in the runtime which does not carry a code of its own.
+    /// </summary>
+    INTERNAL,
+}

app/MindWork AI Studio/Tools/Rust/ModelImageErrorCodeExtensions.cs

@@ -0,0 +1,36 @@
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Translates why an image could not be prepared for an AI model into user-facing text.
+/// </summary>
+/// <remarks>
+/// The runtime writes the technical details into the log. The texts here name what the user can
+/// act on, such as converting the image into another format.
+/// </remarks>
+public static class ModelImageErrorCodeExtensions
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ModelImageErrorCodeExtensions).Namespace, nameof(ModelImageErrorCodeExtensions));
+
+    /// <summary>
+    /// Gets the localized message which explains why an image was not sent to the AI.
+    /// </summary>
+    /// <param name="code">Why the image could not be prepared.</param>
+    /// <param name="fileName">The name of the image file, as shown to the user.</param>
+    /// <returns>The localized message.</returns>
+    public static string ToUserMessage(this ModelImageErrorCode code, string fileName) => string.Format(ToUserMessageFormat(code), fileName);
+
+    private static string ToUserMessageFormat(ModelImageErrorCode code) => code switch
+    {
+        ModelImageErrorCode.FILE_NOT_FOUND => TB("The image '{0}' does not exist anymore and was not sent."),
+        ModelImageErrorCode.FILE_NOT_READABLE => TB("The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable."),
+        ModelImageErrorCode.UNSUPPORTED_FORMAT => TB("The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG."),
+        ModelImageErrorCode.CONTENT_MISMATCH => TB("The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension."),
+        ModelImageErrorCode.NOT_DECODABLE => TB("The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely."),
+        ModelImageErrorCode.TOO_MANY_PIXELS => TB("The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first."),
+        ModelImageErrorCode.BUDGET_NOT_REACHABLE => TB("The image '{0}' could not be made small enough for the selected provider, so it was not sent."),
+
+        _ => TB("AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent."),
+    };
+}

app/MindWork AI Studio/Tools/Rust/PreparedModelImage.cs

@@ -0,0 +1,39 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// An image as the Rust runtime prepared it for an AI model, or why it could not.
+/// </summary>
+/// <remarks>
+/// The base64 and its MIME type travel side by side, so a provider which wants a data URL builds
+/// one and a provider which wants them apart, such as Anthropic, never parses one.
+/// </remarks>
+/// <param name="ErrorCode">NONE when the image was prepared; then the other values describe it.</param>
+/// <param name="Base64">The encoded image as base64, without a data URI prefix.</param>
+/// <param name="MimeType">The MIME type of the encoded image: image/jpeg or image/png.</param>
+/// <param name="Width">The width of the prepared image in pixels.</param>
+/// <param name="Height">The height of the prepared image in pixels.</param>
+/// <param name="WasResized">Whether the runtime scaled the image down.</param>
+public sealed record PreparedModelImage(ModelImageErrorCode ErrorCode, string Base64, string MimeType, uint Width, uint Height, bool WasResized)
+{
+    /// <summary>
+    /// Whether the image can go to the model.
+    /// </summary>
+    /// <remarks>
+    /// The image itself has to be there as well: a success without one would become an empty image
+    /// in the request, which providers refuse along with the whole request.
+    /// </remarks>
+    public bool IsPrepared => this.ErrorCode is ModelImageErrorCode.NONE && !string.IsNullOrEmpty(this.Base64);
+
+    /// <summary>
+    /// The image as a data URL, as the OpenAI-compatible APIs take it.
+    /// </summary>
+    /// <returns>The data URL.</returns>
+    public string ToDataUrl() => $"data:{this.MimeType};base64,{this.Base64}";
+
+    /// <summary>
+    /// An image which could not be prepared.
+    /// </summary>
+    /// <param name="errorCode">Why it could not be prepared.</param>
+    /// <returns>The failed image.</returns>
+    public static PreparedModelImage Failed(ModelImageErrorCode errorCode) => new(errorCode, string.Empty, string.Empty, 0, 0, false);
+}