Scaled down and converted images before sending them to AI models (#1101)

app/MindWork AI Studio/Chat/IImageSourceExtensions.cs

@@ -1,5 +1,5 @@
 using AIStudio.Provider;
-using AIStudio.Tools.MIME;
+using AIStudio.Provider.SelfHosted;
 using AIStudio.Tools.PluginSystem;
 
 namespace AIStudio.Chat;
@@ -9,7 +9,8 @@ public static class IImageSourceExtensions
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IImageSourceExtensions).Namespace, nameof(IImageSourceExtensions));
 
     /// <summary>
-    /// The largest image we send to a model. A larger image is skipped by TryAsBase64.
+    /// The largest image, in bytes, we send anywhere: no image prepared for a model is larger, and
+    /// TryAsBase64 skips every larger file.
     /// </summary>
     public const long MAX_IMAGE_SIZE_BYTES = 10_000_000;
 
@@ -17,79 +18,30 @@ public static class IImageSourceExtensions
     /// The largest image file, in bytes, which we send through the given provider.
     /// </summary>
     /// <remarks>
-    /// The limit of the provider where it states one, but never more than MAX_IMAGE_SIZE_BYTES:
-    /// TryAsBase64 skips every larger image, whatever the provider would accept.
+    /// The smallest of the limits we know: the one of a self-hosted engine, the one of the provider,
+    /// and MAX_IMAGE_SIZE_BYTES, our own. A provider or engine which states no limit gets ours.
     /// </remarks>
-    /// <param name="llmProvider">The provider the image is sent through.</param>
+    /// <param name="provider">The configured provider the image is sent through.</param>
     /// <returns>The size limit of one image file.</returns>
-    public static long MaxImageSizeBytesFor(LLMProviders llmProvider) => Math.Min(llmProvider.GetMaxImageSizeBytes() ?? MAX_IMAGE_SIZE_BYTES, MAX_IMAGE_SIZE_BYTES);
-
-    public static MIMEType DetermineMimeType(this IImageSource image)
+    public static long MaxImageSizeBytesFor(AIStudio.Settings.Provider provider)
     {
-        switch (image.SourceType)
-        {
-            case ContentImageSource.BASE64:
-            {
-                // Try to detect the mime type from the base64 string:
-                var base64Data = image.Source;
-                if (base64Data.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
-                {
-                    var mimeEnd = base64Data.IndexOf(';');
-                    if (mimeEnd > 5)
-                        return Builder.FromTextRepresentation(base64Data[5..mimeEnd]);
-                }
-
-                // Fallback:
-                return Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build();
-            }
-
-            case ContentImageSource.URL:
-            {
-                // Try to detect the mime type from the URL extension:
-                var uri = new Uri(image.Source);
-                var extension = Path.GetExtension(uri.AbsolutePath).ToLowerInvariant();
-                return DeriveMIMETypeFromExtension(extension);
-            }
-
-            case ContentImageSource.LOCAL_PATH:
-            {
-                var extension = Path.GetExtension(image.Source).ToLowerInvariant();
-                return DeriveMIMETypeFromExtension(extension);
-            }
-
-            default:
-                return Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build();
-        }
-    }
-
-    private static MIMEType DeriveMIMETypeFromExtension(string extension)
-    {
-        var imageBuilder = Builder.Create().UseImage();
-        return extension switch
-        {
-            ".png" => imageBuilder.UseSubtype(ImageSubtype.PNG).Build(),
-            ".jpg" or ".jpeg" => imageBuilder.UseSubtype(ImageSubtype.JPEG).Build(),
-            ".gif" => imageBuilder.UseSubtype(ImageSubtype.GIF).Build(),
-            ".webp" => imageBuilder.UseSubtype(ImageSubtype.WEBP).Build(),
-            ".tiff" or ".tif" => imageBuilder.UseSubtype(ImageSubtype.TIFF).Build(),
-            ".heic" or ".heif" => imageBuilder.UseSubtype(ImageSubtype.HEIC).Build(),
-
-            _ => Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build()
-        };
+        var engineLimit = provider.UsedLLMProvider is LLMProviders.SELF_HOSTED ? provider.Host.GetMaxImageSizeBytes() : null;
+        var providerLimit = provider.UsedLLMProvider.GetMaxImageSizeBytes();
+        return Math.Min(Math.Min(engineLimit ?? MAX_IMAGE_SIZE_BYTES, providerLimit ?? MAX_IMAGE_SIZE_BYTES), MAX_IMAGE_SIZE_BYTES);
     }
 
     /// <summary>
     /// Read the image content as a base64 string.
     /// </summary>
     /// <remarks>
+    /// This is for the images inside the content of a chat, which go to external data sources and
+    /// agents as they are. Attached images, which go to a model, take ModelImageService instead: it
+    /// scales them down and encodes them anew within the limit of the provider.<br/>
+    /// <br/>
     /// The images are directly converted to base64 strings. The maximum
     /// size of the image is around 10 MB. If the image is larger, the method
     /// returns an empty string.<br/>
     /// <br/>
-    /// As of now, this method does no sort of image processing. LLMs usually
-    /// do not work with arbitrary image sizes. In the future, we might have
-    /// to resize the images before sending them to the model.<br/>
-    /// <br/>
     /// Note as well that this method returns just the base64 string without
     /// any data URI prefix (like "data:image/png;base64,"). The caller has
     /// to take care of that if needed.