Scaled down and converted images before sending them to AI models (#1101)

app/Tests/Chat/ListContentBlockExtensionsTests.cs

@@ -1,6 +1,11 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Provider.OpenAI;
+using AIStudio.Tests.Tools;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+using Microsoft.Extensions.DependencyInjection;
 
 namespace AIStudio.Tests.Chat;
 
@@ -13,10 +18,47 @@ namespace AIStudio.Tests.Chat;
 /// Somebody who switched image input on for their own installation watched the picture attach and
 /// then watched it disappear on the way to the model -- every chat round and every tool round, with
 /// nothing anywhere saying why.
+///
+/// Each picture is prepared for the model on its way, by the runtime, which a test plays here.
+/// A picture which cannot be prepared has to stay out of the message entirely.
 /// </remarks>
 [TestFixture]
 public sealed class ListContentBlockExtensionsTests
 {
+    private string picturePath = string.Empty;
+    private FakeImageRuntime runtime = null!;
+    private RustService rustService = null!;
+    private ServiceProvider serviceProvider = null!;
+    private IServiceProvider previousServiceProvider = null!;
+
+    [SetUp]
+    public void PrepareServices()
+    {
+        this.picturePath = Path.Combine(Path.GetTempPath(), $"picture-{Guid.NewGuid():N}.webp");
+        File.WriteAllBytes(this.picturePath, [1, 2, 3]);
+
+        this.runtime = new FakeImageRuntime();
+        this.rustService = new RustService(this.runtime);
+        this.previousServiceProvider = Program.SERVICE_PROVIDER;
+        this.serviceProvider = new ServiceCollection()
+            .AddSingleton(this.rustService)
+            .AddSingleton<ModelImageService>()
+            .AddLogging()
+            .BuildServiceProvider();
+
+        Program.SERVICE_PROVIDER = this.serviceProvider;
+    }
+
+    [TearDown]
+    public void RestoreApplicationState()
+    {
+        Program.SERVICE_PROVIDER = this.previousServiceProvider;
+        this.serviceProvider.Dispose();
+        this.rustService.Dispose();
+        this.runtime.Dispose();
+        File.Delete(this.picturePath);
+    }
+
     [Test]
     public async Task ImageInputSwitchedOnByHandReachesTheMessageAsWell()
     {
@@ -31,6 +73,32 @@ public sealed class ListContentBlockExtensionsTests
         Assert.That(messages.Single(), Is.InstanceOf<MultimodalMessage>(), "The picture is part of the message because the person said this model can read one.");
     }
 
+    [Test]
+    public async Task APictureWhichCannotBePreparedLeavesNoEmptyPart()
+    {
+        this.runtime.ErrorCode = "NOT_DECODABLE";
+
+        var messages = await BlocksWithAPicture(this.picturePath).BuildMessagesUsingNestedImageUrlAsync(PictureReadingProvider());
+
+        var message = messages.Single() as MultimodalMessage;
+        Assert.That(message, Is.Not.Null, "The text still goes to the model.");
+        Assert.That(message!.Content, Has.Count.EqualTo(1).And.All.InstanceOf<SubContentText>(), "A picture without data makes the provider refuse the whole request.");
+    }
+
+    [Test]
+    public async Task TheTypeOfAPictureComesFromItsPreparation()
+    {
+        // The file is a WebP, which the runtime prepared as JPEG:
+        var messages = await BlocksWithAPicture(this.picturePath).BuildMessagesUsingDirectImageUrlAsync(PictureReadingProvider());
+
+        var picture = (messages.Single() as MultimodalMessage)!.Content.OfType<SubContentImageUrl>().Single();
+        Assert.Multiple(() =>
+        {
+            Assert.That(picture.ImageUrl, Is.EqualTo($"data:image/jpeg;base64,{FakeImageRuntime.IMAGE_BASE64}"));
+            Assert.That(this.runtime.RequestedPaths, Is.EqualTo(new[] { new FileInfo(this.picturePath).FullName }));
+        });
+    }
+
     [Test]
     public async Task WithoutThatSwitchThePictureStaysOut()
     {
@@ -41,11 +109,20 @@ public sealed class ListContentBlockExtensionsTests
         Assert.That(messages.Single(), Is.InstanceOf<TextMessage>(), "Nothing says this model reads pictures, so the text goes on its own.");
     }
 
+    /// <summary>
+    /// A model which reads text only, according to the rules, but which a person said reads pictures.
+    /// </summary>
+    private static AIStudio.Settings.Provider PictureReadingProvider() => new(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
+    {
+        CapabilityOverrides = new() { MultipleImageInput = true },
+    };
+
     /// <summary>
     /// One block of text with a picture hanging on it.
     /// </summary>
+    /// <param name="picturePath">Where the picture lies.</param>
     /// <returns>The blocks.</returns>
-    private static List<ContentBlock> BlocksWithAPicture() =>
+    private static List<ContentBlock> BlocksWithAPicture(string picturePath = "/tmp/picture.png") =>
     [
         new()
         {
@@ -54,12 +131,12 @@ public sealed class ListContentBlockExtensionsTests
             Content = new ContentText
             {
                 Text = "What is in this picture?",
-                FileAttachments = [new FileAttachmentImage("picture.png", "/tmp/picture.png", 1_024)],
+                FileAttachments = [new FileAttachmentImage(Path.GetFileName(picturePath), picturePath, 1_024)],
             },
         },
     ];
 
     private static ISubContent Text(string text) => new SubContentText { Text = text };
 
-    private static Task<ISubContent> Picture(FileAttachmentImage image) => Task.FromResult<ISubContent>(new SubContentText { Text = image.FileName });
+    private static ISubContent Picture(PreparedModelImage image) => new SubContentText { Text = image.MimeType };
 }