Scaled down and converted images before sending them to AI models (#1101)

app/Tests/Chat/ListContentBlockExtensionsTests.cs

@@ -1,6 +1,11 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Provider.OpenAI;
+using AIStudio.Tests.Tools;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+using Microsoft.Extensions.DependencyInjection;
 
 namespace AIStudio.Tests.Chat;
 
@@ -13,10 +18,47 @@ namespace AIStudio.Tests.Chat;
 /// Somebody who switched image input on for their own installation watched the picture attach and
 /// then watched it disappear on the way to the model -- every chat round and every tool round, with
 /// nothing anywhere saying why.
+///
+/// Each picture is prepared for the model on its way, by the runtime, which a test plays here.
+/// A picture which cannot be prepared has to stay out of the message entirely.
 /// </remarks>
 [TestFixture]
 public sealed class ListContentBlockExtensionsTests
 {
+    private string picturePath = string.Empty;
+    private FakeImageRuntime runtime = null!;
+    private RustService rustService = null!;
+    private ServiceProvider serviceProvider = null!;
+    private IServiceProvider previousServiceProvider = null!;
+
+    [SetUp]
+    public void PrepareServices()
+    {
+        this.picturePath = Path.Combine(Path.GetTempPath(), $"picture-{Guid.NewGuid():N}.webp");
+        File.WriteAllBytes(this.picturePath, [1, 2, 3]);
+
+        this.runtime = new FakeImageRuntime();
+        this.rustService = new RustService(this.runtime);
+        this.previousServiceProvider = Program.SERVICE_PROVIDER;
+        this.serviceProvider = new ServiceCollection()
+            .AddSingleton(this.rustService)
+            .AddSingleton<ModelImageService>()
+            .AddLogging()
+            .BuildServiceProvider();
+
+        Program.SERVICE_PROVIDER = this.serviceProvider;
+    }
+
+    [TearDown]
+    public void RestoreApplicationState()
+    {
+        Program.SERVICE_PROVIDER = this.previousServiceProvider;
+        this.serviceProvider.Dispose();
+        this.rustService.Dispose();
+        this.runtime.Dispose();
+        File.Delete(this.picturePath);
+    }
+
     [Test]
     public async Task ImageInputSwitchedOnByHandReachesTheMessageAsWell()
     {
@@ -31,6 +73,32 @@ public sealed class ListContentBlockExtensionsTests
         Assert.That(messages.Single(), Is.InstanceOf<MultimodalMessage>(), "The picture is part of the message because the person said this model can read one.");
     }
 
+    [Test]
+    public async Task APictureWhichCannotBePreparedLeavesNoEmptyPart()
+    {
+        this.runtime.ErrorCode = "NOT_DECODABLE";
+
+        var messages = await BlocksWithAPicture(this.picturePath).BuildMessagesUsingNestedImageUrlAsync(PictureReadingProvider());
+
+        var message = messages.Single() as MultimodalMessage;
+        Assert.That(message, Is.Not.Null, "The text still goes to the model.");
+        Assert.That(message!.Content, Has.Count.EqualTo(1).And.All.InstanceOf<SubContentText>(), "A picture without data makes the provider refuse the whole request.");
+    }
+
+    [Test]
+    public async Task TheTypeOfAPictureComesFromItsPreparation()
+    {
+        // The file is a WebP, which the runtime prepared as JPEG:
+        var messages = await BlocksWithAPicture(this.picturePath).BuildMessagesUsingDirectImageUrlAsync(PictureReadingProvider());
+
+        var picture = (messages.Single() as MultimodalMessage)!.Content.OfType<SubContentImageUrl>().Single();
+        Assert.Multiple(() =>
+        {
+            Assert.That(picture.ImageUrl, Is.EqualTo($"data:image/jpeg;base64,{FakeImageRuntime.IMAGE_BASE64}"));
+            Assert.That(this.runtime.RequestedPaths, Is.EqualTo(new[] { new FileInfo(this.picturePath).FullName }));
+        });
+    }
+
     [Test]
     public async Task WithoutThatSwitchThePictureStaysOut()
     {
@@ -41,11 +109,20 @@ public sealed class ListContentBlockExtensionsTests
         Assert.That(messages.Single(), Is.InstanceOf<TextMessage>(), "Nothing says this model reads pictures, so the text goes on its own.");
     }
 
+    /// <summary>
+    /// A model which reads text only, according to the rules, but which a person said reads pictures.
+    /// </summary>
+    private static AIStudio.Settings.Provider PictureReadingProvider() => new(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
+    {
+        CapabilityOverrides = new() { MultipleImageInput = true },
+    };
+
     /// <summary>
     /// One block of text with a picture hanging on it.
     /// </summary>
+    /// <param name="picturePath">Where the picture lies.</param>
     /// <returns>The blocks.</returns>
-    private static List<ContentBlock> BlocksWithAPicture() =>
+    private static List<ContentBlock> BlocksWithAPicture(string picturePath = "/tmp/picture.png") =>
     [
         new()
         {
@@ -54,12 +131,12 @@ public sealed class ListContentBlockExtensionsTests
             Content = new ContentText
             {
                 Text = "What is in this picture?",
-                FileAttachments = [new FileAttachmentImage("picture.png", "/tmp/picture.png", 1_024)],
+                FileAttachments = [new FileAttachmentImage(Path.GetFileName(picturePath), picturePath, 1_024)],
             },
         },
     ];
 
     private static ISubContent Text(string text) => new SubContentText { Text = text };
 
-    private static Task<ISubContent> Picture(FileAttachmentImage image) => Task.FromResult<ISubContent>(new SubContentText { Text = image.FileName });
+    private static ISubContent Picture(PreparedModelImage image) => new SubContentText { Text = image.MimeType };
 }

app/Tests/Provider/ImageSizeLimitTests.cs

@@ -1,5 +1,6 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
+using AIStudio.Provider.SelfHosted;
 
 namespace AIStudio.Tests.Provider;
 
@@ -7,8 +8,8 @@ namespace AIStudio.Tests.Provider;
 /// Checks how large an image may be when we send it through a provider.
 /// </summary>
 /// <remarks>
-/// A provider states its limit or says nothing. Either way, TryAsBase64 skips every image above our
-/// own limit, so a larger limit of a provider must never reach the checks done before sending.
+/// A provider or a self-hosted engine states its limit or says nothing. Either way, we never send
+/// more than our own limit, so a larger limit of a provider must never reach the image preparation.
 /// </remarks>
 [TestFixture]
 public sealed class ImageSizeLimitTests
@@ -19,7 +20,7 @@ public sealed class ImageSizeLimitTests
         Assert.Multiple(() =>
         {
             Assert.That(LLMProviders.ANTHROPIC.GetMaxImageSizeBytes(), Is.EqualTo(3_932_160));
-            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(LLMProviders.ANTHROPIC), Is.EqualTo(3_932_160));
+            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.ANTHROPIC)), Is.EqualTo(3_932_160));
         });
     }
 
@@ -29,17 +30,41 @@ public sealed class ImageSizeLimitTests
         Assert.Multiple(() =>
         {
             Assert.That(LLMProviders.OPEN_AI.GetMaxImageSizeBytes(), Is.Null, "Nobody stated a limit, and no number stands in for that.");
-            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(LLMProviders.OPEN_AI), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
+            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.OPEN_AI)), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
         });
     }
 
     [Test]
-    public void NoProviderGetsMoreThanTryAsBase64Sends()
+    public void LlmmanLeavesRoomForTheTextOfTheConversation()
     {
         Assert.Multiple(() =>
         {
-            foreach (var provider in Enum.GetValues<LLMProviders>())
-                Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(provider), Is.LessThanOrEqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), provider.ToString());
+            Assert.That(Host.LLMMAN.GetMaxImageSizeBytes(), Is.EqualTo(1_048_576));
+            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.SELF_HOSTED, Host.LLMMAN)), Is.EqualTo(1_048_576));
+            Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.SELF_HOSTED, Host.OLLAMA)), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), "Ollama states no limit we know of.");
         });
     }
+
+    [Test]
+    public void AnEngineCountsOnlyForSelfHostedProviders()
+    {
+        // A provider keeps the host it was once configured with, even after somebody switched it
+        // to a cloud provider in the dialog:
+        var cloudProviderWithAnOldHost = ProviderOf(LLMProviders.OPEN_AI, Host.LLMMAN);
+
+        Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(cloudProviderWithAnOldHost), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
+    }
+
+    [Test]
+    public void NoProviderGetsMoreThanOurOwnLimit()
+    {
+        Assert.Multiple(() =>
+        {
+            foreach (var llmProvider in Enum.GetValues<LLMProviders>())
+                foreach (var host in Enum.GetValues<Host>())
+                    Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(llmProvider, host)), Is.LessThanOrEqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), $"{llmProvider} with {host}");
+        });
+    }
+
+    private static AIStudio.Settings.Provider ProviderOf(LLMProviders llmProvider, Host host = Host.NONE) => new(0, "test", "Test", llmProvider, new AIStudio.Provider.Model("model", null), Host: host);
 }

app/Tests/Tools/FakeImageRuntime.cs

@@ -0,0 +1,60 @@
+using System.Collections.Concurrent;
+using System.Net;
+using System.Text;
+using System.Text.Json;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Plays the image route of the runtime for a test, and notes what the app asked for.
+/// </summary>
+/// <remarks>
+/// It prepares nothing: every image comes back as the same three bytes, with the code and type the
+/// test sets. The answer is spelled out in the wire format of the runtime, so a test fails as soon
+/// as the app stops understanding what the runtime actually sends.
+/// </remarks>
+internal sealed class FakeImageRuntime : HttpMessageHandler
+{
+    /// <summary>
+    /// The three bytes every prepared image consists of, as base64.
+    /// </summary>
+    public const string IMAGE_BASE64 = "AQID";
+
+    private readonly ConcurrentQueue<string> requestedPaths = new();
+    private readonly ConcurrentQueue<long> requestedBudgets = new();
+
+    /// <summary>
+    /// The code the runtime answers with, as the runtime spells it.
+    /// </summary>
+    public string ErrorCode { get; set; } = "NONE";
+
+    /// <summary>
+    /// The MIME type the runtime reports for every prepared image.
+    /// </summary>
+    public string MimeType { get; set; } = "image/jpeg";
+
+    /// <summary>
+    /// The paths of the images the app asked to prepare, in order.
+    /// </summary>
+    public IReadOnlyList<string> RequestedPaths => [..this.requestedPaths];
+
+    /// <summary>
+    /// The byte budgets the app asked for, in order.
+    /// </summary>
+    public IReadOnlyList<long> RequestedBudgets => [..this.requestedBudgets];
+
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        if (request.RequestUri!.AbsolutePath is not "/image/prepare-for-model")
+            return new HttpResponseMessage(HttpStatusCode.NotFound);
+
+        using (var body = JsonDocument.Parse(await request.Content!.ReadAsStringAsync(cancellationToken)))
+        {
+            this.requestedPaths.Enqueue(body.RootElement.GetProperty("path").GetString() ?? string.Empty);
+            this.requestedBudgets.Enqueue(body.RootElement.GetProperty("max_bytes").GetInt64());
+        }
+
+        var answer = JsonSerializer.Serialize(new { error_code = this.ErrorCode, base64 = IMAGE_BASE64, mime_type = this.MimeType, width = 1, height = 1, was_resized = false });
+        return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(answer, Encoding.UTF8, "application/json") };
+    }
+}

app/Tests/Tools/ModelImageFormatTests.cs

@@ -0,0 +1,46 @@
+using AIStudio.Chat;
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which image formats may be attached for an AI model.
+/// </summary>
+/// <remarks>
+/// The runtime decides which formats it can prepare for a model, see prepare_image_for_model in
+/// runtime/src/image.rs. An image the app lets through but the runtime cannot prepare would only
+/// fail once the message is sent; one the app refuses although the runtime could prepare it would
+/// be refused for nothing.
+/// </remarks>
+[TestFixture]
+public sealed class ModelImageFormatTests
+{
+    /// <summary>
+    /// The extensions model_input_format in runtime/src/image.rs accepts.
+    /// </summary>
+    private static readonly string[] RUNTIME_MODEL_FORMATS = ["jpg", "jpeg", "png", "webp", "gif", "bmp", "tif", "tiff"];
+
+    [Test]
+    public void EveryFormatTheRuntimePreparesMayBeAttached()
+    {
+        Assert.That(FileTypes.MODEL_IMAGE.FlattenExtensions(), Is.EquivalentTo(RUNTIME_MODEL_FORMATS));
+    }
+
+    [Test]
+    public void OnlyFormatsWithoutADecoderAreRefused()
+    {
+        var refused = FileTypes.IMAGE.FlattenExtensions().Except(FileTypes.MODEL_IMAGE.FlattenExtensions());
+
+        Assert.That(refused, Is.EquivalentTo(new[] { "svg", "heic", "avif" }));
+    }
+
+    [Test]
+    public void ATifFileIsAnImage()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(FileAttachment.FromPath("/tmp/scan.tif").IsImage, Is.True);
+            Assert.That(FileTypes.IsAllowedPath("/tmp/scan.tif", FileTypes.MODEL_IMAGE), Is.True);
+        });
+    }
+}

app/Tests/Tools/ModelImageServiceTests.cs

@@ -0,0 +1,132 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks when the image preparation asks the runtime again, and when it answers from its cache.
+/// </summary>
+/// <remarks>
+/// Every message prepares all images of the conversation anew. Asking the runtime each time would
+/// decode every photo again; answering from the cache after the file changed would send the old
+/// picture. The runtime is played in its wire format, so these tests also fail as soon as the app
+/// stops understanding what the runtime sends.
+/// </remarks>
+[TestFixture]
+public sealed class ModelImageServiceTests
+{
+    private string imagePath = string.Empty;
+    private FakeImageRuntime runtime = null!;
+    private RustService rustService = null!;
+    private ModelImageService service = null!;
+
+    [SetUp]
+    public void PrepareService()
+    {
+        this.imagePath = Path.Combine(Path.GetTempPath(), $"model-image-{Guid.NewGuid():N}.png");
+        File.WriteAllBytes(this.imagePath, [1, 2, 3]);
+
+        this.runtime = new FakeImageRuntime();
+        this.rustService = new RustService(this.runtime);
+        this.service = new ModelImageService(this.rustService, NullLogger<ModelImageService>.Instance);
+    }
+
+    [TearDown]
+    public void CleanUp()
+    {
+        this.service.Dispose();
+        this.rustService.Dispose();
+        this.runtime.Dispose();
+        File.Delete(this.imagePath);
+    }
+
+    [Test]
+    public async Task TheSameFileForTheSameProviderIsPreparedOnce()
+    {
+        var first = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+        var second = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(first.IsPrepared, Is.True);
+            Assert.That(first.MimeType, Is.EqualTo("image/jpeg"));
+            Assert.That(second, Is.SameAs(first));
+            Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(1), "The second message decoded the photo again.");
+        });
+    }
+
+    [Test]
+    public async Task AChangedFileIsPreparedAgain()
+    {
+        await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+        await File.WriteAllBytesAsync(this.imagePath, [1, 2, 3, 4]);
+        await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(2), "The model would get the picture as it was before.");
+    }
+
+    [Test]
+    public async Task AnotherBudgetIsPreparedAgainWithThatBudget()
+    {
+        await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+        await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.OPEN_AI));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(this.runtime.RequestedBudgets, Is.EqualTo(new[] { 3_932_160, IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES }));
+            Assert.That(this.runtime.RequestedPaths, Is.All.EqualTo(new FileInfo(this.imagePath).FullName));
+        });
+    }
+
+    [Test]
+    public async Task AFailureIsTriedAgainWithTheNextMessage()
+    {
+        this.runtime.ErrorCode = "NOT_DECODABLE";
+        var failed = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        this.runtime.ErrorCode = "NONE";
+        var prepared = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(failed.ErrorCode, Is.EqualTo(ModelImageErrorCode.NOT_DECODABLE));
+            Assert.That(failed.IsPrepared, Is.False);
+            Assert.That(prepared.IsPrepared, Is.True);
+            Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(2), "A failure must not stick to the image.");
+        });
+    }
+
+    [Test]
+    public async Task ACodeFromANewerRuntimeNeverCountsAsAPreparedImage()
+    {
+        this.runtime.ErrorCode = "SOMETHING_NEW";
+        var image = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(image.ErrorCode, Is.EqualTo(ModelImageErrorCode.UNKNOWN));
+            Assert.That(image.IsPrepared, Is.False, "The base64 of the answer must not go to the model when the code says nothing we know.");
+        });
+    }
+
+    [Test]
+    public async Task AMissingFileDoesNotReachTheRuntime()
+    {
+        File.Delete(this.imagePath);
+        var image = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(image.ErrorCode, Is.EqualTo(ModelImageErrorCode.FILE_NOT_FOUND));
+            Assert.That(this.runtime.RequestedPaths, Is.Empty);
+        });
+    }
+
+    private FileAttachmentImage Image => new("picture.png", this.imagePath, 3);
+
+    private static AIStudio.Settings.Provider ProviderOf(LLMProviders llmProvider) => new(0, "test", "Test", llmProvider, new Model("model", null));
+}