From ba99bf6faed3b8432ef314e351c2d33b17987cea Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 22 Oct 2026 10:57:04 +0200 Subject: [PATCH] Scaled down and converted images before sending them to AI models (#1101) --- .../AssistantBatchProcessing.razor | 2 +- .../AssistantBatchProcessing.razor.Content.cs | 43 +- ...sistantBatchProcessing.razor.Validation.cs | 2 +- .../AssistantBatchProcessing.razor.cs | 3 + .../Assistants/I18N/allTexts.lua | 47 +- app/MindWork AI Studio/Chat/ContentImage.cs | 18 - .../Chat/IImageSourceExtensions.cs | 76 +- .../Chat/ListContentBlockExtensions.cs | 55 +- .../SettingsDialogBatchProcessing.razor | 2 +- .../Plugins/configuration/plugin.lua | 2 +- .../plugin.lua | 47 +- .../plugin.lua | 47 +- app/MindWork AI Studio/Program.cs | 1 + .../Provider/Anthropic/ProviderAnthropic.cs | 14 +- .../Provider/OpenAI/ProviderOpenAI.cs | 18 +- .../Provider/SelfHosted/HostExtensions.cs | 23 + .../Settings/DataModel/DataBatchProcessing.cs | 2 +- .../Tools/Rust/FileTypes.cs | 13 +- .../Tools/Rust/ModelImageErrorCode.cs | 63 ++ .../Rust/ModelImageErrorCodeExtensions.cs | 36 + .../Tools/Rust/PreparedModelImage.cs | 39 + .../Tools/Services/ModelImageService.cs | 93 +++ .../Tools/Services/RustService.Image.cs | 30 + .../Validation/FileExtensionValidation.cs | 38 +- .../wwwroot/changelog/v26.10.1.md | 6 +- .../Chat/ListContentBlockExtensionsTests.cs | 83 +- app/Tests/Provider/ImageSizeLimitTests.cs | 39 +- app/Tests/Tools/FakeImageRuntime.cs | 60 ++ app/Tests/Tools/ModelImageFormatTests.cs | 46 ++ app/Tests/Tools/ModelImageServiceTests.cs | 132 ++++ runtime/Cargo.toml | 2 +- runtime/src/image.rs | 731 +++++++++++++++++- runtime/src/runtime_api.rs | 1 + 33 files changed, 1550 insertions(+), 264 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Rust/ModelImageErrorCode.cs create mode 100644 app/MindWork AI Studio/Tools/Rust/ModelImageErrorCodeExtensions.cs create mode 100644 app/MindWork AI Studio/Tools/Rust/PreparedModelImage.cs create mode 100644 app/MindWork AI Studio/Tools/Services/ModelImageService.cs create mode 100644 app/Tests/Tools/FakeImageRuntime.cs create mode 100644 app/Tests/Tools/ModelImageFormatTests.cs create mode 100644 app/Tests/Tools/ModelImageServiceTests.cs diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor index 11340b928..e528c0e50 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor @@ -26,7 +26,7 @@ - @T("Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.") + @T("Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.") diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs index 36a10a7dc..3dd7c7b05 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs @@ -20,7 +20,7 @@ public partial class AssistantBatchProcessing private async Task LoadInputContentAsync(BatchProcessingFileResult fileResult, CancellationToken token) { if (IsImage(fileResult.FilePath)) - return this.LoadImage(fileResult) is { } image ? new BatchInput(string.Empty, image) : null; + return await this.LoadImageAsync(fileResult, token) is { } image ? new BatchInput(string.Empty, image) : null; var text = IsTranscribableMedia(fileResult.FilePath) ? await this.LoadMediaTranscriptAsync(fileResult, token) @@ -30,13 +30,18 @@ public partial class AssistantBatchProcessing } /// - /// Checks an image before it goes to the model. + /// Prepares an image for the model before the request is built. /// /// /// The request itself would only skip an image it cannot send, with a message, and ask the model /// anyway. The model would then answer about nothing, and the file would count as done, so that - /// continuing the run would skip it as well. A failed image is processed again when the run is - /// continued, e.g., with a model that can read images. + /// continuing the run would skip it as well. Preparing the image here lets it fail as a file + /// instead, with the reason, e.g., an image format AI Studio cannot decode. The request then finds + /// the prepared image in the cache of ModelImageService, so it is not prepared twice. A failed + /// image is processed again when the run is continued, e.g., with a model that can read images. + /// + /// How large an image may be needs no check of its own either: the preparation scales it down + /// until it fits the limit of the provider, and fails only when even that is not enough. /// /// How many images a model accepts needs no check of its own: every file gets a thread of its own, /// see CallAIAsync, so each request carries exactly one image, in every round with the model. A @@ -49,7 +54,7 @@ public partial class AssistantBatchProcessing /// longer send data out, and without free address choice, its default, the Read Web Page tool /// refuses an address which appears only in the image. /// - private FileAttachmentImage? LoadImage(BatchProcessingFileResult fileResult) + private async Task LoadImageAsync(BatchProcessingFileResult fileResult, CancellationToken token) { if (!this.ProviderSettings.SupportsImageInput()) { @@ -57,16 +62,17 @@ public partial class AssistantBatchProcessing return null; } - if (!FileTypes.IsAllowedPath(fileResult.FilePath, FileTypes.MODEL_IMAGE)) - { - this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.")); - return null; - } - - long fileSize; + PreparedModelImage prepared; + FileAttachmentImage image; try { - fileSize = new FileInfo(fileResult.FilePath).Length; + image = new FileAttachmentImage(fileResult.FileName, fileResult.FilePath, new FileInfo(fileResult.FilePath).Length); + prepared = await this.ModelImageService.PrepareAsync(image, this.ProviderSettings, token); + } + catch (OperationCanceledException) + { + this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled.")); + return null; } catch (Exception e) { @@ -74,14 +80,11 @@ public partial class AssistantBatchProcessing return null; } - var maxImageSizeBytes = IImageSourceExtensions.MaxImageSizeBytesFor(this.ProviderSettings.UsedLLMProvider); - if (fileSize > maxImageSizeBytes) - { - this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("The image is larger than {0} MB, which is the most the selected provider accepts for one image."), (maxImageSizeBytes / 1_000_000d).ToString("0.#", Tools.PluginSystem.I18N.I.Culture))); - return null; - } + if (prepared.IsPrepared) + return image; - return new FileAttachmentImage(fileResult.FileName, fileResult.FilePath, fileSize); + this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, prepared.ErrorCode.ToUserMessage(fileResult.FileName)); + return null; } private async Task LoadDocumentContentAsync(BatchProcessingFileResult fileResult, CancellationToken token) diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs index 37294d7f8..dc8643d74 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs @@ -180,7 +180,7 @@ public partial class AssistantBatchProcessing // // A model that cannot read images would get the text of the request alone, without the - // image, and answer about nothing. Such images fail one by one, see LoadImage, while the + // image, and answer about nothing. Such images fail one by one, see LoadImageAsync, while the // documents next to them are processed. A run of nothing but images does not start at all: // if (files.All(IsImage) && !this.ProviderSettings.SupportsImageInput()) diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs index 038e543c6..0a32f5bd9 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs @@ -15,6 +15,9 @@ public partial class AssistantBatchProcessing : AssistantBaseCore