diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor
index 11340b928..e528c0e50 100644
--- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor
+++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor
@@ -26,7 +26,7 @@
- @T("Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.")
+ @T("Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.")
diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs
index 36a10a7dc..3dd7c7b05 100644
--- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs
+++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs
@@ -20,7 +20,7 @@ public partial class AssistantBatchProcessing
private async Task LoadInputContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
{
if (IsImage(fileResult.FilePath))
- return this.LoadImage(fileResult) is { } image ? new BatchInput(string.Empty, image) : null;
+ return await this.LoadImageAsync(fileResult, token) is { } image ? new BatchInput(string.Empty, image) : null;
var text = IsTranscribableMedia(fileResult.FilePath)
? await this.LoadMediaTranscriptAsync(fileResult, token)
@@ -30,13 +30,18 @@ public partial class AssistantBatchProcessing
}
///
- /// Checks an image before it goes to the model.
+ /// Prepares an image for the model before the request is built.
///
///
/// The request itself would only skip an image it cannot send, with a message, and ask the model
/// anyway. The model would then answer about nothing, and the file would count as done, so that
- /// continuing the run would skip it as well. A failed image is processed again when the run is
- /// continued, e.g., with a model that can read images.
+ /// continuing the run would skip it as well. Preparing the image here lets it fail as a file
+ /// instead, with the reason, e.g., an image format AI Studio cannot decode. The request then finds
+ /// the prepared image in the cache of ModelImageService, so it is not prepared twice. A failed
+ /// image is processed again when the run is continued, e.g., with a model that can read images.
+ ///
+ /// How large an image may be needs no check of its own either: the preparation scales it down
+ /// until it fits the limit of the provider, and fails only when even that is not enough.
///
/// How many images a model accepts needs no check of its own: every file gets a thread of its own,
/// see CallAIAsync, so each request carries exactly one image, in every round with the model. A
@@ -49,7 +54,7 @@ public partial class AssistantBatchProcessing
/// longer send data out, and without free address choice, its default, the Read Web Page tool
/// refuses an address which appears only in the image.
///
- private FileAttachmentImage? LoadImage(BatchProcessingFileResult fileResult)
+ private async Task LoadImageAsync(BatchProcessingFileResult fileResult, CancellationToken token)
{
if (!this.ProviderSettings.SupportsImageInput())
{
@@ -57,16 +62,17 @@ public partial class AssistantBatchProcessing
return null;
}
- if (!FileTypes.IsAllowedPath(fileResult.FilePath, FileTypes.MODEL_IMAGE))
- {
- this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."));
- return null;
- }
-
- long fileSize;
+ PreparedModelImage prepared;
+ FileAttachmentImage image;
try
{
- fileSize = new FileInfo(fileResult.FilePath).Length;
+ image = new FileAttachmentImage(fileResult.FileName, fileResult.FilePath, new FileInfo(fileResult.FilePath).Length);
+ prepared = await this.ModelImageService.PrepareAsync(image, this.ProviderSettings, token);
+ }
+ catch (OperationCanceledException)
+ {
+ this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled."));
+ return null;
}
catch (Exception e)
{
@@ -74,14 +80,11 @@ public partial class AssistantBatchProcessing
return null;
}
- var maxImageSizeBytes = IImageSourceExtensions.MaxImageSizeBytesFor(this.ProviderSettings.UsedLLMProvider);
- if (fileSize > maxImageSizeBytes)
- {
- this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("The image is larger than {0} MB, which is the most the selected provider accepts for one image."), (maxImageSizeBytes / 1_000_000d).ToString("0.#", Tools.PluginSystem.I18N.I.Culture)));
- return null;
- }
+ if (prepared.IsPrepared)
+ return image;
- return new FileAttachmentImage(fileResult.FileName, fileResult.FilePath, fileSize);
+ this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, prepared.ErrorCode.ToUserMessage(fileResult.FileName));
+ return null;
}
private async Task LoadDocumentContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs
index 37294d7f8..dc8643d74 100644
--- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs
+++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs
@@ -180,7 +180,7 @@ public partial class AssistantBatchProcessing
//
// A model that cannot read images would get the text of the request alone, without the
- // image, and answer about nothing. Such images fail one by one, see LoadImage, while the
+ // image, and answer about nothing. Such images fail one by one, see LoadImageAsync, while the
// documents next to them are processed. A run of nothing but images does not start at all:
//
if (files.All(IsImage) && !this.ProviderSettings.SupportsImageInput())
diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs
index 038e543c6..0a32f5bd9 100644
--- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs
+++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs
@@ -15,6 +15,9 @@ public partial class AssistantBatchProcessing : AssistantBaseCore
- /// Creates a ContentImage from a local file path.
- ///
- /// The path to the image file.
- /// A new ContentImage instance if the file is valid, null otherwise.
- public static async Task CreateFromFileAsync(string filePath)
- {
- if (!await FileExtensionValidation.IsImageExtensionValidWithNotifyAsync(filePath))
- return null;
-
- return new ContentImage
- {
- SourceType = ContentImageSource.LOCAL_PATH,
- Source = filePath,
- };
- }
-
///
/// The type of the image source.
///
diff --git a/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs b/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs
index b79ad86fb..2f8210b67 100644
--- a/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs
+++ b/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs
@@ -1,5 +1,5 @@
using AIStudio.Provider;
-using AIStudio.Tools.MIME;
+using AIStudio.Provider.SelfHosted;
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Chat;
@@ -9,7 +9,8 @@ public static class IImageSourceExtensions
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IImageSourceExtensions).Namespace, nameof(IImageSourceExtensions));
///
- /// The largest image we send to a model. A larger image is skipped by TryAsBase64.
+ /// The largest image, in bytes, we send anywhere: no image prepared for a model is larger, and
+ /// TryAsBase64 skips every larger file.
///
public const long MAX_IMAGE_SIZE_BYTES = 10_000_000;
@@ -17,79 +18,30 @@ public static class IImageSourceExtensions
/// The largest image file, in bytes, which we send through the given provider.
///
///
- /// The limit of the provider where it states one, but never more than MAX_IMAGE_SIZE_BYTES:
- /// TryAsBase64 skips every larger image, whatever the provider would accept.
+ /// The smallest of the limits we know: the one of a self-hosted engine, the one of the provider,
+ /// and MAX_IMAGE_SIZE_BYTES, our own. A provider or engine which states no limit gets ours.
///
- /// The provider the image is sent through.
+ /// The configured provider the image is sent through.
/// The size limit of one image file.
- public static long MaxImageSizeBytesFor(LLMProviders llmProvider) => Math.Min(llmProvider.GetMaxImageSizeBytes() ?? MAX_IMAGE_SIZE_BYTES, MAX_IMAGE_SIZE_BYTES);
-
- public static MIMEType DetermineMimeType(this IImageSource image)
+ public static long MaxImageSizeBytesFor(AIStudio.Settings.Provider provider)
{
- switch (image.SourceType)
- {
- case ContentImageSource.BASE64:
- {
- // Try to detect the mime type from the base64 string:
- var base64Data = image.Source;
- if (base64Data.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
- {
- var mimeEnd = base64Data.IndexOf(';');
- if (mimeEnd > 5)
- return Builder.FromTextRepresentation(base64Data[5..mimeEnd]);
- }
-
- // Fallback:
- return Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build();
- }
-
- case ContentImageSource.URL:
- {
- // Try to detect the mime type from the URL extension:
- var uri = new Uri(image.Source);
- var extension = Path.GetExtension(uri.AbsolutePath).ToLowerInvariant();
- return DeriveMIMETypeFromExtension(extension);
- }
-
- case ContentImageSource.LOCAL_PATH:
- {
- var extension = Path.GetExtension(image.Source).ToLowerInvariant();
- return DeriveMIMETypeFromExtension(extension);
- }
-
- default:
- return Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build();
- }
- }
-
- private static MIMEType DeriveMIMETypeFromExtension(string extension)
- {
- var imageBuilder = Builder.Create().UseImage();
- return extension switch
- {
- ".png" => imageBuilder.UseSubtype(ImageSubtype.PNG).Build(),
- ".jpg" or ".jpeg" => imageBuilder.UseSubtype(ImageSubtype.JPEG).Build(),
- ".gif" => imageBuilder.UseSubtype(ImageSubtype.GIF).Build(),
- ".webp" => imageBuilder.UseSubtype(ImageSubtype.WEBP).Build(),
- ".tiff" or ".tif" => imageBuilder.UseSubtype(ImageSubtype.TIFF).Build(),
- ".heic" or ".heif" => imageBuilder.UseSubtype(ImageSubtype.HEIC).Build(),
-
- _ => Builder.Create().UseApplication().UseSubtype(ApplicationSubtype.OCTET_STREAM).Build()
- };
+ var engineLimit = provider.UsedLLMProvider is LLMProviders.SELF_HOSTED ? provider.Host.GetMaxImageSizeBytes() : null;
+ var providerLimit = provider.UsedLLMProvider.GetMaxImageSizeBytes();
+ return Math.Min(Math.Min(engineLimit ?? MAX_IMAGE_SIZE_BYTES, providerLimit ?? MAX_IMAGE_SIZE_BYTES), MAX_IMAGE_SIZE_BYTES);
}
///
/// Read the image content as a base64 string.
///
///
+ /// This is for the images inside the content of a chat, which go to external data sources and
+ /// agents as they are. Attached images, which go to a model, take ModelImageService instead: it
+ /// scales them down and encodes them anew within the limit of the provider.
+ ///
/// The images are directly converted to base64 strings. The maximum
/// size of the image is around 10 MB. If the image is larger, the method
/// returns an empty string.
///
- /// As of now, this method does no sort of image processing. LLMs usually
- /// do not work with arbitrary image sizes. In the future, we might have
- /// to resize the images before sending them to the model.
- ///
/// Note as well that this method returns just the base64 string without
/// any data URI prefix (like "data:image/png;base64,"). The caller has
/// to take care of that if needed.
diff --git a/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs b/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs
index 3bc5e9b89..e1fce1b3a 100644
--- a/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs
+++ b/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs
@@ -1,7 +1,9 @@
using AIStudio.Provider;
using AIStudio.Provider.OpenAI;
using AIStudio.Settings;
+using AIStudio.Tools.Rust;
using AIStudio.Tools.Security;
+using AIStudio.Tools.Services;
namespace AIStudio.Chat;
@@ -14,14 +16,16 @@ public static class ListContentBlockExtensions
/// A function that transforms each content block into a message result asynchronously.
/// The configured provider, whose model is being written to.
/// A factory function to create text sub-content.
- /// A factory function to create image sub-content.
+ /// A factory function to create image sub-content from an image prepared for the model.
+ /// The cancellation token.
/// An asynchronous task that resolves to a list of transformed results.
public static async Task> BuildMessagesAsync(
this List blocks,
AIStudio.Settings.Provider provider,
Func roleTransformer,
Func textSubContentFactory,
- Func> imageSubContentFactory)
+ Func imageSubContentFactory,
+ CancellationToken token = default)
{
//
// Asked through the configured provider, so that what a person set in their expert settings
@@ -59,7 +63,7 @@ public static class ListContentBlockExtensions
// Text prompt with images as attachments, and the provider/model can process images:
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && text.FileAttachments.ContainsImages():
- messageTaskList.Add(CreateMultimodalMessageAsync(block, text, textSubContentFactory, imageSubContentFactory));
+ messageTaskList.Add(CreateMultimodalMessageAsync(block, text));
break;
}
}
@@ -81,34 +85,41 @@ public static class ListContentBlockExtensions
}
// Local function to create a multimodal message asynchronously.
- Task CreateMultimodalMessageAsync(
- ContentBlock block,
- ContentText text,
- Func innerTextSubContentFactory,
- Func> innerImageSubContentFactory)
+ Task CreateMultimodalMessageAsync(ContentBlock block, ContentText text)
{
return Task.Run(async () =>
{
- var imagesTasks = text.FileAttachments
+ var images = await Task.WhenAll(text.FileAttachments
.Where(x => x is { IsImage: true, Exists: true })
.Cast()
- .Select(innerImageSubContentFactory)
- .ToList();
-
- Task.WaitAll(imagesTasks);
- var images = imagesTasks.Select(t => t.Result).ToList();
+ .Select(CreateImageSubContentAsync));
return new MultimodalMessage
{
Role = roleTransformer(block.Role),
Content =
[
- innerTextSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
- ..images,
+ textSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
+ ..images.OfType(),
]
} as IMessageBase;
});
}
+
+ //
+ // Local function to prepare one image for the model. An image which cannot be prepared
+ // stays out of the message, and the user learns why. It used to go along as an image
+ // without data, which the providers refuse together with the whole request.
+ //
+ async Task CreateImageSubContentAsync(FileAttachmentImage image)
+ {
+ var prepared = await Program.SERVICE_PROVIDER.GetRequiredService().PrepareAsync(image, provider, token);
+ if (prepared.IsPrepared)
+ return imageSubContentFactory(prepared);
+
+ await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, prepared.ErrorCode.ToUserMessage(image.FileName)));
+ return null;
+ }
}
///
@@ -158,20 +169,16 @@ public static class ListContentBlockExtensions
Text = text,
};
- private static async Task DirectImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrl
+ private static ISubContent DirectImageSubContentFactory(PreparedModelImage image) => new SubContentImageUrl
{
- ImageUrl = await attachment.TryAsBase64() is (true, var base64Content)
- ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
- : string.Empty,
+ ImageUrl = image.ToDataUrl(),
};
- private static async Task NestedImageSubContentFactory(FileAttachmentImage attachment) => new SubContentImageUrlNested
+ private static ISubContent NestedImageSubContentFactory(PreparedModelImage image) => new SubContentImageUrlNested
{
ImageUrl = new SubContentImageUrlData
{
- Url = await attachment.TryAsBase64() is (true, var base64Content)
- ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
- : string.Empty,
+ Url = image.ToDataUrl(),
},
};
diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor
index 52d51ad96..c16ca3f6c 100644
--- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor
+++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor
@@ -21,7 +21,7 @@
-
+
diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua
index 7a5744216..5b3e9ae1f 100644
--- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua
+++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua
@@ -650,7 +650,7 @@ CONFIG["SETTINGS"] = {}
--
-- Configure the default file patterns and whether subfolders as well as hidden files and folders are included.
-- Separate multiple patterns with semicolons.
--- CONFIG["SETTINGS"]["DataBatchProcessing.FilePatterns"] = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.jpg;*.jpeg;*.png;*.webp;*.gif;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm"
+-- CONFIG["SETTINGS"]["DataBatchProcessing.FilePatterns"] = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.jpg;*.jpeg;*.png;*.webp;*.gif;*.bmp;*.tif;*.tiff;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm"
-- CONFIG["SETTINGS"]["DataBatchProcessing.IncludeSubdirectories"] = false
-- CONFIG["SETTINGS"]["DataBatchProcessing.IncludeHiddenFilesAndFolders"] = false
--
diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
index 753ac48cb..424aeee7e 100644
--- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
+++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
@@ -534,8 +534,8 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
-- Was not able to write the result file: {0}
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Die Ergebnisdatei konnte nicht geschrieben werden: {0}"
--- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Bilder im JPG-, PNG-, WebP- oder GIF-Format werden direkt an das Modell übergeben, daher benötigen Sie ein Modell, das Bilder versteht. Falls das ausgewählte Modell keine Bilder lesen kann, entfernen Sie die Dateimuster für Bilder, z. B. *.jpg oder *.png, bevor Sie starten."
+-- Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3011193639"] = "Bilder im JPG-, PNG-, WebP-, GIF-, BMP- oder TIFF-Format werden als Bilder an das Modell übergeben, daher benötigen Sie ein Modell, das Bilder versteht. Falls das ausgewählte Modell keine Bilder lesen kann, entfernen Sie die Dateimuster für Bilder, z. B. *.jpg oder *.png, bevor Sie starten."
-- The batch run finished, but one file could not be processed. See the progress table and log for details.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "Die Stapelverarbeitung ist abgeschlossen, aber eine Datei konnte nicht verarbeitet werden. Weitere Informationen finden Sie in der Fortschrittstabelle und im Protokoll."
@@ -663,9 +663,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
-- Start batch processing
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Stapelverarbeitung starten"
--- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "Dieses Bildformat kann nicht an das Modell übergeben werden. Bitte konvertieren Sie das Bild in JPG, PNG, WebP oder GIF."
-
-- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Die Ergebnistabelle des vorherigen Durchlaufs konnte nicht gelesen werden. Die bereits abgeschlossenen Dokumente können nicht wiederhergestellt werden und werden erneut verarbeitet."
@@ -690,9 +687,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
-- Include hidden files and folders?
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Versteckte Dateien und Ordner einbeziehen?"
--- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "Das Bild überschreitet die maximale Größe von {0} MB, die der ausgewählte Anbieter pro Bild akzeptiert."
-
-- Restore default
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Standard wiederherstellen"
@@ -8592,9 +8586,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
-- Default minimum pause between files
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Standardmäßige Mindestpause zwischen Dateien"
--- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Trennen Sie mehrere Dateimuster durch ein Semikolon, z. B. *.pdf;*.docx. Die Standardmuster umfassen alle unterstützten Audio- und Videoformate sowie Bilder im JPG-, PNG-, WebP- und GIF-Format."
-
-- Instructions
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Anweisungen"
@@ -8637,6 +8628,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T22
-- Hidden files and folders are included
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2288334048"] = "Versteckte Dateien und Ordner werden einbezogen"
+-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T232588799"] = "Trennen Sie mehrere Dateimuster durch ein Semikolon, z. B. *.pdf;*.docx. Die Standardmuster umfassen alle unterstützten Audio- und Videoformate sowie Bilder im JPG-, PNG-, WebP-, GIF-, BMP- und TIFF-Format."
+
-- The lower end of the random pause interval. AI Studio never allows less than 6 seconds.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T237774509"] = "Das untere Ende des Intervalls für die zufällige Pause. AI Studio erlaubt niemals weniger als 6 Sekunden."
@@ -13443,6 +13437,30 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Bild für KI-
-- Plugin archive
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin-Archiv"
+-- AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1345506748"] = "AI Studio konnte das Bild „{0}“ nicht für die KI vorbereiten und hat es daher nicht gesendet."
+
+-- The image '{0}' could not be made small enough for the selected provider, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1463083567"] = "Das Bild „{0}“ konnte für den ausgewählten Anbieter nicht ausreichend verkleinert werden und wurde daher nicht gesendet."
+
+-- The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2022724454"] = "Das Bild „{0}“ konnte nicht geöffnet und daher nicht gesendet werden. Wenn das Bild auf einem Netzlaufwerk gespeichert ist, ist das Laufwerk möglicherweise nicht verfügbar."
+
+-- The image '{0}' does not exist anymore and was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2330229240"] = "Das Bild „{0}“ existiert nicht mehr und wurde nicht gesendet."
+
+-- The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T3457568536"] = "Das Bild „{0}“ hat nicht das Format, das seine Dateiendung angibt, und wurde daher nicht gesendet. Bitte korrigieren Sie die Dateiendung."
+
+-- The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T4029166607"] = "Das Bild „{0}“ konnte nicht gelesen und daher nicht gesendet werden. Möglicherweise ist es beschädigt oder unvollständig übertragen worden."
+
+-- The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T567941882"] = "Das Bild „{0}“ hat zu viele Pixel, als dass AI Studio es vorbereiten könnte, und wurde daher nicht gesendet. Bitte verkleinern Sie das Bild zuerst."
+
+-- The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T568830141"] = "Das Bild „{0}“ liegt in einem Format vor, das AI Studio nicht an ein KI-Modell übergeben kann, und wurde daher nicht gesendet. Bitte konvertieren Sie das Bild in JPG oder PNG."
+
-- Attempt to override instructions
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONFINDINGCATEGORYEXTENSIONS::T161976090"] = "Versuch, Anweisungen zu überschreiben"
@@ -14673,11 +14691,8 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T90753429"]
-- Please enter the secret necessary for authentication.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T968385876"] = "Bitte geben Sie das für die Authentifizierung benötigte Geheimnis ein."
--- Unsupported image format
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1398282880"] = "Nicht unterstütztes Bildformat"
-
--- File has no extension
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1555980031"] = "Datei hat keine Erweiterung"
+-- AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1578559061"] = "AI Studio kann keine Bilder im Format {0} an ein KI-Modell übergeben. Bitte konvertieren Sie das Bild in JPG oder PNG."
-- Audio files are not supported yet
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T2919730669"] = "Audio-Dateien werden noch nicht unterstützt."
diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
index b34e8b6ba..d3210b17e 100644
--- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
+++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
@@ -534,8 +534,8 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
-- Was not able to write the result file: {0}
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Was not able to write the result file: {0}"
--- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
+-- Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3011193639"] = "Images in JPG, PNG, WebP, GIF, BMP, or TIFF format are given to the model as images, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
-- The batch run finished, but one file could not be processed. See the progress table and log for details.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "The batch run finished, but one file could not be processed. See the progress table and log for details."
@@ -663,9 +663,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
-- Start batch processing
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Start batch processing"
--- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."
-
-- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again."
@@ -690,9 +687,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
-- Include hidden files and folders?
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Include hidden files and folders?"
--- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
-UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "The image is larger than {0} MB, which is the most the selected provider accepts for one image."
-
-- Restore default
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Restore default"
@@ -8592,9 +8586,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
-- Default minimum pause between files
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Default minimum pause between files"
--- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format."
-
-- Instructions
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Instructions"
@@ -8637,6 +8628,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T22
-- Hidden files and folders are included
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2288334048"] = "Hidden files and folders are included"
+-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T232588799"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, GIF, BMP, and TIFF format."
+
-- The lower end of the random pause interval. AI Studio never allows less than 6 seconds.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T237774509"] = "The lower end of the random pause interval. AI Studio never allows less than 6 seconds."
@@ -13443,6 +13437,30 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Image for AI
-- Plugin archive
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin archive"
+-- AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1345506748"] = "AI Studio was not able to prepare the image '{0}' for the AI, so it was not sent."
+
+-- The image '{0}' could not be made small enough for the selected provider, so it was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T1463083567"] = "The image '{0}' could not be made small enough for the selected provider, so it was not sent."
+
+-- The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2022724454"] = "The image '{0}' could not be opened and was not sent. When the image is stored on a network drive, the drive might be unavailable."
+
+-- The image '{0}' does not exist anymore and was not sent.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T2330229240"] = "The image '{0}' does not exist anymore and was not sent."
+
+-- The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T3457568536"] = "The image '{0}' is not what its file extension claims, so it was not sent. Please correct its file extension."
+
+-- The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T4029166607"] = "The image '{0}' could not be read and was not sent. It might be damaged or transferred incompletely."
+
+-- The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T567941882"] = "The image '{0}' has too many pixels for AI Studio to prepare it, so it was not sent. Please scale the image down first."
+
+-- The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::MODELIMAGEERRORCODEEXTENSIONS::T568830141"] = "The image '{0}' uses a format which AI Studio cannot give to an AI model, so it was not sent. Please convert the image to JPG or PNG."
+
-- Attempt to override instructions
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONFINDINGCATEGORYEXTENSIONS::T161976090"] = "Attempt to override instructions"
@@ -14673,11 +14691,8 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T90753429"]
-- Please enter the secret necessary for authentication.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T968385876"] = "Please enter the secret necessary for authentication."
--- Unsupported image format
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1398282880"] = "Unsupported image format"
-
--- File has no extension
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1555980031"] = "File has no extension"
+-- AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T1578559061"] = "AI Studio cannot give {0} images to an AI model. Please convert the image to JPG or PNG."
-- Audio files are not supported yet
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T2919730669"] = "Audio files are not supported yet"
diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs
index 39700b223..26d47ca32 100644
--- a/app/MindWork AI Studio/Program.cs
+++ b/app/MindWork AI Studio/Program.cs
@@ -175,6 +175,7 @@ internal sealed class Program
builder.Services.AddSingleton();
builder.Services.AddSingleton();
builder.Services.AddSingleton();
+ builder.Services.AddSingleton();
builder.Services.AddSingleton();
builder.Services.AddSingleton();
builder.Services.AddSingleton();
diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs
index df3e66612..dfa60864a 100644
--- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs
+++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs
@@ -59,18 +59,16 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
Text = text,
},
- // Anthropic-specific image sub-content:
- async attachment => new SubContentImage
+ // Anthropic-specific image sub-content, which takes the base64 and its type apart:
+ image => new SubContentImage
{
Source = new SubContentBase64Image
{
- Data = await attachment.TryAsBase64(token: token) is (true, var base64Content)
- ? base64Content
- : string.Empty,
-
- MediaType = attachment.DetermineMimeType(),
+ Data = image.Base64,
+ MediaType = image.MimeType,
}
- }
+ },
+ token
);
//
diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs
index 33a4d9823..13e8eb4f1 100644
--- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs
+++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs
@@ -113,15 +113,14 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
{
Text = text,
},
- async attachment => new SubContentImageUrlNested
+ image => new SubContentImageUrlNested
{
ImageUrl = new SubContentImageUrlData
{
- Url = await attachment.TryAsBase64(token: token) is (true, var base64Content)
- ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
- : string.Empty,
+ Url = image.ToDataUrl(),
},
- });
+ },
+ token);
return new ChatCompletionAPIRequest
{
@@ -192,12 +191,11 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
{
Text = text,
},
- async attachment => new SubContentInputImage
+ image => new SubContentInputImage
{
- ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content)
- ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
- : string.Empty,
- });
+ ImageUrl = image.ToDataUrl(),
+ },
+ token);
var baseInput = new List
[TestFixture]
public sealed class ListContentBlockExtensionsTests
{
+ private string picturePath = string.Empty;
+ private FakeImageRuntime runtime = null!;
+ private RustService rustService = null!;
+ private ServiceProvider serviceProvider = null!;
+ private IServiceProvider previousServiceProvider = null!;
+
+ [SetUp]
+ public void PrepareServices()
+ {
+ this.picturePath = Path.Combine(Path.GetTempPath(), $"picture-{Guid.NewGuid():N}.webp");
+ File.WriteAllBytes(this.picturePath, [1, 2, 3]);
+
+ this.runtime = new FakeImageRuntime();
+ this.rustService = new RustService(this.runtime);
+ this.previousServiceProvider = Program.SERVICE_PROVIDER;
+ this.serviceProvider = new ServiceCollection()
+ .AddSingleton(this.rustService)
+ .AddSingleton()
+ .AddLogging()
+ .BuildServiceProvider();
+
+ Program.SERVICE_PROVIDER = this.serviceProvider;
+ }
+
+ [TearDown]
+ public void RestoreApplicationState()
+ {
+ Program.SERVICE_PROVIDER = this.previousServiceProvider;
+ this.serviceProvider.Dispose();
+ this.rustService.Dispose();
+ this.runtime.Dispose();
+ File.Delete(this.picturePath);
+ }
+
[Test]
public async Task ImageInputSwitchedOnByHandReachesTheMessageAsWell()
{
@@ -31,6 +73,32 @@ public sealed class ListContentBlockExtensionsTests
Assert.That(messages.Single(), Is.InstanceOf(), "The picture is part of the message because the person said this model can read one.");
}
+ [Test]
+ public async Task APictureWhichCannotBePreparedLeavesNoEmptyPart()
+ {
+ this.runtime.ErrorCode = "NOT_DECODABLE";
+
+ var messages = await BlocksWithAPicture(this.picturePath).BuildMessagesUsingNestedImageUrlAsync(PictureReadingProvider());
+
+ var message = messages.Single() as MultimodalMessage;
+ Assert.That(message, Is.Not.Null, "The text still goes to the model.");
+ Assert.That(message!.Content, Has.Count.EqualTo(1).And.All.InstanceOf(), "A picture without data makes the provider refuse the whole request.");
+ }
+
+ [Test]
+ public async Task TheTypeOfAPictureComesFromItsPreparation()
+ {
+ // The file is a WebP, which the runtime prepared as JPEG:
+ var messages = await BlocksWithAPicture(this.picturePath).BuildMessagesUsingDirectImageUrlAsync(PictureReadingProvider());
+
+ var picture = (messages.Single() as MultimodalMessage)!.Content.OfType().Single();
+ Assert.Multiple(() =>
+ {
+ Assert.That(picture.ImageUrl, Is.EqualTo($"data:image/jpeg;base64,{FakeImageRuntime.IMAGE_BASE64}"));
+ Assert.That(this.runtime.RequestedPaths, Is.EqualTo(new[] { new FileInfo(this.picturePath).FullName }));
+ });
+ }
+
[Test]
public async Task WithoutThatSwitchThePictureStaysOut()
{
@@ -41,11 +109,20 @@ public sealed class ListContentBlockExtensionsTests
Assert.That(messages.Single(), Is.InstanceOf(), "Nothing says this model reads pictures, so the text goes on its own.");
}
+ ///
+ /// A model which reads text only, according to the rules, but which a person said reads pictures.
+ ///
+ private static AIStudio.Settings.Provider PictureReadingProvider() => new(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
+ {
+ CapabilityOverrides = new() { MultipleImageInput = true },
+ };
+
///
/// One block of text with a picture hanging on it.
///
+ /// Where the picture lies.
/// The blocks.
- private static List BlocksWithAPicture() =>
+ private static List BlocksWithAPicture(string picturePath = "/tmp/picture.png") =>
[
new()
{
@@ -54,12 +131,12 @@ public sealed class ListContentBlockExtensionsTests
Content = new ContentText
{
Text = "What is in this picture?",
- FileAttachments = [new FileAttachmentImage("picture.png", "/tmp/picture.png", 1_024)],
+ FileAttachments = [new FileAttachmentImage(Path.GetFileName(picturePath), picturePath, 1_024)],
},
},
];
private static ISubContent Text(string text) => new SubContentText { Text = text };
- private static Task Picture(FileAttachmentImage image) => Task.FromResult(new SubContentText { Text = image.FileName });
+ private static ISubContent Picture(PreparedModelImage image) => new SubContentText { Text = image.MimeType };
}
\ No newline at end of file
diff --git a/app/Tests/Provider/ImageSizeLimitTests.cs b/app/Tests/Provider/ImageSizeLimitTests.cs
index fd4911c4d..d7f836e8e 100644
--- a/app/Tests/Provider/ImageSizeLimitTests.cs
+++ b/app/Tests/Provider/ImageSizeLimitTests.cs
@@ -1,5 +1,6 @@
using AIStudio.Chat;
using AIStudio.Provider;
+using AIStudio.Provider.SelfHosted;
namespace AIStudio.Tests.Provider;
@@ -7,8 +8,8 @@ namespace AIStudio.Tests.Provider;
/// Checks how large an image may be when we send it through a provider.
///
///
-/// A provider states its limit or says nothing. Either way, TryAsBase64 skips every image above our
-/// own limit, so a larger limit of a provider must never reach the checks done before sending.
+/// A provider or a self-hosted engine states its limit or says nothing. Either way, we never send
+/// more than our own limit, so a larger limit of a provider must never reach the image preparation.
///
[TestFixture]
public sealed class ImageSizeLimitTests
@@ -19,7 +20,7 @@ public sealed class ImageSizeLimitTests
Assert.Multiple(() =>
{
Assert.That(LLMProviders.ANTHROPIC.GetMaxImageSizeBytes(), Is.EqualTo(3_932_160));
- Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(LLMProviders.ANTHROPIC), Is.EqualTo(3_932_160));
+ Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.ANTHROPIC)), Is.EqualTo(3_932_160));
});
}
@@ -29,17 +30,41 @@ public sealed class ImageSizeLimitTests
Assert.Multiple(() =>
{
Assert.That(LLMProviders.OPEN_AI.GetMaxImageSizeBytes(), Is.Null, "Nobody stated a limit, and no number stands in for that.");
- Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(LLMProviders.OPEN_AI), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
+ Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.OPEN_AI)), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
});
}
[Test]
- public void NoProviderGetsMoreThanTryAsBase64Sends()
+ public void LlmmanLeavesRoomForTheTextOfTheConversation()
{
Assert.Multiple(() =>
{
- foreach (var provider in Enum.GetValues())
- Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(provider), Is.LessThanOrEqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), provider.ToString());
+ Assert.That(Host.LLMMAN.GetMaxImageSizeBytes(), Is.EqualTo(1_048_576));
+ Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.SELF_HOSTED, Host.LLMMAN)), Is.EqualTo(1_048_576));
+ Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(LLMProviders.SELF_HOSTED, Host.OLLAMA)), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), "Ollama states no limit we know of.");
});
}
+
+ [Test]
+ public void AnEngineCountsOnlyForSelfHostedProviders()
+ {
+ // A provider keeps the host it was once configured with, even after somebody switched it
+ // to a cloud provider in the dialog:
+ var cloudProviderWithAnOldHost = ProviderOf(LLMProviders.OPEN_AI, Host.LLMMAN);
+
+ Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(cloudProviderWithAnOldHost), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
+ }
+
+ [Test]
+ public void NoProviderGetsMoreThanOurOwnLimit()
+ {
+ Assert.Multiple(() =>
+ {
+ foreach (var llmProvider in Enum.GetValues())
+ foreach (var host in Enum.GetValues())
+ Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(ProviderOf(llmProvider, host)), Is.LessThanOrEqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), $"{llmProvider} with {host}");
+ });
+ }
+
+ private static AIStudio.Settings.Provider ProviderOf(LLMProviders llmProvider, Host host = Host.NONE) => new(0, "test", "Test", llmProvider, new AIStudio.Provider.Model("model", null), Host: host);
}
\ No newline at end of file
diff --git a/app/Tests/Tools/FakeImageRuntime.cs b/app/Tests/Tools/FakeImageRuntime.cs
new file mode 100644
index 000000000..fb919d607
--- /dev/null
+++ b/app/Tests/Tools/FakeImageRuntime.cs
@@ -0,0 +1,60 @@
+using System.Collections.Concurrent;
+using System.Net;
+using System.Text;
+using System.Text.Json;
+
+namespace AIStudio.Tests.Tools;
+
+///
+/// Plays the image route of the runtime for a test, and notes what the app asked for.
+///
+///
+/// It prepares nothing: every image comes back as the same three bytes, with the code and type the
+/// test sets. The answer is spelled out in the wire format of the runtime, so a test fails as soon
+/// as the app stops understanding what the runtime actually sends.
+///
+internal sealed class FakeImageRuntime : HttpMessageHandler
+{
+ ///
+ /// The three bytes every prepared image consists of, as base64.
+ ///
+ public const string IMAGE_BASE64 = "AQID";
+
+ private readonly ConcurrentQueue requestedPaths = new();
+ private readonly ConcurrentQueue requestedBudgets = new();
+
+ ///
+ /// The code the runtime answers with, as the runtime spells it.
+ ///
+ public string ErrorCode { get; set; } = "NONE";
+
+ ///
+ /// The MIME type the runtime reports for every prepared image.
+ ///
+ public string MimeType { get; set; } = "image/jpeg";
+
+ ///
+ /// The paths of the images the app asked to prepare, in order.
+ ///
+ public IReadOnlyList RequestedPaths => [..this.requestedPaths];
+
+ ///
+ /// The byte budgets the app asked for, in order.
+ ///
+ public IReadOnlyList RequestedBudgets => [..this.requestedBudgets];
+
+ protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+ {
+ if (request.RequestUri!.AbsolutePath is not "/image/prepare-for-model")
+ return new HttpResponseMessage(HttpStatusCode.NotFound);
+
+ using (var body = JsonDocument.Parse(await request.Content!.ReadAsStringAsync(cancellationToken)))
+ {
+ this.requestedPaths.Enqueue(body.RootElement.GetProperty("path").GetString() ?? string.Empty);
+ this.requestedBudgets.Enqueue(body.RootElement.GetProperty("max_bytes").GetInt64());
+ }
+
+ var answer = JsonSerializer.Serialize(new { error_code = this.ErrorCode, base64 = IMAGE_BASE64, mime_type = this.MimeType, width = 1, height = 1, was_resized = false });
+ return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(answer, Encoding.UTF8, "application/json") };
+ }
+}
\ No newline at end of file
diff --git a/app/Tests/Tools/ModelImageFormatTests.cs b/app/Tests/Tools/ModelImageFormatTests.cs
new file mode 100644
index 000000000..b0c074d1b
--- /dev/null
+++ b/app/Tests/Tools/ModelImageFormatTests.cs
@@ -0,0 +1,46 @@
+using AIStudio.Chat;
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tests.Tools;
+
+///
+/// Checks which image formats may be attached for an AI model.
+///
+///
+/// The runtime decides which formats it can prepare for a model, see prepare_image_for_model in
+/// runtime/src/image.rs. An image the app lets through but the runtime cannot prepare would only
+/// fail once the message is sent; one the app refuses although the runtime could prepare it would
+/// be refused for nothing.
+///
+[TestFixture]
+public sealed class ModelImageFormatTests
+{
+ ///
+ /// The extensions model_input_format in runtime/src/image.rs accepts.
+ ///
+ private static readonly string[] RUNTIME_MODEL_FORMATS = ["jpg", "jpeg", "png", "webp", "gif", "bmp", "tif", "tiff"];
+
+ [Test]
+ public void EveryFormatTheRuntimePreparesMayBeAttached()
+ {
+ Assert.That(FileTypes.MODEL_IMAGE.FlattenExtensions(), Is.EquivalentTo(RUNTIME_MODEL_FORMATS));
+ }
+
+ [Test]
+ public void OnlyFormatsWithoutADecoderAreRefused()
+ {
+ var refused = FileTypes.IMAGE.FlattenExtensions().Except(FileTypes.MODEL_IMAGE.FlattenExtensions());
+
+ Assert.That(refused, Is.EquivalentTo(new[] { "svg", "heic", "avif" }));
+ }
+
+ [Test]
+ public void ATifFileIsAnImage()
+ {
+ Assert.Multiple(() =>
+ {
+ Assert.That(FileAttachment.FromPath("/tmp/scan.tif").IsImage, Is.True);
+ Assert.That(FileTypes.IsAllowedPath("/tmp/scan.tif", FileTypes.MODEL_IMAGE), Is.True);
+ });
+ }
+}
\ No newline at end of file
diff --git a/app/Tests/Tools/ModelImageServiceTests.cs b/app/Tests/Tools/ModelImageServiceTests.cs
new file mode 100644
index 000000000..30f3704a5
--- /dev/null
+++ b/app/Tests/Tools/ModelImageServiceTests.cs
@@ -0,0 +1,132 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools;
+
+///
+/// Checks when the image preparation asks the runtime again, and when it answers from its cache.
+///
+///
+/// Every message prepares all images of the conversation anew. Asking the runtime each time would
+/// decode every photo again; answering from the cache after the file changed would send the old
+/// picture. The runtime is played in its wire format, so these tests also fail as soon as the app
+/// stops understanding what the runtime sends.
+///
+[TestFixture]
+public sealed class ModelImageServiceTests
+{
+ private string imagePath = string.Empty;
+ private FakeImageRuntime runtime = null!;
+ private RustService rustService = null!;
+ private ModelImageService service = null!;
+
+ [SetUp]
+ public void PrepareService()
+ {
+ this.imagePath = Path.Combine(Path.GetTempPath(), $"model-image-{Guid.NewGuid():N}.png");
+ File.WriteAllBytes(this.imagePath, [1, 2, 3]);
+
+ this.runtime = new FakeImageRuntime();
+ this.rustService = new RustService(this.runtime);
+ this.service = new ModelImageService(this.rustService, NullLogger.Instance);
+ }
+
+ [TearDown]
+ public void CleanUp()
+ {
+ this.service.Dispose();
+ this.rustService.Dispose();
+ this.runtime.Dispose();
+ File.Delete(this.imagePath);
+ }
+
+ [Test]
+ public async Task TheSameFileForTheSameProviderIsPreparedOnce()
+ {
+ var first = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+ var second = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(first.IsPrepared, Is.True);
+ Assert.That(first.MimeType, Is.EqualTo("image/jpeg"));
+ Assert.That(second, Is.SameAs(first));
+ Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(1), "The second message decoded the photo again.");
+ });
+ }
+
+ [Test]
+ public async Task AChangedFileIsPreparedAgain()
+ {
+ await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+ await File.WriteAllBytesAsync(this.imagePath, [1, 2, 3, 4]);
+ await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+ Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(2), "The model would get the picture as it was before.");
+ }
+
+ [Test]
+ public async Task AnotherBudgetIsPreparedAgainWithThatBudget()
+ {
+ await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+ await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.OPEN_AI));
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(this.runtime.RequestedBudgets, Is.EqualTo(new[] { 3_932_160, IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES }));
+ Assert.That(this.runtime.RequestedPaths, Is.All.EqualTo(new FileInfo(this.imagePath).FullName));
+ });
+ }
+
+ [Test]
+ public async Task AFailureIsTriedAgainWithTheNextMessage()
+ {
+ this.runtime.ErrorCode = "NOT_DECODABLE";
+ var failed = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+ this.runtime.ErrorCode = "NONE";
+ var prepared = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(failed.ErrorCode, Is.EqualTo(ModelImageErrorCode.NOT_DECODABLE));
+ Assert.That(failed.IsPrepared, Is.False);
+ Assert.That(prepared.IsPrepared, Is.True);
+ Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(2), "A failure must not stick to the image.");
+ });
+ }
+
+ [Test]
+ public async Task ACodeFromANewerRuntimeNeverCountsAsAPreparedImage()
+ {
+ this.runtime.ErrorCode = "SOMETHING_NEW";
+ var image = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(image.ErrorCode, Is.EqualTo(ModelImageErrorCode.UNKNOWN));
+ Assert.That(image.IsPrepared, Is.False, "The base64 of the answer must not go to the model when the code says nothing we know.");
+ });
+ }
+
+ [Test]
+ public async Task AMissingFileDoesNotReachTheRuntime()
+ {
+ File.Delete(this.imagePath);
+ var image = await this.service.PrepareAsync(this.Image, ProviderOf(LLMProviders.ANTHROPIC));
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(image.ErrorCode, Is.EqualTo(ModelImageErrorCode.FILE_NOT_FOUND));
+ Assert.That(this.runtime.RequestedPaths, Is.Empty);
+ });
+ }
+
+ private FileAttachmentImage Image => new("picture.png", this.imagePath, 3);
+
+ private static AIStudio.Settings.Provider ProviderOf(LLMProviders llmProvider) => new(0, "test", "Test", llmProvider, new Model("model", null));
+}
\ No newline at end of file
diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index cc6e3609d..3111ad423 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -77,7 +77,7 @@ qdrant-edge = "0.8.0"
regex = "1.13.1"
aho-corasick = "1.1.5"
toml = "1.1.4"
-image = { version = "0.25.10", default-features = false, features = ["jpeg", "png", "webp"] }
+image = { version = "0.25.10", default-features = false, features = ["jpeg", "png", "webp", "gif", "bmp", "tiff"] }
tokenizers = "0.23.1"
[dev-dependencies]
diff --git a/runtime/src/image.rs b/runtime/src/image.rs
index 78439d67a..7ce6abb51 100644
--- a/runtime/src/image.rs
+++ b/runtime/src/image.rs
@@ -3,6 +3,9 @@
//! This module is deliberately free of any feature-specific behavior so that every part of
//! AI Studio that needs an embeddable image can use it. The size policy is a single maximum edge
//! length; callers that want the original bytes pass `optimize = false`.
+//!
+//! Images for AI models take a route of their own, see [`prepare_image_for_model`]: they always
+//! leave as JPEG or PNG and have to fit into the byte budget of the provider.
use std::io::Cursor;
use std::path::Path;
@@ -12,8 +15,10 @@ use axum::http::StatusCode;
use base64::{Engine as _, engine::general_purpose};
use image::codecs::jpeg::JpegEncoder;
use image::imageops::FilterType;
-use image::{DynamicImage, ImageFormat, ImageReader};
+use image::{DynamicImage, ImageDecoder, ImageError, ImageFormat, ImageReader, Limits, Rgb, RgbImage};
+use log::{info, warn};
use serde::{Deserialize, Serialize};
+use tokio::sync::Semaphore;
/// The longest edge an optimized image may have. Larger images are scaled down proportionally.
const MAX_EDGE_PIXELS: u32 = 2_560;
@@ -86,29 +91,14 @@ fn prepare_image_sync(
}
let format = supported_format(path)?;
- let reader = ImageReader::open(path)
- .and_then(|reader| reader.with_guessed_format())
- .map_err(|error| {
- (
- StatusCode::BAD_REQUEST,
- format!("The image could not be opened: {error}"),
- )
- })?;
-
- if reader.format() != Some(format) {
- return Err((
- StatusCode::BAD_REQUEST,
- "The image content does not match its file extension.".to_string(),
- ));
- }
- let decoded = reader.decode().map_err(|error| {
- (
- StatusCode::BAD_REQUEST,
- format!("The image could not be decoded: {error}"),
- )
- })?;
-
+ //
+ // Upright as the photo was taken: re-encoding drops the EXIF data which told a viewer how to
+ // turn it. The original bytes keep that note, and browsers apply it, so the dimensions we
+ // report fit both ways:
+ //
+ let decoded = decode_oriented(path, format).map_err(|failure| (StatusCode::BAD_REQUEST, failure.message))?;
+
let original_width = decoded.width();
let original_height = decoded.height();
let should_resize = request.optimize && original_width.max(original_height) > MAX_EDGE_PIXELS;
@@ -179,9 +169,16 @@ fn supported_format(path: &Path) -> Result {
///
/// The aspect ratio is preserved, and both edges stay at least one pixel wide.
fn resize_to_max_edge(image: DynamicImage) -> DynamicImage {
+ resize_to_longest_edge(image, MAX_EDGE_PIXELS)
+}
+
+/// Scales an image so that its longest edge equals the given length.
+///
+/// The aspect ratio is preserved, and both edges stay at least one pixel wide.
+fn resize_to_longest_edge(image: DynamicImage, longest_edge: u32) -> DynamicImage {
let width = image.width();
let height = image.height();
- let scale = MAX_EDGE_PIXELS as f64 / width.max(height) as f64;
+ let scale = longest_edge as f64 / width.max(height) as f64;
let target_width = (width as f64 * scale).round().max(1.0) as u32;
let target_height = (height as f64 * scale).round().max(1.0) as u32;
image.resize_exact(target_width, target_height, FilterType::Lanczos3)
@@ -218,6 +215,426 @@ fn encode(image: &DynamicImage, format: ImageFormat) -> Result, (StatusC
Ok(bytes)
}
+//
+// Preparation for AI models
+//
+
+/// The JPEG qualities tried, in this order, while an image does not fit into its byte budget.
+///
+/// Below the last one, compression artifacts start to blur small text such as the labels of a
+/// diagram. A smaller image keeps such text more legible than a still lower quality does, which
+/// is why the budget shrinks the image instead from there on.
+const BUDGET_JPEG_QUALITIES: [u8; 3] = [JPEG_QUALITY, 70, 55];
+
+/// How much the longest edge shrinks in each round, once no quality fits into the budget.
+const BUDGET_EDGE_FACTOR: f64 = 0.75;
+
+/// The longest edge the budget may shrink an image to. Models no longer read the details of a
+/// photo or a document below that, so such an image is refused instead of sent unreadable.
+const MIN_BUDGET_EDGE_PIXELS: u32 = 1_024;
+
+/// How many images are prepared for models at the same time.
+///
+/// A request with several photos would otherwise decode all of them at once, and every decoded
+/// photo holds a pixel buffer of up to the decoder's limit of 512 MiB. That is too much for small
+/// devices such as a Raspberry Pi, while two at a time still keep a multi-core computer busy.
+const MAX_PARALLEL_MODEL_PREPARATIONS: usize = 2;
+
+/// Lets no more than [`MAX_PARALLEL_MODEL_PREPARATIONS`] images be prepared at once.
+static MODEL_PREPARATION_SLOTS: Semaphore = Semaphore::const_new(MAX_PARALLEL_MODEL_PREPARATIONS);
+
+/// Why an image could not be prepared, so the .NET app can tell the user in their language.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub enum ImageErrorCode {
+ /// The image was prepared.
+ None,
+
+ /// The path is not absolute, or no file exists there.
+ FileNotFound,
+
+ /// The file exists, but could not be opened, e.g., for lack of permission.
+ FileNotReadable,
+
+ /// AI Studio cannot decode this image format, e.g., HEIC, AVIF, or SVG.
+ UnsupportedFormat,
+
+ /// The content of the file does not match its extension.
+ ContentMismatch,
+
+ /// The file claims the right format but is damaged or uses a variant we cannot decode.
+ NotDecodable,
+
+ /// Decoding the image would need more memory than the decoder allows.
+ TooManyPixels,
+
+ /// Not even the smallest image and lowest quality we accept fit into the byte budget.
+ BudgetNotReachable,
+
+ /// Any failure which does not carry a code of its own, e.g., a failed encoder.
+ Internal,
+}
+
+/// A failed preparation: the code for the app and a message for the log.
+#[derive(Debug)]
+struct ImageFailure {
+ code: ImageErrorCode,
+ message: String,
+}
+
+impl ImageFailure {
+ fn new(code: ImageErrorCode, message: impl Into) -> Self {
+ Self { code, message: message.into() }
+ }
+}
+
+/// The request to prepare one local image file for an AI model.
+#[derive(Debug, Deserialize)]
+pub struct PrepareModelImageRequest {
+ /// The absolute path of the image file to read.
+ path: String,
+
+ /// The largest number of bytes the encoded image may have, before base64.
+ max_bytes: u64,
+}
+
+/// The image as it goes to the model, or why it cannot go.
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub struct PrepareModelImageResponse {
+ /// `None` when the image was prepared; then all other fields describe it.
+ error_code: ImageErrorCode,
+
+ /// The encoded image as base64, without a `data:` prefix. The MIME type travels next to it.
+ base64: String,
+
+ /// Either `image/jpeg` or `image/png`, describing the encoded bytes.
+ mime_type: String,
+
+ /// The width of the prepared image in pixels.
+ width: u32,
+
+ /// The height of the prepared image in pixels.
+ height: u32,
+
+ /// Whether the image was scaled down, by the size policy or to fit into the byte budget.
+ was_resized: bool,
+}
+
+impl PrepareModelImageResponse {
+ fn failure(code: ImageErrorCode) -> Self {
+ Self {
+ error_code: code,
+ base64: String::new(),
+ mime_type: String::new(),
+ width: 0,
+ height: 0,
+ was_resized: false,
+ }
+ }
+}
+
+/// The encoding an image leaves in for a model.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ModelEncoding {
+ Jpeg,
+ Png,
+}
+
+/// An image encoded for a model, before base64.
+struct EncodedModelImage {
+ bytes: Vec,
+ encoding: ModelEncoding,
+
+ /// The JPEG quality which fit into the budget; PNG has none.
+ jpeg_quality: Option,
+
+ width: u32,
+ height: u32,
+ was_resized: bool,
+}
+
+/// Prepares one local image for an AI model.
+///
+/// Every image is decoded, turned upright according to its EXIF orientation, scaled down to
+/// [`MAX_EDGE_PIXELS`], and encoded anew as JPEG or PNG, the two formats every provider and every
+/// self-hosted engine accepts. Encoding anew also drops the metadata of the file, among them the
+/// location where a photo was taken. When the result exceeds the byte budget, the quality and then
+/// the size go down until it fits.
+///
+/// The answer is always a success at the HTTP level; `error_code` tells whether the image was
+/// prepared. Decoding runs on a blocking worker, and only a few images at a time, see
+/// [`MAX_PARALLEL_MODEL_PREPARATIONS`].
+pub async fn prepare_image_for_model(
+ Json(request): Json,
+) -> Json {
+ //
+ // The semaphore is never closed, so acquiring it cannot fail. Should it fail anyway, the
+ // image is prepared without a slot rather than not at all:
+ //
+ let _slot = MODEL_PREPARATION_SLOTS.acquire().await.ok();
+ let response = tokio::task::spawn_blocking(move || prepare_image_for_model_sync(&request))
+ .await
+ .unwrap_or_else(|error| {
+ warn!("The image worker for an AI model failed: {error}");
+ PrepareModelImageResponse::failure(ImageErrorCode::Internal)
+ });
+
+ Json(response)
+}
+
+/// Performs the blocking part of [`prepare_image_for_model`].
+fn prepare_image_for_model_sync(request: &PrepareModelImageRequest) -> PrepareModelImageResponse {
+ match prepare_for_model(Path::new(&request.path), request.max_bytes) {
+ Ok(encoded) => PrepareModelImageResponse {
+ error_code: ImageErrorCode::None,
+ base64: general_purpose::STANDARD.encode(&encoded.bytes),
+ mime_type: match encoded.encoding {
+ ModelEncoding::Jpeg => "image/jpeg",
+ ModelEncoding::Png => "image/png",
+ }
+ .to_string(),
+ width: encoded.width,
+ height: encoded.height,
+ was_resized: encoded.was_resized,
+ },
+
+ Err(failure) => {
+ warn!("Preparing the image '{}' for an AI model failed ({:?}): {}", request.path, failure.code, failure.message);
+ PrepareModelImageResponse::failure(failure.code)
+ }
+ }
+}
+
+/// Decodes, orients, scales, and encodes one image for a model.
+fn prepare_for_model(path: &Path, max_bytes: u64) -> Result {
+ if !path.is_absolute() || !path.is_file() {
+ return Err(ImageFailure::new(ImageErrorCode::FileNotFound, "The image path is not an accessible absolute file path."));
+ }
+
+ let format = model_input_format(path)?;
+ let source_bytes = std::fs::metadata(path).map(|metadata| metadata.len()).unwrap_or(0);
+ let decoded = decode_oriented(path, format)?;
+ let (source_width, source_height) = (decoded.width(), decoded.height());
+ let was_resized = source_width.max(source_height) > MAX_EDGE_PIXELS;
+ let image = if was_resized {
+ resize_to_max_edge(decoded)
+ } else {
+ decoded
+ };
+
+ let encoding = model_encoding(format, &image);
+ let encoded = fit_into_budget(image, encoding, max_bytes, was_resized)?;
+ let encoded_bytes = encoded.bytes.len() as u64;
+ info!(
+ "Image prepared for model: path='{}', source={format:?} {source_width}x{source_height} {}, result={:?} {}x{} {} ({} of the original), quality={}, budget={}, resized={}",
+ path.display(),
+ megabytes(source_bytes),
+ encoded.encoding,
+ encoded.width,
+ encoded.height,
+ megabytes(encoded_bytes),
+ percentage(encoded_bytes, source_bytes),
+ encoded.jpeg_quality.map_or_else(|| "lossless".to_string(), |quality| quality.to_string()),
+ megabytes(max_bytes),
+ encoded.was_resized,
+ );
+
+ Ok(encoded)
+}
+
+/// Writes a number of bytes as megabytes, for the log.
+fn megabytes(bytes: u64) -> String {
+ format!("{:.2} MB", bytes as f64 / 1_000_000.0)
+}
+
+/// Writes how large a part is compared to the whole, for the log.
+fn percentage(part: u64, whole: u64) -> String {
+ if whole == 0 {
+ return "n/a".to_string();
+ }
+
+ format!("{:.0} %", part as f64 * 100.0 / whole as f64)
+}
+
+/// Maps a file extension to the image format we expect for a model input.
+///
+/// The result is only the expected format; [`decode_oriented`] still verifies it against the
+/// actual file content.
+fn model_input_format(path: &Path) -> Result {
+ match path
+ .extension()
+ .and_then(|extension| extension.to_str())
+ .map(str::to_ascii_lowercase)
+ .as_deref()
+ {
+ Some("jpg" | "jpeg") => Ok(ImageFormat::Jpeg),
+ Some("png") => Ok(ImageFormat::Png),
+ Some("webp") => Ok(ImageFormat::WebP),
+ Some("gif") => Ok(ImageFormat::Gif),
+ Some("bmp") => Ok(ImageFormat::Bmp),
+ Some("tif" | "tiff") => Ok(ImageFormat::Tiff),
+
+ _ => Err(ImageFailure::new(ImageErrorCode::UnsupportedFormat, "Images for AI models must be JPEG, PNG, WebP, GIF, BMP, or TIFF files.")),
+ }
+}
+
+/// Decodes an image of the expected format and turns it upright according to its EXIF orientation.
+///
+/// Phones store a photo as the sensor saw it and only note in the EXIF data how to turn it.
+/// Without applying that note, a photo taken upright would lie on its side once encoded anew,
+/// because the new file no longer carries the note. The content has to match the expected format,
+/// so that a file which merely claims to be an image is not decoded as one.
+fn decode_oriented(path: &Path, format: ImageFormat) -> Result {
+ let reader = ImageReader::open(path)
+ .and_then(|reader| reader.with_guessed_format())
+ .map_err(|error| ImageFailure::new(ImageErrorCode::FileNotReadable, format!("The image could not be opened: {error}")))?;
+
+ if reader.format() != Some(format) {
+ return Err(ImageFailure::new(ImageErrorCode::ContentMismatch, "The image content does not match its file extension."));
+ }
+
+ let mut decoder = reader.into_decoder().map_err(decoding_failure)?;
+ let orientation = decoder.orientation().map_err(decoding_failure)?;
+
+ //
+ // The same check ImageReader::decode makes: the pixel buffer must fit into the decoder's
+ // memory limit. into_decoder only checks the dimensions.
+ //
+ Limits::default().reserve(decoder.total_bytes()).map_err(decoding_failure)?;
+
+ let mut image = DynamicImage::from_decoder(decoder).map_err(decoding_failure)?;
+ image.apply_orientation(orientation);
+ Ok(image)
+}
+
+/// Tells a decoder which hit its memory limit apart from any other decoding failure.
+fn decoding_failure(error: ImageError) -> ImageFailure {
+ match error {
+ ImageError::Limits(_) => ImageFailure::new(ImageErrorCode::TooManyPixels, format!("The image is too large to decode: {error}")),
+ _ => ImageFailure::new(ImageErrorCode::NotDecodable, format!("The image could not be decoded: {error}")),
+ }
+}
+
+/// Chooses the encoding of an image for a model.
+///
+/// JPEG and PNG keep their format: PNG holds screenshots and diagrams without loss, and JPEG
+/// holds photos. Any other format becomes PNG when it uses transparency, which JPEG cannot hold,
+/// and JPEG otherwise.
+fn model_encoding(source: ImageFormat, image: &DynamicImage) -> ModelEncoding {
+ match source {
+ ImageFormat::Jpeg => ModelEncoding::Jpeg,
+ ImageFormat::Png => ModelEncoding::Png,
+ _ if uses_transparency(image) => ModelEncoding::Png,
+ _ => ModelEncoding::Jpeg,
+ }
+}
+
+/// Whether any pixel of the image is not fully opaque.
+///
+/// An alpha channel alone says nothing: GIF decodes into RGBA, and many files carry a channel in
+/// which every pixel is opaque.
+fn uses_transparency(image: &DynamicImage) -> bool {
+ match image {
+ DynamicImage::ImageLumaA8(buffer) => buffer.pixels().any(|pixel| pixel.0[1] < u8::MAX),
+ DynamicImage::ImageRgba8(buffer) => buffer.pixels().any(|pixel| pixel.0[3] < u8::MAX),
+ DynamicImage::ImageLumaA16(buffer) => buffer.pixels().any(|pixel| pixel.0[1] < u16::MAX),
+ DynamicImage::ImageRgba16(buffer) => buffer.pixels().any(|pixel| pixel.0[3] < u16::MAX),
+ DynamicImage::ImageRgba32F(buffer) => buffer.pixels().any(|pixel| pixel.0[3] < 1.0),
+ _ => image.color().has_alpha() && image.to_rgba8().pixels().any(|pixel| pixel.0[3] < u8::MAX),
+ }
+}
+
+/// Encodes the image and lowers quality and size until it fits into the byte budget.
+fn fit_into_budget(image: DynamicImage, encoding: ModelEncoding, max_bytes: u64, was_resized: bool) -> Result {
+ if encoding == ModelEncoding::Png {
+ let png = to_png_pixels(&image);
+ let bytes = encode_model_image(&png, ModelEncoding::Png, JPEG_QUALITY)?;
+ if bytes.len() as u64 <= max_bytes {
+ return Ok(EncodedModelImage { bytes, encoding, jpeg_quality: None, width: png.width(), height: png.height(), was_resized });
+ }
+ }
+
+ //
+ // From here on, the image goes as JPEG. A PNG which does not fit loses its transparency,
+ // which is laid onto white, the background most documents and chats show it on:
+ //
+ let mut current = to_jpeg_pixels(&image);
+ drop(image);
+
+ let mut was_resized = was_resized;
+ loop {
+ for quality in BUDGET_JPEG_QUALITIES {
+ let bytes = encode_model_image(¤t, ModelEncoding::Jpeg, quality)?;
+ if bytes.len() as u64 <= max_bytes {
+ return Ok(EncodedModelImage { bytes, encoding: ModelEncoding::Jpeg, jpeg_quality: Some(quality), width: current.width(), height: current.height(), was_resized });
+ }
+ }
+
+ let longest_edge = current.width().max(current.height());
+ if longest_edge <= MIN_BUDGET_EDGE_PIXELS {
+ return Err(ImageFailure::new(ImageErrorCode::BudgetNotReachable, format!("The image does not fit into {max_bytes} bytes, not even with {longest_edge} pixels on its longest edge.")));
+ }
+
+ let next_edge = ((longest_edge as f64 * BUDGET_EDGE_FACTOR).round() as u32).max(MIN_BUDGET_EDGE_PIXELS);
+ current = resize_to_longest_edge(current, next_edge);
+ was_resized = true;
+ }
+}
+
+/// Brings an image into 8 bits per channel for PNG, keeping the transparency where it is used.
+fn to_png_pixels(image: &DynamicImage) -> DynamicImage {
+ let is_gray = !image.color().has_color();
+ match (is_gray, uses_transparency(image)) {
+ (true, true) => DynamicImage::ImageLumaA8(image.to_luma_alpha8()),
+ (true, false) => DynamicImage::ImageLuma8(image.to_luma8()),
+ (false, true) => DynamicImage::ImageRgba8(image.to_rgba8()),
+ (false, false) => DynamicImage::ImageRgb8(image.to_rgb8()),
+ }
+}
+
+/// Brings an image into 8 bits per channel for JPEG, which has no alpha channel.
+fn to_jpeg_pixels(image: &DynamicImage) -> DynamicImage {
+ if uses_transparency(image) {
+ return flatten_onto_white(image);
+ }
+
+ if image.color().has_color() {
+ DynamicImage::ImageRgb8(image.to_rgb8())
+ } else {
+ DynamicImage::ImageLuma8(image.to_luma8())
+ }
+}
+
+/// Lays a partly transparent image onto a white background.
+///
+/// Dropping the alpha channel alone would keep whatever color the transparent pixels happen to
+/// hold, which is often black.
+fn flatten_onto_white(image: &DynamicImage) -> DynamicImage {
+ let rgba = image.to_rgba8();
+ let mut flattened = RgbImage::new(rgba.width(), rgba.height());
+ for (source, target) in rgba.pixels().zip(flattened.pixels_mut()) {
+ let [red, green, blue, alpha] = source.0;
+ let alpha = u16::from(alpha);
+ let blend = |channel: u8| ((u16::from(channel) * alpha + 255 * (255 - alpha) + 127) / 255) as u8;
+ *target = Rgb([blend(red), blend(green), blend(blue)]);
+ }
+
+ DynamicImage::ImageRgb8(flattened)
+}
+
+/// Encodes an image whose pixels already suit the encoding.
+fn encode_model_image(image: &DynamicImage, encoding: ModelEncoding, jpeg_quality: u8) -> Result, ImageFailure> {
+ let mut bytes = Vec::new();
+ let result = match encoding {
+ ModelEncoding::Jpeg => JpegEncoder::new_with_quality(&mut bytes, jpeg_quality).encode_image(image),
+ ModelEncoding::Png => image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png),
+ };
+
+ result.map_err(|error| ImageFailure::new(ImageErrorCode::Internal, format!("The image could not be encoded: {error}")))?;
+ Ok(bytes)
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -391,4 +808,270 @@ mod tests {
assert_eq!(issue.0, StatusCode::BAD_REQUEST);
std::fs::remove_file(path).unwrap();
}
+
+ /// Noise which compresses badly, the same on every run.
+ fn noise_image(width: u32, height: u32) -> RgbImage {
+ let mut state: u32 = 0x1234_5678;
+ RgbImage::from_fn(width, height, |_, _| {
+ let mut channel = || {
+ state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
+ (state >> 24) as u8
+ };
+
+ Rgb([channel(), channel(), channel()])
+ })
+ }
+
+ /// Writes a black JPEG whose EXIF data says how to turn it.
+ ///
+ /// The encoder writes no EXIF data, so the segment is put in by hand, right after the start
+ /// of the image: a big-endian TIFF header with one entry, the orientation.
+ fn write_jpeg_with_orientation(path: &Path, width: u32, height: u32, orientation: u8) {
+ let mut jpeg = Vec::new();
+ JpegEncoder::new_with_quality(&mut jpeg, JPEG_QUALITY)
+ .encode_image(&DynamicImage::new_rgb8(width, height))
+ .unwrap();
+
+ let exif: [u8; 36] = [
+ 0xFF, 0xE1, 0x00, 0x22, // APP1 and its length, which counts itself but not the marker
+ b'E', b'x', b'i', b'f', 0x00, 0x00,
+ b'M', b'M', 0x00, 0x2A, 0x00, 0x00, 0x00, 0x08, // big-endian, first IFD at offset 8
+ 0x00, 0x01, // one entry
+ 0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, orientation, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, // no further IFD
+ ];
+
+ let mut file = jpeg[..2].to_vec();
+ file.extend_from_slice(&exif);
+ file.extend_from_slice(&jpeg[2..]);
+ std::fs::write(path, file).unwrap();
+ }
+
+ #[test]
+ fn model_preparation_scales_down_only_above_the_maximum_edge() {
+ let larger = temporary_image_path("png");
+ DynamicImage::new_rgb8(MAX_EDGE_PIXELS + 1, 1).save_with_format(&larger, ImageFormat::Png).unwrap();
+ let prepared = prepare_for_model(&larger, u64::MAX).unwrap();
+ assert_eq!((prepared.width, prepared.height), (MAX_EDGE_PIXELS, 1));
+ assert!(prepared.was_resized);
+ std::fs::remove_file(larger).unwrap();
+
+ let at_the_edge = temporary_image_path("png");
+ DynamicImage::new_rgb8(MAX_EDGE_PIXELS, 2).save_with_format(&at_the_edge, ImageFormat::Png).unwrap();
+ let prepared = prepare_for_model(&at_the_edge, u64::MAX).unwrap();
+ assert_eq!((prepared.width, prepared.height), (MAX_EDGE_PIXELS, 2));
+ assert!(!prepared.was_resized);
+ std::fs::remove_file(at_the_edge).unwrap();
+ }
+
+ #[test]
+ fn model_preparation_keeps_jpeg_and_png_and_turns_other_formats_into_one_of_them() {
+ let opaque = DynamicImage::ImageRgb8(RgbImage::from_pixel(9, 7, Rgb([200, 10, 10])));
+ let mut transparent = opaque.to_rgba8();
+ transparent.get_pixel_mut(0, 0).0[3] = 0;
+ let transparent = DynamicImage::ImageRgba8(transparent);
+
+ for (extension, source_format, image, expected) in [
+ ("jpg", ImageFormat::Jpeg, &opaque, ImageFormat::Jpeg),
+ ("png", ImageFormat::Png, &opaque, ImageFormat::Png),
+ ("webp", ImageFormat::WebP, &opaque, ImageFormat::Jpeg),
+ ("webp", ImageFormat::WebP, &transparent, ImageFormat::Png),
+ ("gif", ImageFormat::Gif, &opaque, ImageFormat::Jpeg),
+ ("gif", ImageFormat::Gif, &transparent, ImageFormat::Png),
+ ("bmp", ImageFormat::Bmp, &opaque, ImageFormat::Jpeg),
+ ("tif", ImageFormat::Tiff, &opaque, ImageFormat::Jpeg),
+ ("tiff", ImageFormat::Tiff, &transparent, ImageFormat::Png),
+ ] {
+ let path = temporary_image_path(extension);
+ image.save_with_format(&path, source_format).unwrap();
+ let prepared = prepare_for_model(&path, u64::MAX).unwrap();
+ assert_eq!(image::guess_format(&prepared.bytes).unwrap(), expected, "{extension} with alpha={}", image.color().has_alpha());
+ assert_eq!(prepared.encoding == ModelEncoding::Png, expected == ImageFormat::Png, "{extension}");
+ assert_eq!((prepared.width, prepared.height), (9, 7), "{extension}");
+ std::fs::remove_file(path).unwrap();
+ }
+ }
+
+ #[test]
+ fn model_preparation_refuses_formats_it_cannot_decode() {
+ for extension in ["heic", "heif", "avif", "svg", "ico"] {
+ let path = temporary_image_path(extension);
+ std::fs::write(&path, b"whatever").unwrap();
+ let failure = prepare_for_model(&path, u64::MAX).err().unwrap();
+ assert_eq!(failure.code, ImageErrorCode::UnsupportedFormat, "{extension}");
+ std::fs::remove_file(path).unwrap();
+ }
+ }
+
+ #[test]
+ fn model_preparation_tells_missing_mismatched_and_damaged_files_apart() {
+ let missing = prepare_for_model(&temporary_image_path("png"), u64::MAX).err().unwrap();
+ assert_eq!(missing.code, ImageErrorCode::FileNotFound);
+
+ let relative = prepare_for_model(Path::new("picture.png"), u64::MAX).err().unwrap();
+ assert_eq!(relative.code, ImageErrorCode::FileNotFound);
+
+ let jpeg_named_png = temporary_image_path("png");
+ DynamicImage::new_rgb8(4, 3).save_with_format(&jpeg_named_png, ImageFormat::Jpeg).unwrap();
+ let mismatch = prepare_for_model(&jpeg_named_png, u64::MAX).err().unwrap();
+ assert_eq!(mismatch.code, ImageErrorCode::ContentMismatch);
+ std::fs::remove_file(jpeg_named_png).unwrap();
+
+ let damaged = temporary_image_path("png");
+ std::fs::write(&damaged, b"not an image").unwrap();
+ let failure = prepare_for_model(&damaged, u64::MAX).err().unwrap();
+ assert_eq!(failure.code, ImageErrorCode::NotDecodable);
+ std::fs::remove_file(damaged).unwrap();
+ }
+
+ #[test]
+ fn model_preparation_turns_photos_upright() {
+ let path = temporary_image_path("jpg");
+ write_jpeg_with_orientation(&path, 20, 10, 6);
+ let prepared = prepare_for_model(&path, u64::MAX).unwrap();
+ assert_eq!((prepared.width, prepared.height), (10, 20), "Orientation 6 turns the image by 90 degrees.");
+ std::fs::remove_file(path).unwrap();
+ }
+
+ #[test]
+ fn optimization_turns_photos_upright() {
+ let path = temporary_image_path("jpg");
+ write_jpeg_with_orientation(&path, 20, 10, 6);
+ let response = prepare_image_sync(&PrepareImageRequest {
+ path: path.to_string_lossy().into_owned(),
+ optimize: true,
+ })
+ .unwrap();
+
+ let bytes = general_purpose::STANDARD
+ .decode(response.data_url.split_once(',').unwrap().1)
+ .unwrap();
+ let encoded = image::load_from_memory(&bytes).unwrap();
+ assert_eq!((response.width, response.height), (10, 20), "Orientation 6 turns the image by 90 degrees.");
+ assert_eq!((encoded.width(), encoded.height()), (10, 20), "The encoded image carries no orientation of its own any longer.");
+ std::fs::remove_file(path).unwrap();
+ }
+
+ #[test]
+ fn model_preparation_gives_the_same_bytes_for_the_same_image() {
+ let path = temporary_image_path("jpg");
+ DynamicImage::ImageRgb8(noise_image(64, 48)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+ let first = prepare_for_model(&path, u64::MAX).unwrap();
+ let second = prepare_for_model(&path, u64::MAX).unwrap();
+ assert_eq!(first.bytes, second.bytes);
+ std::fs::remove_file(path).unwrap();
+ }
+
+ #[test]
+ fn budget_lowers_the_quality_before_the_size() {
+ let path = temporary_image_path("jpg");
+ DynamicImage::ImageRgb8(noise_image(320, 200)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+
+ //
+ // The sizes the budget will meet, measured on the very pixels it works with:
+ //
+ let decoded = to_jpeg_pixels(&decode_oriented(&path, ImageFormat::Jpeg).unwrap());
+ let at_first_quality = encode_model_image(&decoded, ModelEncoding::Jpeg, BUDGET_JPEG_QUALITIES[0]).unwrap();
+ let at_second_quality = encode_model_image(&decoded, ModelEncoding::Jpeg, BUDGET_JPEG_QUALITIES[1]).unwrap();
+ assert!(at_second_quality.len() < at_first_quality.len());
+
+ let prepared = prepare_for_model(&path, at_first_quality.len() as u64 - 1).unwrap();
+ assert_eq!(prepared.bytes, at_second_quality);
+ assert_eq!((prepared.width, prepared.height), (320, 200));
+ assert!(!prepared.was_resized);
+ std::fs::remove_file(path).unwrap();
+ }
+
+ #[test]
+ fn budget_shrinks_the_image_once_no_quality_fits() {
+ let path = temporary_image_path("jpg");
+ DynamicImage::ImageRgb8(noise_image(1_200, 600)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+
+ let decoded = to_jpeg_pixels(&decode_oriented(&path, ImageFormat::Jpeg).unwrap());
+ let at_last_quality = encode_model_image(&decoded, ModelEncoding::Jpeg, *BUDGET_JPEG_QUALITIES.last().unwrap()).unwrap();
+ let budget = at_last_quality.len() as u64 - 1;
+
+ let prepared = prepare_for_model(&path, budget).unwrap();
+ assert!(prepared.bytes.len() as u64 <= budget);
+ assert_eq!((prepared.width, prepared.height), (MIN_BUDGET_EDGE_PIXELS, 512));
+ assert!(prepared.was_resized);
+ assert_eq!(prepared.encoding, ModelEncoding::Jpeg);
+ std::fs::remove_file(path).unwrap();
+ }
+
+ #[test]
+ fn budget_lays_a_transparent_png_onto_white_when_it_has_to_become_jpeg() {
+ let mut pixels = DynamicImage::ImageRgb8(noise_image(300, 200)).to_rgba8();
+ for (x, _, pixel) in pixels.enumerate_pixels_mut() {
+ if x < 150 {
+ pixel.0 = [0, 0, 0, 0];
+ }
+ }
+
+ let path = temporary_image_path("png");
+ DynamicImage::ImageRgba8(pixels).save_with_format(&path, ImageFormat::Png).unwrap();
+ let unlimited = prepare_for_model(&path, u64::MAX).unwrap();
+ assert_eq!(unlimited.encoding, ModelEncoding::Png);
+
+ let prepared = prepare_for_model(&path, unlimited.bytes.len() as u64 - 1).unwrap();
+ assert_eq!(prepared.encoding, ModelEncoding::Jpeg);
+
+ let corner = image::load_from_memory(&prepared.bytes).unwrap().to_rgb8().get_pixel(5, 5).0;
+ assert!(corner.iter().all(|&channel| channel > 240), "The transparent half has to be white, not black: {corner:?}");
+ std::fs::remove_file(path).unwrap();
+ }
+
+ #[test]
+ fn budget_reports_the_quality_which_fit() {
+ let path = temporary_image_path("jpg");
+ DynamicImage::ImageRgb8(noise_image(64, 48)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+ assert_eq!(prepare_for_model(&path, u64::MAX).unwrap().jpeg_quality, Some(JPEG_QUALITY));
+ std::fs::remove_file(path).unwrap();
+
+ let path = temporary_image_path("png");
+ DynamicImage::new_rgb8(4, 3).save_with_format(&path, ImageFormat::Png).unwrap();
+ assert_eq!(prepare_for_model(&path, u64::MAX).unwrap().jpeg_quality, None, "PNG has no quality.");
+ std::fs::remove_file(path).unwrap();
+ }
+
+ #[test]
+ fn log_writes_sizes_in_megabytes_and_the_ratio_in_percent() {
+ assert_eq!(megabytes(5_812_345), "5.81 MB");
+ assert_eq!(megabytes(0), "0.00 MB");
+ assert_eq!(percentage(912_000, 5_812_345), "16 %");
+ assert_eq!(percentage(1, 0), "n/a", "An empty source has no ratio.");
+ }
+
+ #[test]
+ fn budget_refuses_an_image_it_cannot_make_fit() {
+ let path = temporary_image_path("jpg");
+ DynamicImage::ImageRgb8(noise_image(64, 48)).save_with_format(&path, ImageFormat::Jpeg).unwrap();
+ let failure = prepare_for_model(&path, 100).err().unwrap();
+ assert_eq!(failure.code, ImageErrorCode::BudgetNotReachable);
+ std::fs::remove_file(path).unwrap();
+ }
+
+ #[test]
+ fn serializes_the_model_response_for_the_rust_service_contract() {
+ let failure = serde_json::to_value(PrepareModelImageResponse::failure(ImageErrorCode::BudgetNotReachable)).unwrap();
+ assert_eq!(failure["error_code"], "BUDGET_NOT_REACHABLE");
+ assert_eq!(failure["base64"], "");
+
+ let path = temporary_image_path("png");
+ DynamicImage::new_rgb8(4, 3).save_with_format(&path, ImageFormat::Png).unwrap();
+ let success = serde_json::to_value(prepare_image_for_model_sync(&PrepareModelImageRequest {
+ path: path.to_string_lossy().into_owned(),
+ max_bytes: u64::MAX,
+ }))
+ .unwrap();
+
+ assert_eq!(success["error_code"], "NONE");
+ assert_eq!(success["mime_type"], "image/png");
+ assert_eq!(success["width"], 4);
+ assert_eq!(success["height"], 3);
+ assert_eq!(success["was_resized"], false);
+ assert!(!success["base64"].as_str().unwrap().starts_with("data:"));
+ assert!(success.get("mimeType").is_none());
+ std::fs::remove_file(path).unwrap();
+ }
}
\ No newline at end of file
diff --git a/runtime/src/runtime_api.rs b/runtime/src/runtime_api.rs
index 4ec0fc8ec..b67221ede 100644
--- a/runtime/src/runtime_api.rs
+++ b/runtime/src/runtime_api.rs
@@ -86,6 +86,7 @@ fn create_router() -> Router {
.route("/media/jobs/{id}/events", get(crate::media::get_job_events))
.route("/media/jobs/{id}", delete(crate::media::cancel_job))
.route("/image/prepare", post(crate::image::prepare_image))
+ .route("/image/prepare-for-model", post(crate::image::prepare_image_for_model))
.route("/log/paths", get(crate::log::get_log_paths))
.route("/log/event", post(crate::log::log_event))
.route("/tokenizer/count", post(crate::tokenizer::token_count))