Allowed the Batch Processing assistant to process images (#1087)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor

@@ -25,6 +25,10 @@
     @T("Supported audio and video files are transcribed automatically without an additional dialog. Each transcript is stored next to its media file as '<media-file>.transcript.md' and reused when an interrupted run is continued.")
 </MudJustifiedText>
 
+<MudJustifiedText Typo="Typo.body2" Class="mb-3">
+    @T("Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.")
+</MudJustifiedText>
+
 <MudStack Class="mb-3">
     <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
         @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs

@@ -1,18 +1,87 @@
+using AIStudio.Chat;
+using AIStudio.Settings;
 using AIStudio.Tools.Media;
+using AIStudio.Tools.Rust;
 
 namespace AIStudio.Assistants.BatchProcessing;
 
 public partial class AssistantBatchProcessing
 {
     /// <summary>
-    /// Loads a document through the Rust content stream or resolves a persistent
-    /// transcript for an audio or video file.
+    /// What one file of the batch gives to the model: either text or an image.
     /// </summary>
-    private Task<string?> LoadInputContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
+    private readonly record struct BatchInput(string Text, FileAttachmentImage? Image);
+
+    /// <summary>
+    /// Loads a document through the Rust content stream, resolves a persistent
+    /// transcript for an audio or video file, or prepares an image for the model.
+    /// </summary>
+    /// <returns>The input for the model, or <c>null</c> when the file already got its final status.</returns>
+    private async Task<BatchInput?> LoadInputContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
     {
-        return IsTranscribableMedia(fileResult.FilePath)
-            ? this.LoadMediaTranscriptAsync(fileResult, token)
-            : this.LoadDocumentContentAsync(fileResult, token);
+        if (IsImage(fileResult.FilePath))
+            return this.LoadImage(fileResult) is { } image ? new BatchInput(string.Empty, image) : null;
+
+        var text = IsTranscribableMedia(fileResult.FilePath)
+            ? await this.LoadMediaTranscriptAsync(fileResult, token)
+            : await this.LoadDocumentContentAsync(fileResult, token);
+
+        return text is null ? null : new BatchInput(text, null);
+    }
+
+    /// <summary>
+    /// Checks an image before it goes to the model.
+    /// </summary>
+    /// <remarks>
+    /// The request itself would only skip an image it cannot send, with a message, and ask the model
+    /// anyway. The model would then answer about nothing, and the file would count as done, so that
+    /// continuing the run would skip it as well. A failed image is processed again when the run is
+    /// continued, e.g., with a model that can read images.
+    ///
+    /// How many images a model accepts needs no check of its own: every file gets a thread of its own,
+    /// see CallAIAsync, so each request carries exactly one image, in every round with the model. A
+    /// limit of one image is therefore always met, and SupportsImageInput already refuses a limit of zero.
+    ///
+    /// Unlike the text of documents and transcripts, an image does not pass the prompt injection
+    /// filter of the runtime, which reads text only. Images in a chat do not either. Instructions
+    /// within an image therefore meet only the safeguards behind the filter: the system prompt tells
+    /// the model to ignore them, a thread which read internal content, such as a mailbox, may no
+    /// longer send data out, and without free address choice, its default, the Read Web Page tool
+    /// refuses an address which appears only in the image.
+    /// </remarks>
+    private FileAttachmentImage? LoadImage(BatchProcessingFileResult fileResult)
+    {
+        if (!this.ProviderSettings.SupportsImageInput())
+        {
+            this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("The selected model cannot read images. Select a model with image support to process this image."));
+            return null;
+        }
+
+        if (!FileTypes.IsAllowedPath(fileResult.FilePath, FileTypes.MODEL_IMAGE))
+        {
+            this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."));
+            return null;
+        }
+
+        long fileSize;
+        try
+        {
+            fileSize = new FileInfo(fileResult.FilePath).Length;
+        }
+        catch (Exception e)
+        {
+            this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("Was not able to read the file: {0}"), e.Message), e);
+            return null;
+        }
+
+        var maxImageSizeBytes = IImageSourceExtensions.MaxImageSizeBytesFor(this.ProviderSettings.UsedLLMProvider);
+        if (fileSize > maxImageSizeBytes)
+        {
+            this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("The image is larger than {0} MB, which is the most the selected provider accepts for one image."), (maxImageSizeBytes / 1_000_000d).ToString("0.#", Tools.PluginSystem.I18N.I.Culture)));
+            return null;
+        }
+
+        return new FileAttachmentImage(fileResult.FileName, fileResult.FilePath, fileSize);
     }
 
     private async Task<string?> LoadDocumentContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
@@ -109,5 +178,7 @@ public partial class AssistantBatchProcessing
 
     private static bool IsTranscribableMedia(string filePath) => PersistentMediaTranscript.IsMediaPath(filePath);
 
+    private static bool IsImage(string filePath) => FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE);
+
     private static bool HasReusableTranscript(string mediaFilePath) => MediaSidecarTranscripts.HasReusableTranscript(mediaFilePath);
 }

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Prompts.cs

@@ -56,6 +56,8 @@ public partial class AssistantBatchProcessing
                 # Task description
                 You are a batch document processing agent. Each request contains exactly one DOCUMENT.
                 Your task is to process this DOCUMENT strictly according to the instructions below.
+                A DOCUMENT may also be an image attached to the request. Then, the content of the
+                DOCUMENT is what the image shows, including any text within it.
 
                 # Scope and precedence
                 Use only information explicitly contained in the DOCUMENT and the instructions.
@@ -88,11 +90,20 @@ public partial class AssistantBatchProcessing
                 """;
     }
 
+    private static string BuildImageUserPrompt(string fileName)
+    {
+        return $"""
+                # DOCUMENT
+                File name: {fileName}
+                Content: The DOCUMENT is the image attached to this request.
+                """;
+    }
+
     /// <param name="fileName">The name of the document being processed.</param>
-    /// <param name="fileContent">The content handed to the model.</param>
+    /// <param name="input">The text or the image handed to the model.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>The answer of the model, and which tools it used to get there.</returns>
-    private async Task<(string Answer, string UsedTools)> CallAIAsync(string fileName, string fileContent, CancellationToken token)
+    private async Task<(string Answer, string UsedTools)> CallAIAsync(string fileName, BatchInput input, CancellationToken token)
     {
         //
         // Every file of the batch gets the tools the user picked for the job. The batch builds its
@@ -120,10 +131,9 @@ public partial class AssistantBatchProcessing
         };
 
         this.AssignRuntimeIdentity(chatThread);
-        var userPrompt = new ContentText
-        {
-            Text = BuildUserPrompt(fileName, fileContent),
-        };
+        var userPrompt = input.Image is null
+            ? new ContentText { Text = BuildUserPrompt(fileName, input.Text) }
+            : new ContentText { Text = BuildImageUserPrompt(fileName), FileAttachments = [input.Image] };
 
         chatThread.Blocks.Add(new ContentBlock
         {

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Run.cs

@@ -1,6 +1,8 @@
 using System.Diagnostics;
 using System.Globalization;
 
+using AIStudio.Settings;
+
 namespace AIStudio.Assistants.BatchProcessing;
 
 public partial class AssistantBatchProcessing
@@ -42,6 +44,7 @@ public partial class AssistantBatchProcessing
         }
 
         this.PrepareFileResults(resolvedOutputDirectory, files, previousLog, previousResults);
+        await this.WarnAboutUnreadableImagesAsync();
         await this.CheckpointAssistantSession();
         await this.RunBatchAsync(resolvedOutputDirectory);
     }
@@ -88,6 +91,27 @@ public partial class AssistantBatchProcessing
         }
     }
 
+    /// <summary>
+    /// Tells the user before the first file when the images of the run will fail, because the
+    /// selected model cannot read images.
+    /// </summary>
+    /// <remarks>
+    /// Counts only the images which this run still processes. Images done by an earlier run of a
+    /// continued batch keep their results and do not fail.
+    /// </remarks>
+    private async Task WarnAboutUnreadableImagesAsync()
+    {
+        var numImages = this.fileResults.Count(fileResult => fileResult.Status is not BatchProcessingFileStatus.DONE && IsImage(fileResult.FilePath));
+        if (numImages == 0 || this.ProviderSettings.SupportsImageInput())
+            return;
+
+        var warning = numImages == 1
+            ? T("The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual.")
+            : string.Format(T("The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual."), numImages);
+
+        await this.MessageBus.SendWarning(new(Icons.Material.Filled.ImageNotSupported, warning));
+    }
+
     /// <summary>
     /// Processes all documents which are not restored from a previous run.
     /// </summary>
@@ -188,14 +212,14 @@ public partial class AssistantBatchProcessing
     /// </remarks>
     private async Task ProcessOneFileAsync(BatchProcessingFileResult fileResult, string resolvedOutputDirectory, CancellationToken token)
     {
-        var fileContent = await this.LoadInputContentAsync(fileResult, token);
-        if (fileContent is null)
+        var input = await this.LoadInputContentAsync(fileResult, token);
+        if (input is null)
             return;
 
         string aiAnswer;
         try
         {
-            (aiAnswer, fileResult.UsedTools) = await this.CallAIAsync(fileResult.FileName, fileContent, token);
+            (aiAnswer, fileResult.UsedTools) = await this.CallAIAsync(fileResult.FileName, input.Value, token);
         }
         catch (OperationCanceledException)
         {
@@ -217,6 +241,17 @@ public partial class AssistantBatchProcessing
             return;
         }
 
+        //
+        // The request reads the image anew in every round with the model, e.g., after each tool
+        // call, and quietly leaves out an image which is gone by then. The answer would be about
+        // nothing and must not count as a result, so that continuing the run processes the image again:
+        //
+        if (input.Value.Image is { Exists: false })
+        {
+            this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("The image was no longer available while the model was asked about it."));
+            return;
+        }
+
         if (string.IsNullOrWhiteSpace(aiAnswer))
         {
             this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("The AI answer was empty."));

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs

@@ -1,3 +1,4 @@
+using AIStudio.Settings;
 using AIStudio.Tools.Media;
 
 namespace AIStudio.Assistants.BatchProcessing;
@@ -217,6 +218,17 @@ public partial class AssistantBatchProcessing
             return null;
         }
 
+        //
+        // A model that cannot read images would get the text of the request alone, without the
+        // image, and answer about nothing. Such images fail one by one, see LoadImage, while the
+        // documents next to them are processed. A run of nothing but images does not start at all:
+        //
+        if (files.All(IsImage) && !this.ProviderSettings.SupportsImageInput())
+        {
+            this.AddInputIssue(T("All selected files are images, but the selected model cannot read images. Select a model with image support."));
+            return null;
+        }
+
         try
         {
             Directory.CreateDirectory(resolvedOutputDirectory);

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -424,6 +424,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- The selected folder does not exist.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T17705912"] = "The selected folder does not exist."
 
+-- The image was no longer available while the model was asked about it.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1805791865"] = "The image was no longer available while the model was asked about it."
+
 -- Minimum pause between files
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1829787634"] = "Minimum pause between files"
 
@@ -529,9 +532,15 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Was not able to write the result file: {0}
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Was not able to write the result file: {0}"
 
+-- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
+
 -- The batch run finished, but one file could not be processed. See the progress table and log for details.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "The batch run finished, but one file could not be processed. See the progress table and log for details."
 
+-- The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T324865195"] = "The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual."
+
 -- Maximum pause between files
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3250003796"] = "Maximum pause between files"
 
@@ -550,6 +559,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3361153305"] = "Show Expert Settings"
 
+-- All selected files are images, but the selected model cannot read images. Select a model with image support.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3365102967"] = "All selected files are images, but the selected model cannot read images. Select a model with image support."
+
 -- Header of the result column (optional)
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T340994102"] = "Header of the result column (optional)"
 
@@ -562,6 +574,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3439247329"] = "The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'."
 
+-- The selected model cannot read images. Select a model with image support to process this image.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T346986514"] = "The selected model cannot read images. Select a model with image support to process this image."
+
 -- Document analysis policy
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3510564924"] = "Document analysis policy"
 
@@ -646,6 +661,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Start batch processing
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Start batch processing"
 
+-- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."
+
 -- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again."
 
@@ -670,12 +688,18 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
 -- Include hidden files and folders?
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Include hidden files and folders?"
 
+-- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "The image is larger than {0} MB, which is the most the selected provider accepts for one image."
+
 -- Restore default
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Restore default"
 
 -- The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here.
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T967206794"] = "The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here."
 
+-- The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual.
+UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T9738678"] = "The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual."
+
 -- Comma (,)
 UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T1676507543"] = "Comma (,)"
 
@@ -8566,6 +8590,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
 -- Default minimum pause between files
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Default minimum pause between files"
 
+-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format."
+
 -- Instructions
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Instructions"
 
@@ -8620,9 +8647,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T24
 -- When enabled, new batch runs start with the defaults configured below.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2592677194"] = "When enabled, new batch runs start with the defaults configured below."
 
--- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2594325620"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats."
-
 -- Subfolders are included
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2607092632"] = "Subfolders are included"
 
@@ -13411,6 +13435,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like p
 -- Document
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document"
 
+-- Image for AI models
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Image for AI models"
+
 -- Plugin archive
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin archive"