Fixed the Batch Processing assistant with an output folder above the input folder (#1098)

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs

@@ -83,39 +83,18 @@ public partial class AssistantBatchProcessing
     private IReadOnlyList<string> FindInputFiles(string resolvedOutputDirectory)
     {
         var selection = new FolderFileSelection(FileNamePatterns.Parse(this.filePatterns), FileExclusions.Parse(this.excludePatterns), this.includeSubdirectories, this.includeHiddenFilesAndFolders, this.maxFileSizeMegabytes);
+        var outputFolder = new BatchProcessingOutputFolder(this.inputDirectory, resolvedOutputDirectory, this.ResolveResultsFileName());
         var files = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
         this.numTooLargeFiles = 0;
 
-        var normalizedInputDirectory = TrimDirectorySeparator(Path.GetFullPath(this.inputDirectory));
-        var normalizedOutputDirectory = TrimDirectorySeparator(Path.GetFullPath(resolvedOutputDirectory));
-
-        // When the output folder is a folder of its own, we skip everything
-        // inside it. When it is the input folder itself, we must not skip the
-        // whole folder: we would not find any document at all. We then skip
-        // our own output artifacts instead.
-        var isOutputSeparateFolder = !string.Equals(normalizedInputDirectory, normalizedOutputDirectory, StringComparison.OrdinalIgnoreCase);
-
-        // The separator is essential: without it, an output folder named 'out'
-        // would also exclude a document named 'output-notes.md':
-        var outputDirectoryPrefix = normalizedOutputDirectory + Path.DirectorySeparatorChar;
-
         foreach (var file in selection.EnumerateCandidates(
                      this.inputDirectory,
-                     includeDirectory: directory => !isOutputSeparateFolder
-                                                    || !string.Equals(TrimDirectorySeparator(Path.GetFullPath(directory)), normalizedOutputDirectory, StringComparison.OrdinalIgnoreCase),
+                     includeDirectory: outputFolder.Enters,
                      reportInaccessibleDirectory: (directory, exception) =>
                          throw new IOException($"Could not access directory '{directory}'.", exception)))
         {
             var normalizedFile = Path.GetFullPath(file);
-            if (!selection.Selects(normalizedFile) || IsTranscriptArtifact(normalizedFile))
-                continue;
-
-            if (isOutputSeparateFolder)
-            {
-                if (normalizedFile.StartsWith(outputDirectoryPrefix, StringComparison.OrdinalIgnoreCase))
-                    continue;
-            }
-            else if (this.IsOwnOutputArtifact(normalizedFile))
+            if (!selection.Selects(normalizedFile) || IsTranscriptArtifact(normalizedFile) || outputFolder.IsOutputOfRun(normalizedFile))
                 continue;
 
             //
@@ -137,25 +116,6 @@ public partial class AssistantBatchProcessing
         return [.. files];
     }
 
-    private static string TrimDirectorySeparator(string path) => path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
-
-    /// <summary>
-    /// Checks whether a file is an output artifact of this assistant. We need
-    /// this when the output folder is the input folder: without it, the results
-    /// of a previous run would be processed as documents.
-    /// </summary>
-    private bool IsOwnOutputArtifact(string filePath)
-    {
-        var fileName = Path.GetFileName(filePath);
-        if (string.Equals(fileName, LOG_FILENAME, StringComparison.OrdinalIgnoreCase))
-            return true;
-
-        if (string.Equals(fileName, this.ResolveResultsFileName(), StringComparison.OrdinalIgnoreCase))
-            return true;
-
-        return fileName.EndsWith(RESULT_FILE_SUFFIX, StringComparison.OrdinalIgnoreCase);
-    }
-
     /// <summary>
     /// Checks for persistent or interrupted media transcript artifacts. They
     /// always live beside their source file, independently of the output folder.

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs

@@ -18,7 +18,7 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
     private const string DEFAULT_OUTPUT_DIRECTORY_NAME = "ai-results";
     private const string DEFAULT_RESULTS_FILENAME = "batch-results.csv";
     private const string CSV_EXTENSION = ".csv";
-    private const string RESULT_FILE_SUFFIX = "_result";
+    internal const string RESULT_FILE_SUFFIX = "_result";
     private const string TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
     private const char LOG_SEPARATOR = ';';
 
@@ -26,7 +26,7 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
     /// The name of the log file. It is fixed, so that a later batch run finds
     /// the log of a previous run and can continue it.
     /// </summary>
-    private const string LOG_FILENAME = "log.csv";
+    internal const string LOG_FILENAME = "log.csv";
 
     protected override Tools.Components Component => Tools.Components.BATCH_PROCESSING_ASSISTANT;
 

app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingOutputFolder.cs

@@ -0,0 +1,88 @@
+namespace AIStudio.Assistants.BatchProcessing;
+
+/// <summary>
+/// Tells the search for documents which of the files it finds a batch run wrote itself.
+/// </summary>
+/// <remarks>
+/// A run writes its result files, its log, and its results table directly into the output folder.
+/// What the search must leave out therefore depends on where this folder lies. Below the input
+/// folder, the search leaves out all of it. When it is the input folder itself, leaving out the
+/// whole folder would leave no document at all, so the search leaves out what a run writes, by
+/// name. Anywhere else, above the input folder included, the search never comes across the output
+/// folder and leaves out nothing.
+/// </remarks>
+/// <param name="inputDirectory">The folder the run reads its documents from.</param>
+/// <param name="outputDirectory">The folder the run writes into.</param>
+/// <param name="resultsFileName">The file name of the results table.</param>
+internal sealed class BatchProcessingOutputFolder(string inputDirectory, string outputDirectory, string resultsFileName)
+{
+    private readonly string fullOutputDirectory = Path.GetFullPath(outputDirectory);
+
+    private readonly bool isInputFolder = IsSameFolder(inputDirectory, outputDirectory);
+
+    //
+    // The same folder with a separator at its end would count as lying below itself:
+    //
+    private readonly bool liesBelowInputFolder = !IsSameFolder(inputDirectory, outputDirectory) && PathTools.IsInsideDirectory(inputDirectory, outputDirectory);
+
+    /// <summary>
+    /// Whether the search enters a subfolder of the input folder: every one but the output folder.
+    /// </summary>
+    /// <param name="directory">The subfolder.</param>
+    public bool Enters(string directory) => !this.liesBelowInputFolder || !IsSameFolder(directory, this.fullOutputDirectory);
+
+    /// <summary>
+    /// Whether a file the search found was written by a run instead of being a document.
+    /// </summary>
+    /// <param name="filePath">The full path of the file.</param>
+    public bool IsOutputOfRun(string filePath)
+    {
+        //
+        // Checked with the separator after the name of the folder: otherwise, an output folder
+        // named 'out' would also leave out a document named 'output-notes.md':
+        //
+        if (this.liesBelowInputFolder)
+            return PathTools.IsInsideDirectory(this.fullOutputDirectory, filePath);
+
+        //
+        // A run writes into the output folder itself, never into one of its subfolders:
+        //
+        return this.isInputFolder
+               && Path.GetDirectoryName(Path.GetFullPath(filePath)) is { } fileDirectory
+               && IsSameFolder(fileDirectory, this.fullOutputDirectory)
+               && this.IsOwnOutputArtifact(filePath);
+    }
+
+    /// <summary>
+    /// Checks whether a file is an output artifact of the assistant. We need
+    /// this when the output folder is the input folder: without it, the results
+    /// of a previous run would be processed as documents.
+    /// </summary>
+    /// <remarks>
+    /// A result file is recognized by its name without the extension, e.g., report_result.md or
+    /// report_result_2.docx, cf. AssistantBatchProcessing.CreateResultFileName. Its extension does
+    /// not count, because the user may have chosen another file format since a previous run.
+    /// </remarks>
+    private bool IsOwnOutputArtifact(string filePath)
+    {
+        var fileName = Path.GetFileName(filePath);
+        if (string.Equals(fileName, AssistantBatchProcessing.LOG_FILENAME, StringComparison.OrdinalIgnoreCase)
+            || string.Equals(fileName, resultsFileName, StringComparison.OrdinalIgnoreCase))
+            return true;
+
+        var stem = Path.GetFileNameWithoutExtension(fileName);
+        var suffixIndex = stem.LastIndexOf(AssistantBatchProcessing.RESULT_FILE_SUFFIX, StringComparison.OrdinalIgnoreCase);
+        if (suffixIndex < 0)
+            return false;
+
+        //
+        // A second result of the same name gets a counter, e.g., report_result_2.md:
+        //
+        var counter = stem[(suffixIndex + AssistantBatchProcessing.RESULT_FILE_SUFFIX.Length)..];
+        return counter.Length is 0 || (counter.Length > 1 && counter[0] is '_' && counter[1..].All(char.IsAsciiDigit));
+    }
+
+    private static bool IsSameFolder(string firstDirectory, string secondDirectory) => string.Equals(TrimDirectorySeparator(Path.GetFullPath(firstDirectory)), TrimDirectorySeparator(Path.GetFullPath(secondDirectory)), PathTools.COMPARISON);
+
+    private static string TrimDirectorySeparator(string path) => path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
+}

app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md

@@ -109,6 +109,8 @@
 - Fixed AI Studio not checking the transcripts of audio and video files for suspicious instructions (prompt injection) in the Batch Processing assistant and when you load such a file into a field of an assistant. AI Studio now checks these transcripts like all other documents before the AI sees them, even very long ones.
 - Fixed AI Studio deleting the API keys you entered yourself for the providers of your organization when your organization changed the secret that protects its configuration. These keys now stay, so you no longer need to enter them again.
 - Fixed sharing plugins when the share dialog of Windows or macOS does not open or stops responding. AI Studio now notices this and offers to save the plugin archive to a location of your choice instead. Until you restart AI Studio, the plugin page then offers this export instead of sharing.
+- Fixed the Batch Processing assistant finding no documents when you chose an output folder that contains the input folder, such as its parent folder.
+- Fixed the Batch Processing assistant processing its own result files again as new documents when you chose the input folder as the output folder.
 - Updated the code contributions on the supporters page, which now thank everyone who has contributed code to AI Studio so far.
 - Upgraded several libraries to improve security.
 - Upgraded to Rust v1.99.1

app/Tests/Assistants/BatchProcessingOutputFolderTests.cs

@@ -0,0 +1,165 @@
+using AIStudio.Assistants.BatchProcessing;
+
+namespace AIStudio.Tests.Assistants;
+
+/// <summary>
+/// Checks which files the search of the batch processing assistant leaves out because a batch run
+/// wrote them.
+/// </summary>
+/// <remarks>
+/// A run writes everything directly into its output folder. The search comes across any of it only
+/// when that folder lies below the input folder or is the input folder itself. An output folder
+/// above the input folder once made the search leave out every document.
+/// </remarks>
+[TestFixture]
+public sealed class BatchProcessingOutputFolderTests
+{
+    private const string RESULTS_FILE_NAME = "batch-results.csv";
+
+    private static readonly string DESKTOP = Path.Combine(Path.GetTempPath(), "ai-studio-batch-output-folder", "Desktop");
+    private static readonly string PROJECT_FOLDER = Path.Combine(DESKTOP, "Batch");
+    private static readonly string INPUT_FOLDER = Path.Combine(PROJECT_FOLDER, "Documents");
+
+    [Test]
+    public void AnOutputFolderAboveTheInputFolderLeavesOutNoDocument()
+    {
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, PROJECT_FOLDER, RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "notes", "minutes.md")), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "log.csv")), Is.False, "The log of a run lies in the output folder, so a file of this name in the input folder is a document.");
+            Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True);
+        });
+    }
+
+    [Test]
+    public void AnOutputFolderSeveralLevelsAboveTheInputFolderLeavesOutNoDocument()
+    {
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, DESKTOP, RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "notes", "minutes.md")), Is.False);
+            Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True);
+        });
+    }
+
+    [Test]
+    public void TheDefaultOutputFolderBelowTheInputFolderIsLeftOut()
+    {
+        var resultsFolder = Path.Combine(INPUT_FOLDER, "ai-results");
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, resultsFolder, RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(resultsFolder, "report_result.md")), Is.True);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
+            Assert.That(outputFolder.Enters(resultsFolder), Is.False);
+            Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True);
+        });
+    }
+
+    [Test]
+    public void AnOutputFolderDeepBelowTheInputFolderIsLeftOutAlone()
+    {
+        var parentFolder = Path.Combine(INPUT_FOLDER, "2026", "october");
+        var resultsFolder = Path.Combine(parentFolder, "results");
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, resultsFolder, RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(resultsFolder, "report_result.md")), Is.True);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(parentFolder, "report.pdf")), Is.False);
+            Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "2026")), Is.True);
+            Assert.That(outputFolder.Enters(parentFolder), Is.True);
+            Assert.That(outputFolder.Enters(resultsFolder), Is.False);
+        });
+    }
+
+    [Test]
+    public void ADocumentWhoseNameBeginsLikeTheOutputFolderIsNoOutput()
+    {
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, Path.Combine(INPUT_FOLDER, "out"), RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "output-notes.md")), Is.False);
+            Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "output")), Is.True);
+        });
+    }
+
+    [Test]
+    public void AnOutputFolderNextToTheInputFolderWithTheSameBeginningLeavesOutNothing()
+    {
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, $"{INPUT_FOLDER}-results", RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "log.csv")), Is.False);
+            Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True);
+        });
+    }
+
+    [TestCase(false)]
+    [TestCase(true)]
+    public void TheInputFolderAsOutputFolderLeavesOutTheLogAndTheResultsTable(bool withTrailingSeparator)
+    {
+        var outputDirectory = withTrailingSeparator ? INPUT_FOLDER + Path.DirectorySeparatorChar : INPUT_FOLDER;
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, outputDirectory, RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "log.csv")), Is.True);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, RESULTS_FILE_NAME)), Is.True);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
+            Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True, "Leaving out the output folder would leave out the input folder, too.");
+        });
+    }
+
+    [Test]
+    public void TheInputFolderAsOutputFolderLeavesOutTheResultFilesOfEveryFormat()
+    {
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, INPUT_FOLDER, RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report_result.md")), Is.True);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report_result_2.docx")), Is.True, "A second result of the same name gets a counter.");
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "Report_Result.HTML")), Is.True);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.v2_result.md")), Is.True, "Only the last extension of the document makes way for the suffix.");
+        });
+    }
+
+    [Test]
+    public void TheInputFolderAsOutputFolderKeepsDocumentsWithSimilarNames()
+    {
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, INPUT_FOLDER, RESULTS_FILE_NAME);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.md")), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "result.md")), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "my_results.md")), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report_result_draft.md")), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report_result_.md")), Is.False);
+        });
+    }
+
+    [Test]
+    public void TheInputFolderAsOutputFolderKeepsFilesInSubfolders()
+    {
+        var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, INPUT_FOLDER, RESULTS_FILE_NAME);
+        var subfolder = Path.Combine(INPUT_FOLDER, "notes");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(subfolder, "log.csv")), Is.False, "A run writes into the output folder itself, never into one of its subfolders.");
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(subfolder, RESULTS_FILE_NAME)), Is.False);
+            Assert.That(outputFolder.IsOutputOfRun(Path.Combine(subfolder, "report_result.md")), Is.False);
+        });
+    }
+}