MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-11-04 21:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added file patterns and exclusions to folder data sources (#1057)

Browse Source 
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and GitHub committed 2026-10-17 19:12:43 +02:00
1 parent 249fc778a2
commit 031874cebf
75 files changed
+4808 -589
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs
+43 -73
View File
@@ -1,4 +1,3 @@
using System.IO.Enumeration;
namespace AIStudio.Assistants.BatchProcessing;
@@ -15,35 +14,9 @@ public partial class AssistantBatchProcessing
return null;
}
private string? ValidateFilePatterns(string patterns)
{
if (string.IsNullOrWhiteSpace(patterns))
return T("Please provide at least one file pattern, e.g., *.pdf. Separate multiple patterns with a semicolon.");
private string? ValidateFilePatterns(string patterns) => FileNamePatterns.Validate(patterns);
var individualPatterns = patterns.Split(';');
if (individualPatterns.Any(string.IsNullOrWhiteSpace))
return T("Please remove empty file patterns. Separate valid patterns with a single semicolon.");
foreach (var patternEntry in individualPatterns)
{
var pattern = patternEntry.Trim();
if (pattern.Contains("**", StringComparison.Ordinal))
return T("Please use only single asterisks as wildcards, e.g., *.pdf or report-*.docx.");
if (pattern is "." or ".."
|| pattern.EndsWith("..", StringComparison.Ordinal)
|| pattern.IndexOfAny([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar, '/', '\\']) >= 0)
return T("Please use file name patterns without folder paths, e.g., *.pdf or report-*.docx.");
var invalidCharacters = Path.GetInvalidFileNameChars()
.Where(character => character is not '*' and not '?')
.ToArray();
if (pattern.IndexOfAny(invalidCharacters) >= 0)
return T("One of the file patterns contains an invalid character.");
}
return null;
}
private string? ValidateExcludePatterns(string patterns) => FileExclusions.Validate(patterns);
private string? ValidateCsvFileName(string fileName)
{
@@ -107,12 +80,9 @@ public partial class AssistantBatchProcessing
private IReadOnlyList<string> FindInputFiles(string resolvedOutputDirectory)
{
var patterns = this.filePatterns
.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.ToList();
var searchOption = this.includeSubdirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
var selection = new FolderFileSelection(FileNamePatterns.Parse(this.filePatterns), FileExclusions.Parse(this.excludePatterns), this.includeSubdirectories, this.includeHiddenFilesAndFolders, this.maxFileSizeMegabytes);
var files = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
this.numTooLargeFiles = 0;
var normalizedInputDirectory = TrimDirectorySeparator(Path.GetFullPath(this.inputDirectory));
var normalizedOutputDirectory = TrimDirectorySeparator(Path.GetFullPath(resolvedOutputDirectory));
@@ -127,55 +97,46 @@ public partial class AssistantBatchProcessing
// would also exclude a document named 'output-notes.md':
var outputDirectoryPrefix = normalizedOutputDirectory + Path.DirectorySeparatorChar;
foreach (var pattern in patterns)
foreach (var file in selection.EnumerateCandidates(
this.inputDirectory,
includeDirectory: directory => !isOutputSeparateFolder
|| !string.Equals(TrimDirectorySeparator(Path.GetFullPath(directory)), normalizedOutputDirectory, StringComparison.OrdinalIgnoreCase),
reportInaccessibleDirectory: (directory, exception) =>
throw new IOException($"Could not access directory '{directory}'.", exception)))
{
foreach (var file in Directory.EnumerateFiles(this.inputDirectory, pattern, searchOption))
var normalizedFile = Path.GetFullPath(file);
if (!selection.Selects(normalizedFile) || IsTranscriptArtifact(normalizedFile))
continue;
if (isOutputSeparateFolder)
{
var normalizedFile = Path.GetFullPath(file);
if (IsTranscriptArtifact(normalizedFile))
if (normalizedFile.StartsWith(outputDirectoryPrefix, StringComparison.OrdinalIgnoreCase))
continue;
if (isOutputSeparateFolder)
{
if (normalizedFile.StartsWith(outputDirectoryPrefix, StringComparison.OrdinalIgnoreCase))
continue;
}
else if (this.IsOwnOutputArtifact(normalizedFile))
continue;
// On Windows, a pattern with a three-character extension also
// matches longer extensions: '*.pdf' also returns 'report.pdfx'.
// We therefore check the pattern ourselves:
if (!MatchesAnyPattern(normalizedFile, patterns))
continue;
files.Add(normalizedFile);
}
else if (this.IsOwnOutputArtifact(normalizedFile))
continue;
//
// Left out without a status of its own: the file never becomes part of the run. The
// line below the file list tells the user, and the log names the limit:
//
if (selection.IsTooLarge(new FileInfo(normalizedFile)))
{
this.numTooLargeFiles++;
continue;
}
files.Add(normalizedFile);
}
if (this.numTooLargeFiles > 0)
this.Logger.LogInformation("Left out {NumTooLargeFiles} matching file(s) in '{InputDirectory}' from the batch run because they are larger than {MaxFileSize}.", this.numTooLargeFiles, this.inputDirectory, selection.MaxFileSizeBytes.FileSize());
return [.. files];
}
private static string TrimDirectorySeparator(string path) => path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
private static bool MatchesAnyPattern(string filePath, IReadOnlyList<string> patterns)
{
var fileName = Path.GetFileName(filePath);
foreach (var pattern in patterns)
{
// A pattern may contain a folder part, which does not take part in
// matching the file name:
var namePattern = Path.GetFileName(pattern);
if (string.IsNullOrWhiteSpace(namePattern))
continue;
if (FileSystemName.MatchesSimpleExpression(namePattern, fileName))
return true;
}
return false;
}
/// <summary>
/// Checks whether a file is an output artifact of this assistant. We need
/// this when the output folder is the input folder: without it, the results
@@ -211,6 +172,13 @@ public partial class AssistantBatchProcessing
{
await this.Form!.Validate();
//
// The exclusion list sits in the expert settings, which are folded away by default. Unfold
// them, so the user sees the message next to the list:
//
if (this.ValidateExcludePatterns(this.excludePatterns) is not null)
this.showExpertSettings = true;
var instructionIssue = this.ValidateInstructionSource();
if (instructionIssue is not null)
{
@@ -236,7 +204,9 @@ public partial class AssistantBatchProcessing
if (files.Count == 0)
{
this.AddInputIssue(T("No matching files were found in the selected folder."));
this.AddInputIssue(this.numTooLargeFiles > 0
? string.Format(T("Every matching file in the selected folder is larger than {0}. Raise the size limit to process them."), FolderFileSelection.MegabytesToBytes(this.maxFileSizeMegabytes).FileSize())
: T("No matching files were found in the selected folder."));
return null;
}