using System.Globalization; using AIStudio.Settings.DataModel; namespace AIStudio.Tools; /// /// Which files below a folder a folder-based feature works with. /// /// /// The file patterns select files, while the exclusion list and the rule for hidden files and /// folders leave some out again, cf. FileNamePatterns, FileExclusions, and DirectoryFileTraversal. /// Data sources, their information dialog, and the batch processing assistant all decide through /// this class, so the files the dialog counts are the files which get indexed. /// /// The parsed file patterns. /// The parsed list of excluded files and folders. /// Whether files in subfolders are selected as well. /// Whether hidden files and the files in hidden folders are selected as well. /// The largest file the feature works with, in megabytes, cf. IsTooLarge. internal sealed class FolderFileSelection(IReadOnlyList filePatterns, IReadOnlyList exclusions, bool includeSubdirectories, bool includeHiddenFilesAndFolders, int maxFileSizeMegabytes) { /// /// The size limit of files unless the user or the organization chooses another one. /// /// /// A safety net, not a brake on costs: large scanned PDFs, often 100 to 500 MB, still pass. It /// catches the images of virtual machines, ISO files, database dumps, and huge logs, which end /// up in a selection by * or by an unknown extension. There is no value for "no limit"; the /// limit can be raised instead. /// public const int DEFAULT_MAX_FILE_SIZE_MEGABYTES = 1024; /// /// The smallest size limit which can be set. /// public const int SMALLEST_MAX_FILE_SIZE_MEGABYTES = 1; private const long BYTES_PER_MEGABYTE = 1024 * 1024; /// /// Every file below a folder, hidden ones included, whatever its size. Only what nobody ever /// wants is left out. /// public static readonly FolderFileSelection EVERY_FILE = new(["*"], [], true, true, int.MaxValue); /// /// The files a local folder data source indexes. /// public static FolderFileSelection Of(DataSourceLocalDirectory dataSource) => new( FileNamePatterns.Parse(dataSource.EffectiveFilePatterns), FileExclusions.Parse(dataSource.EffectiveExcludePatterns), dataSource.IncludeSubdirectories, dataSource.IncludeHiddenFilesAndFolders, dataSource.EffectiveMaxFileSizeMegabytes); /// /// The size limit in bytes. /// public long MaxFileSizeBytes => MegabytesToBytes(maxFileSizeMegabytes); public static long MegabytesToBytes(int megabytes) => megabytes * BYTES_PER_MEGABYTE; /// /// Describes the selection as text, which is the same for two selections exactly when they /// select the same files. /// /// /// File patterns and exclusions ignore case and order, so the description does as well. It names /// every argument of this class: whoever adds one adds it here, or changing it would not restart /// a run which still works with the old selection, cf. DataSourceEmbeddingService.BuildSelectionSignature. /// public string Describe() => string.Create( CultureInfo.InvariantCulture, $"patterns:{Canonical(filePatterns)}|exclusions:{Canonical(exclusions)}|subfolders:{includeSubdirectories}|hidden:{includeHiddenFilesAndFolders}|max-mb:{maxFileSizeMegabytes}"); /// /// Enumerates the files below a folder which are neither hidden nor in an excluded or a hidden /// folder. Whether one of them is selected tells Selects. /// /// The folder. /// Told about every folder the exclusion list leaves out. /// Whether the caller wants a folder entered, e.g., not the output folder of a batch run. /// Told about every folder which could not be read. /// The full paths of the files. public IEnumerable EnumerateCandidates( string rootPath, Action? reportExcludedFolder = null, Func? includeDirectory = null, Action? reportInaccessibleDirectory = null) => DirectoryFileTraversal.EnumerateFiles( rootPath, includeSubdirectories, includeHiddenFilesAndFolders, includeHiddenFile: filePath => FileNamePatterns.NamesHiddenFile(filePath, filePatterns), includeDirectory: directoryPath => this.IsNotExcluded(directoryPath, reportExcludedFolder) && includeDirectory?.Invoke(directoryPath) is not false, reportInaccessibleDirectory: reportInaccessibleDirectory); /// /// Whether a file which EnumerateCandidates found is selected: the exclusion list does not name /// it, and one of the file patterns does. Exclusion comes first. /// public bool Selects(string filePath) => !FileExclusions.ExcludesFile(filePath, exclusions) && FileNamePatterns.Matches(filePath, filePatterns); /// /// Whether a selected file is larger than the limit. A file of exactly the limit is not. /// /// /// Kept apart from Selects on purpose: a file the user selected but which is too large is no /// intentional exclusion, so the caller tells the user about it. /// public bool IsTooLarge(FileInfo file) => file.Length > this.MaxFileSizeBytes; /// /// Whether a path below the folder lies where EnumerateCandidates never looks: in a hidden or /// an excluded folder, or as a hidden or an excluded file itself. /// /// /// Meant for the file system watcher. It judges each folder between the root and the path by /// its name, so it also works for a path which is gone, e.g., after a rename. Whether the /// operating system hid something can be told only while it exists. A path which is gone can /// have been a file or a folder, so it is left out only when it would be either way. /// /// The folder. /// The path below it. public bool LeavesOut(string rootPath, string path) { var relativePath = Path.GetRelativePath(rootPath, path); // // The folder itself, or a path outside of it, is nothing this selection has a say about: // if (relativePath is "." or ".." || relativePath.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) || Path.IsPathRooted(relativePath)) return false; var names = relativePath.Split([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], StringSplitOptions.RemoveEmptyEntries); if (!includeSubdirectories && names.Length > 1) return true; var folderPath = rootPath; foreach (var folderName in names[..^1]) { folderPath = Path.Combine(folderPath, folderName); if (this.LeavesOutFolder(folderPath)) return true; } if (Directory.Exists(path)) return this.LeavesOutFolder(path); if (File.Exists(path)) return this.LeavesOutFile(path); return this.LeavesOutFolder(path) && this.LeavesOutFile(path); } /// /// Whether a file which is gone now could have been selected. /// /// /// A selector reads the content of a file, which is gone. So any file which a selector may /// have taken counts, i.e., every one but media, which never matches a selector. /// public bool CouldHaveSelected(string filePath) => this.Selects(filePath) || (filePatterns.Any(FileNamePatterns.IsSelector) && !FileExclusions.ExcludesFile(filePath, exclusions) && !FileNamePatterns.IsMediaFile(filePath)); private bool LeavesOutFolder(string folderPath) => FileExclusions.ExcludesFolder(folderPath, exclusions) || (!includeHiddenFilesAndFolders && DirectoryFileTraversal.IsHiddenPath(folderPath)); private bool LeavesOutFile(string filePath) => FileExclusions.ExcludesFile(filePath, exclusions) || (!includeHiddenFilesAndFolders && DirectoryFileTraversal.IsHiddenPath(filePath) && !FileNamePatterns.NamesHiddenFile(filePath, filePatterns)); private static string Canonical(IEnumerable entries) => string.Join(';', entries .Select(entry => entry.ToLowerInvariant()) .Distinct(StringComparer.Ordinal) .Order(StringComparer.Ordinal)); private bool IsNotExcluded(string directoryPath, Action? reportExcludedFolder) { if (!FileExclusions.ExcludesFolder(directoryPath, exclusions)) return true; reportExcludedFolder?.Invoke(directoryPath); return false; } }