using System.Globalization;
using AIStudio.Settings.DataModel;
namespace AIStudio.Tools;
///
/// Which files below a folder a folder-based feature works with.
///
///
/// The file patterns select files, while the exclusion list and the rule for hidden files and
/// folders leave some out again, cf. FileNamePatterns, FileExclusions, and DirectoryFileTraversal.
/// Data sources, their information dialog, and the batch processing assistant all decide through
/// this class, so the files the dialog counts are the files which get indexed.
///
/// The parsed file patterns.
/// The parsed list of excluded files and folders.
/// Whether files in subfolders are selected as well.
/// Whether hidden files and the files in hidden folders are selected as well.
/// The largest file the feature works with, in megabytes, cf. IsTooLarge.
internal sealed class FolderFileSelection(IReadOnlyList filePatterns, IReadOnlyList exclusions, bool includeSubdirectories, bool includeHiddenFilesAndFolders, int maxFileSizeMegabytes)
{
///
/// The size limit of files unless the user or the organization chooses another one.
///
///
/// A safety net, not a brake on costs: large scanned PDFs, often 100 to 500 MB, still pass. It
/// catches the images of virtual machines, ISO files, database dumps, and huge logs, which end
/// up in a selection by * or by an unknown extension. There is no value for "no limit"; the
/// limit can be raised instead.
///
public const int DEFAULT_MAX_FILE_SIZE_MEGABYTES = 1024;
///
/// The smallest size limit which can be set.
///
public const int SMALLEST_MAX_FILE_SIZE_MEGABYTES = 1;
private const long BYTES_PER_MEGABYTE = 1024 * 1024;
///
/// Every file below a folder, hidden ones included, whatever its size. Only what nobody ever
/// wants is left out.
///
public static readonly FolderFileSelection EVERY_FILE = new(["*"], [], true, true, int.MaxValue);
///
/// The files a local folder data source indexes.
///
public static FolderFileSelection Of(DataSourceLocalDirectory dataSource) => new(
FileNamePatterns.Parse(dataSource.EffectiveFilePatterns),
FileExclusions.Parse(dataSource.EffectiveExcludePatterns),
dataSource.IncludeSubdirectories,
dataSource.IncludeHiddenFilesAndFolders,
dataSource.EffectiveMaxFileSizeMegabytes);
///
/// The size limit in bytes.
///
public long MaxFileSizeBytes => MegabytesToBytes(maxFileSizeMegabytes);
public static long MegabytesToBytes(int megabytes) => megabytes * BYTES_PER_MEGABYTE;
///
/// Describes the selection as text, which is the same for two selections exactly when they
/// select the same files.
///
///
/// File patterns and exclusions ignore case and order, so the description does as well. It names
/// every argument of this class: whoever adds one adds it here, or changing it would not restart
/// a run which still works with the old selection, cf. DataSourceEmbeddingService.BuildSelectionSignature.
///
public string Describe() => string.Create(
CultureInfo.InvariantCulture,
$"patterns:{Canonical(filePatterns)}|exclusions:{Canonical(exclusions)}|subfolders:{includeSubdirectories}|hidden:{includeHiddenFilesAndFolders}|max-mb:{maxFileSizeMegabytes}");
///
/// Enumerates the files below a folder which are neither hidden nor in an excluded or a hidden
/// folder. Whether one of them is selected tells Selects.
///
/// The folder.
/// Told about every folder the exclusion list leaves out.
/// Whether the caller wants a folder entered, e.g., not the output folder of a batch run.
/// Told about every folder which could not be read.
/// The full paths of the files.
public IEnumerable EnumerateCandidates(
string rootPath,
Action? reportExcludedFolder = null,
Func? includeDirectory = null,
Action? reportInaccessibleDirectory = null) => DirectoryFileTraversal.EnumerateFiles(
rootPath,
includeSubdirectories,
includeHiddenFilesAndFolders,
includeHiddenFile: filePath => FileNamePatterns.NamesHiddenFile(filePath, filePatterns),
includeDirectory: directoryPath => this.IsNotExcluded(directoryPath, reportExcludedFolder) && includeDirectory?.Invoke(directoryPath) is not false,
reportInaccessibleDirectory: reportInaccessibleDirectory);
///
/// Whether a file which EnumerateCandidates found is selected: the exclusion list does not name
/// it, and one of the file patterns does. Exclusion comes first.
///
public bool Selects(string filePath) => !FileExclusions.ExcludesFile(filePath, exclusions) && FileNamePatterns.Matches(filePath, filePatterns);
///
/// Whether a selected file is larger than the limit. A file of exactly the limit is not.
///
///
/// Kept apart from Selects on purpose: a file the user selected but which is too large is no
/// intentional exclusion, so the caller tells the user about it.
///
public bool IsTooLarge(FileInfo file) => file.Length > this.MaxFileSizeBytes;
///
/// Whether a path below the folder lies where EnumerateCandidates never looks: in a hidden or
/// an excluded folder, or as a hidden or an excluded file itself.
///
///
/// Meant for the file system watcher. It judges each folder between the root and the path by
/// its name, so it also works for a path which is gone, e.g., after a rename. Whether the
/// operating system hid something can be told only while it exists. A path which is gone can
/// have been a file or a folder, so it is left out only when it would be either way.
///
/// The folder.
/// The path below it.
public bool LeavesOut(string rootPath, string path)
{
var relativePath = Path.GetRelativePath(rootPath, path);
//
// The folder itself, or a path outside of it, is nothing this selection has a say about:
//
if (relativePath is "." or ".." || relativePath.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) || Path.IsPathRooted(relativePath))
return false;
var names = relativePath.Split([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], StringSplitOptions.RemoveEmptyEntries);
if (!includeSubdirectories && names.Length > 1)
return true;
var folderPath = rootPath;
foreach (var folderName in names[..^1])
{
folderPath = Path.Combine(folderPath, folderName);
if (this.LeavesOutFolder(folderPath))
return true;
}
if (Directory.Exists(path))
return this.LeavesOutFolder(path);
if (File.Exists(path))
return this.LeavesOutFile(path);
return this.LeavesOutFolder(path) && this.LeavesOutFile(path);
}
///
/// Whether a file which is gone now could have been selected.
///
///
/// A selector reads the content of a file, which is gone. So any file which a selector may
/// have taken counts, i.e., every one but media, which never matches a selector.
///
public bool CouldHaveSelected(string filePath) => this.Selects(filePath)
|| (filePatterns.Any(FileNamePatterns.IsSelector)
&& !FileExclusions.ExcludesFile(filePath, exclusions)
&& !FileNamePatterns.IsMediaFile(filePath));
private bool LeavesOutFolder(string folderPath) => FileExclusions.ExcludesFolder(folderPath, exclusions)
|| (!includeHiddenFilesAndFolders && DirectoryFileTraversal.IsHiddenPath(folderPath));
private bool LeavesOutFile(string filePath) => FileExclusions.ExcludesFile(filePath, exclusions)
|| (!includeHiddenFilesAndFolders && DirectoryFileTraversal.IsHiddenPath(filePath) && !FileNamePatterns.NamesHiddenFile(filePath, filePatterns));
private static string Canonical(IEnumerable entries) => string.Join(';', entries
.Select(entry => entry.ToLowerInvariant())
.Distinct(StringComparer.Ordinal)
.Order(StringComparer.Ordinal));
private bool IsNotExcluded(string directoryPath, Action? reportExcludedFolder)
{
if (!FileExclusions.ExcludesFolder(directoryPath, exclusions))
return true;
reportExcludedFolder?.Invoke(directoryPath);
return false;
}
}