using System.IO.Enumeration; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; namespace AIStudio.Tools; /// /// Parses and matches the semicolon-separated file-name patterns and semantic selectors used by /// folder-based features. /// internal static class FileNamePatterns { /// /// A semantic pattern for every document AI Studio can read: the known document types, such as /// PDF, Office files, and source code, plus every other file whose contents can be read as text. /// Unlike *, it does not select media or binary files. /// public const string ALL_DOCUMENTS_PATTERN = ":documents:"; /// /// A semantic pattern for every file whose contents can be read as text, including files /// without a known extension. Unlike *, it does not select media or binary documents. /// public const string ALL_TEXT_FILES_PATTERN = ":text:"; /// /// What every selector begins and ends with, so that a mistyped selector can be told apart from /// a file name pattern. /// private const char SELECTOR_DELIMITER = ':'; private enum ValidationIssue { NONE, MISSING, UNKNOWN_SELECTOR, RECURSIVE_WILDCARD, PATH, INVALID_CHARACTER, } /// /// What a file name pattern must not contain: what no file name may contain, apart from the wildcards. /// internal static readonly char[] INVALID_PATTERN_CHARACTERS = Path.GetInvalidFileNameChars() .Where(character => character is not '*' and not '?') .ToArray(); private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileNamePatterns).Namespace, nameof(FileNamePatterns)); /// /// Splits the list, ignoring empty entries, including a trailing semicolon. /// public static string[] Parse(string? patterns) => (patterns ?? string.Empty) .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); public static string Normalize(string? patterns) => string.Join(';', Parse(patterns)); public static bool Matches(string filePath, IReadOnlyList patterns) { var fileName = Path.GetFileName(filePath); return !string.IsNullOrWhiteSpace(fileName) && patterns.Any(pattern => MatchesPattern(filePath, fileName, pattern)); } /// /// Whether one of the patterns asks for a hidden file by itself. /// /// /// As in a shell, only a pattern which begins with a dot, such as .gitignore or .*, selects a /// hidden file while hidden files are left out. Neither * nor a selector does, so .env does not /// end up in an index by accident, cf. DirectoryFileTraversal. /// public static bool NamesHiddenFile(string filePath, IReadOnlyList patterns) { var fileName = Path.GetFileName(filePath); return !string.IsNullOrWhiteSpace(fileName) && patterns.Any(pattern => pattern.StartsWith(DirectoryFileTraversal.HIDDEN_NAME_PREFIX) && FileSystemName.MatchesSimpleExpression(pattern, fileName, ignoreCase: true)); } /// /// Whether the patterns may select audio or video files, which a data source sends to the transcription provider. /// /// /// Deliberately cautious, since it answers without looking at a single file: a pattern which /// may match any extension, such as * or meeting*, counts. The selectors never select media. /// /// The parsed patterns. /// True when audio or video files may match. public static bool MaySelectMedia(IReadOnlyList patterns) => patterns.Any(MaySelectMedia); private static bool MaySelectMedia(string pattern) { if (pattern.StartsWith(SELECTOR_DELIMITER) && pattern.EndsWith(SELECTOR_DELIMITER)) return false; var extensionStart = pattern.LastIndexOf('.'); if (extensionStart < 0) return pattern.IndexOfAny(['*', '?']) >= 0; var extension = pattern[(extensionStart + 1)..]; if (extension.IndexOfAny(['*', '?']) >= 0) return true; return FileTypes.IsAllowedPath($"media.{extension}", FileTypes.AUDIO, FileTypes.VIDEO); } public static bool AreEquivalent(IReadOnlyCollection? left, IReadOnlyCollection? right) { if (left is null || right is null) return left is null && right is null; return left.ToHashSet(StringComparer.OrdinalIgnoreCase).SetEquals(right); } public static bool IsValid(string? patterns) => GetValidationIssue(patterns) is ValidationIssue.NONE; /// /// Validates a semicolon-separated list of simple file-name patterns and semantic selectors. /// public static string? Validate(string? patterns) => GetValidationIssue(patterns) switch { ValidationIssue.NONE => null, ValidationIssue.MISSING => TB("Please provide at least one file pattern, e.g., :documents: or *.txt. Separate multiple patterns with a semicolon."), ValidationIssue.UNKNOWN_SELECTOR => TB("One of the file patterns is an unknown selector. Please use :documents: or :text:, or a file name pattern such as *.txt."), ValidationIssue.RECURSIVE_WILDCARD => TB("Please use only single asterisks as wildcards, e.g., *.txt or report-*.md."), ValidationIssue.PATH => TB("Please use file name patterns without folder paths, e.g., *.txt or report-*.md."), ValidationIssue.INVALID_CHARACTER => TB("One of the file patterns contains an invalid character."), _ => throw new ArgumentOutOfRangeException(nameof(patterns)), }; private static ValidationIssue GetValidationIssue(string? patterns) { var individualPatterns = Parse(patterns); if (individualPatterns.Length == 0) return ValidationIssue.MISSING; foreach (var pattern in individualPatterns) { if (IsSelector(pattern)) continue; // // Colons are allowed in file names on macOS and Linux, so a mistyped selector such as // ':document:' would be a valid pattern which matches nothing. Every file would then // leave the index: // if (LooksLikeSelector(pattern)) return ValidationIssue.UNKNOWN_SELECTOR; if (pattern.Contains("**", StringComparison.Ordinal)) return ValidationIssue.RECURSIVE_WILDCARD; if (pattern is "." or ".." || pattern.EndsWith("..", StringComparison.Ordinal) || pattern.IndexOfAny([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar, '/', '\\']) >= 0) return ValidationIssue.PATH; if (pattern.IndexOfAny(INVALID_PATTERN_CHARACTERS) >= 0) return ValidationIssue.INVALID_CHARACTER; } return ValidationIssue.NONE; } private static bool MatchesPattern(string filePath, string fileName, string pattern) { // // Media never matches a selector: a data source cannot index it, and an SVG or a renamed // file would pass the check for readable text. The media check looks at the extension // only, so it comes first and saves reading the file: // if (IsSelector(pattern, ALL_DOCUMENTS_PATTERN)) return !IsMediaFile(filePath) && FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT); if (IsSelector(pattern, ALL_TEXT_FILES_PATTERN)) return !IsMediaFile(filePath) && FileTypes.IsAllowedPath(filePath, FileTypes.READABLE_TEXT); return FileSystemName.MatchesSimpleExpression(pattern, fileName, ignoreCase: true); } internal static bool IsSelector(string pattern) => IsSelector(pattern, ALL_DOCUMENTS_PATTERN) || IsSelector(pattern, ALL_TEXT_FILES_PATTERN); private static bool IsSelector(string pattern, string selector) => string.Equals(pattern, selector, StringComparison.OrdinalIgnoreCase); internal static bool LooksLikeSelector(string pattern) => pattern.StartsWith(SELECTOR_DELIMITER) && pattern.EndsWith(SELECTOR_DELIMITER); internal static bool IsMediaFile(string filePath) => FileTypes.IsAllowedPath(filePath, FileTypes.MEDIA); }