using System.Text.Json; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools; using AIStudio.Tools.Rust; using AIStudio.Tools.Validation; namespace AIStudio.Tests.Tools; public sealed class LocalDirectoryFilePatternsTests { [Test] public void MissingDirectorySelectionSettingsUseTheirDefaults() { var dataSource = JsonSerializer.Deserialize("{}", SettingsManager.JSON_OPTIONS); Assert.Multiple(() => { Assert.That(dataSource.FilePatterns, Is.EqualTo(FileNamePatterns.ALL_DOCUMENTS_PATTERN), "A data source from before file patterns has to keep indexing what it indexed."); Assert.That(dataSource.IncludeSubdirectories, Is.True); Assert.That(dataSource.IncludeHiddenFilesAndFolders, Is.False); }); } [Test] public void ExistingBatchSettingsDefaultToSkippingHiddenFilesAndFolders() { var settings = JsonSerializer.Deserialize("{}", SettingsManager.JSON_OPTIONS); Assert.Multiple(() => { Assert.That(settings, Is.Not.Null); Assert.That(settings!.IncludeSubdirectories, Is.EqualTo(DataBatchProcessing.DEFAULT_INCLUDE_SUBDIRECTORIES)); Assert.That(settings.IncludeHiddenFilesAndFolders, Is.EqualTo(DataBatchProcessing.DEFAULT_INCLUDE_HIDDEN_FILES_AND_FOLDERS)); }); } [TestCase(null)] [TestCase("")] [TestCase(" ")] [TestCase("*.txt;")] [TestCase("folder/*.txt")] public void InvalidStoredPatternsFallBackWithoutRemovingExistingDocuments(string? storedPatterns) { var dataSource = new DataSourceLocalDirectory { FilePatterns = storedPatterns! }; Assert.That(dataSource.EffectiveFilePatterns, Is.EqualTo(DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS)); } [Test] public void PatternsAreSplitAndTrimmed() { Assert.That(FileNamePatterns.Parse(" *.txt ; *.tf ; .gitignore "), Is.EqualTo(new[] { "*.txt", "*.tf", ".gitignore" })); } [Test] public void NormalizingPatternsRemovesCaseInsensitiveDuplicates() { Assert.That(FileNamePatterns.Normalize(" *.txt ; *.TF ; *.tf ; :text: ; :TEXT: "), Is.EqualTo("*.txt;*.TF;:text:")); } [Test] public void PatternSetsAreEquivalentRegardlessOfOrderOrDuplicates() { Assert.That( FileNamePatterns.AreEquivalent(["*.txt", "*.tf", "*.TF"], ["*.TF", "*.TXT"]), Is.True); } [Test] public void FilePatternsSurvivePolymorphicSettingsSerialization() { IDataSource written = new DataSourceLocalDirectory { Type = DataSourceType.LOCAL_DIRECTORY, FilePatterns = "*.txt;*.tf;.gitignore", IncludeSubdirectories = false, IncludeHiddenFilesAndFolders = true, }; var json = JsonSerializer.Serialize(written, SettingsManager.JSON_OPTIONS); var read = JsonSerializer.Deserialize(json, SettingsManager.JSON_OPTIONS); Assert.Multiple(() => { Assert.That(json, Does.Not.Contain(nameof(DataSourceLocalDirectory.EffectiveFilePatterns))); Assert.That(read, Is.TypeOf()); Assert.That(((DataSourceLocalDirectory)read!).FilePatterns, Is.EqualTo("*.txt;*.tf;.gitignore")); Assert.That(((DataSourceLocalDirectory)read).IncludeSubdirectories, Is.False); Assert.That(((DataSourceLocalDirectory)read).IncludeHiddenFilesAndFolders, Is.True); }); } [Test] public async Task DirectoryStatisticsCanExcludeSubdirectories() { var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-nonrecursive-statistics-{Guid.NewGuid():N}"); var subdirectoryPath = Path.Combine(directoryPath, "nested"); Directory.CreateDirectory(subdirectoryPath); try { await File.WriteAllBytesAsync(Path.Combine(directoryPath, "included.txt"), new byte[5]); await File.WriteAllBytesAsync(Path.Combine(subdirectoryPath, "excluded.txt"), new byte[10]); long totalSize = 0; long fileCount = 0; var reportedFiles = new List(); await new DirectoryInfo(directoryPath).DetermineContentSize( size => totalSize = size, count => fileCount = count, reportedFiles.Add, selection: new FolderFileSelection(["*"], [], false, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES)); Assert.Multiple(() => { Assert.That(totalSize, Is.EqualTo(5)); Assert.That(fileCount, Is.EqualTo(1)); Assert.That(reportedFiles, Has.Count.EqualTo(1)); Assert.That(reportedFiles[0], Does.EndWith("included.txt")); }); } finally { Directory.Delete(directoryPath, true); } } [Test] public async Task DirectoryStatisticsUseTheConfiguredHiddenFilesAndFoldersRule() { var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-hidden-statistics-{Guid.NewGuid():N}"); var hiddenDirectoryPath = Path.Combine(directoryPath, ".hidden"); Directory.CreateDirectory(hiddenDirectoryPath); try { await File.WriteAllBytesAsync(Path.Combine(directoryPath, "visible.txt"), new byte[5]); await File.WriteAllBytesAsync(Path.Combine(directoryPath, ".env"), new byte[20]); await File.WriteAllBytesAsync(Path.Combine(hiddenDirectoryPath, "hidden.txt"), new byte[10]); long visibleSize = 0; long visibleCount = 0; await new DirectoryInfo(directoryPath).DetermineContentSize( size => visibleSize = size, count => visibleCount = count, _ => { }, selection: new FolderFileSelection(["*"], [], true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES)); long completeSize = 0; long completeCount = 0; await new DirectoryInfo(directoryPath).DetermineContentSize( size => completeSize = size, count => completeCount = count, _ => { }, selection: new FolderFileSelection(["*"], [], true, true, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES)); Assert.Multiple(() => { Assert.That(visibleSize, Is.EqualTo(5)); Assert.That(visibleCount, Is.EqualTo(1)); Assert.That(completeSize, Is.EqualTo(35)); Assert.That(completeCount, Is.EqualTo(3)); }); } finally { Directory.Delete(directoryPath, true); } } [TestCase("notes.txt")] [TestCase("main.TF")] [TestCase(".gitignore")] [TestCase("nested/folder/variables.tf")] public void MatchingUsesTheFileName(string path) { var patterns = FileNamePatterns.Parse("*.txt;*.tf;.gitignore"); Assert.That(FileNamePatterns.Matches(path, patterns), Is.True); } [TestCase("Dockerfile", "Dockerfile")] [TestCase("Dockerfile*", "Dockerfile.development")] [TestCase("Containerfile*", "nested/Containerfile.arm64")] [TestCase("Jenkinsfile*", "Jenkinsfile.release")] [TestCase("Makefile", "nested/Makefile")] [TestCase("GNUmakefile", "GNUmakefile")] [TestCase("Procfile", "Procfile")] [TestCase("Vagrantfile", "Vagrantfile")] [TestCase("Caddyfile*", "Caddyfile.production")] [TestCase("BUILD;WORKSPACE", "WORKSPACE")] [TestCase("*.env", ".env")] [TestCase("deployment notes*#?.txt", "deployment notes (final) [β] #1.txt")] public void GlobPatternsMatchSourceLikeAndUnusualFileNames(string pattern, string path) { Assert.That(FileNamePatterns.Matches(path, FileNamePatterns.Parse(pattern)), Is.True); } [Test] public void GlobPatternsMatchFileNamesAtTheEndOfExtremelyLongPaths() { var directorySegments = Enumerable.Range(0, 8) .Select(index => $"nested-{index}-{new string('x', 32)}") .ToArray(); var path = Path.Combine([.. directorySegments, "Dockerfile.development"]); Assert.Multiple(() => { Assert.That(path.Length, Is.GreaterThan(300)); Assert.That(FileNamePatterns.Matches(path, FileNamePatterns.Parse("Dockerfile*")), Is.True); }); } [TestCase("image.png")] [TestCase("notes.txt.bak")] [TestCase("gitignore")] public void FilesOutsideThePatternsDoNotMatch(string path) { var patterns = FileNamePatterns.Parse("*.txt;*.tf;.gitignore"); Assert.That(FileNamePatterns.Matches(path, patterns), Is.False); } [TestCase(".gitignore")] [TestCase("LICENSE")] [TestCase("recording.mp3")] public void CatchAllPatternIncludesEveryFileName(string fileName) { Assert.That(FileNamePatterns.Matches(fileName, FileNamePatterns.Parse("*")), Is.True); } [Test] public async Task ReadableTextStillHasToMatchAFilePattern() { var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-readable-pattern-{Guid.NewGuid():N}"); var filePath = Path.Combine(directoryPath, "main.tf"); Directory.CreateDirectory(directoryPath); try { await File.WriteAllTextAsync(filePath, "resource \"example\" \"main\" {}"); Assert.Multiple(() => { Assert.That(FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT), Is.True); Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse("*.txt")), Is.False); Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse(":text:")), Is.True); Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse("*")), Is.True); }); } finally { Directory.Delete(directoryPath, true); } } [Test] public async Task AllTextFilesPatternRejectsBinaryAndMediaFiles() { var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-text-pattern-binary-{Guid.NewGuid():N}"); Directory.CreateDirectory(directoryPath); try { var binaryPath = Path.Combine(directoryPath, "archive.data"); var mediaPath = Path.Combine(directoryPath, "recording.mp3"); var imagePath = Path.Combine(directoryPath, "drawing.svg"); await File.WriteAllBytesAsync(binaryPath, [0x50, 0x4B, 0x03, 0x04, 0x00]); await File.WriteAllTextAsync(mediaPath, "This has readable bytes but a media file name."); await File.WriteAllTextAsync(imagePath, ""); var patterns = FileNamePatterns.Parse(":text:"); Assert.Multiple(() => { Assert.That(FileNamePatterns.Matches(binaryPath, patterns), Is.False); Assert.That(FileNamePatterns.Matches(mediaPath, patterns), Is.False); Assert.That(FileNamePatterns.Matches(imagePath, patterns), Is.False, "An image cannot be indexed, even when it is written as text."); }); } finally { Directory.Delete(directoryPath, true); } } [TestCase(".env", "*")] [TestCase(".env", ":text:")] [TestCase(".env", ":documents:")] [TestCase(".env", "*.env")] [TestCase("notes.txt", ".*")] public void OnlyAPatternWhichBeginsWithADotNamesAHiddenFile(string fileName, string patterns) { Assert.That(FileNamePatterns.NamesHiddenFile(fileName, FileNamePatterns.Parse(patterns)), Is.False); } [TestCase(".env", ".env")] [TestCase(".gitignore", "*.md;.gitignore")] [TestCase(".env.local", ".env*")] [TestCase(".editorconfig", ".*")] public void APatternWhichBeginsWithADotNamesHiddenFiles(string fileName, string patterns) { Assert.That(FileNamePatterns.NamesHiddenFile(fileName, FileNamePatterns.Parse(patterns)), Is.True); } [TestCase("report.pdf")] [TestCase("letter.docx")] [TestCase("budget.xlsx")] [TestCase("notes.md")] [TestCase("Program.cs")] public void DocumentsSelectorKeepsWhatFolderDataSourcesIndexedBefore(string fileName) { // Known document types are told by their extension, so the file needs no content: var filePath = Path.Combine(Path.GetTempPath(), "test-data", fileName); Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse(FileNamePatterns.ALL_DOCUMENTS_PATTERN)), Is.True); } [Test] public async Task DocumentsSelectorAddsReadableTextButNoMediaOrBinaries() { var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-documents-pattern-{Guid.NewGuid():N}"); Directory.CreateDirectory(directoryPath); try { var terraformPath = Path.Combine(directoryPath, "main.tf"); var photoPath = Path.Combine(directoryPath, "photo.png"); var drawingPath = Path.Combine(directoryPath, "drawing.svg"); var recordingPath = Path.Combine(directoryPath, "recording.mp3"); var archivePath = Path.Combine(directoryPath, "archive.zip"); await File.WriteAllTextAsync(terraformPath, "resource \"example\" \"main\" {}"); await File.WriteAllBytesAsync(photoPath, [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00]); await File.WriteAllTextAsync(drawingPath, ""); await File.WriteAllTextAsync(recordingPath, "This has readable bytes but a media file name."); await File.WriteAllBytesAsync(archivePath, [0x50, 0x4B, 0x03, 0x04, 0x00]); var patterns = FileNamePatterns.Parse(FileNamePatterns.ALL_DOCUMENTS_PATTERN); Assert.Multiple(() => { Assert.That(FileNamePatterns.Matches(terraformPath, patterns), Is.True, "Readable text is what the selector adds to the documents."); Assert.That(FileNamePatterns.Matches(photoPath, patterns), Is.False); Assert.That(FileNamePatterns.Matches(drawingPath, patterns), Is.False, "An image cannot be indexed, even when it is written as text."); Assert.That(FileNamePatterns.Matches(recordingPath, patterns), Is.False); Assert.That(FileNamePatterns.Matches(archivePath, patterns), Is.False); }); } finally { Directory.Delete(directoryPath, true); } } [Test] public async Task ExplicitGlobSelectsMatchingBinaryContent() { var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-explicit-binary-pattern-{Guid.NewGuid():N}"); var filePath = Path.Combine(directoryPath, "archive.data"); Directory.CreateDirectory(directoryPath); try { await File.WriteAllBytesAsync(filePath, [0x50, 0x4B, 0x03, 0x04, 0x00]); Assert.Multiple(() => { Assert.That(FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT), Is.False); Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse("*.data")), Is.True); }); } finally { Directory.Delete(directoryPath, true); } } [Test] public async Task DirectoryStatisticsIncludeOnlyMatchingFiles() { var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-pattern-statistics-{Guid.NewGuid():N}"); Directory.CreateDirectory(directoryPath); try { await File.WriteAllBytesAsync(Path.Combine(directoryPath, "included.tf"), new byte[5]); await File.WriteAllBytesAsync(Path.Combine(directoryPath, "excluded.md"), new byte[10]); long totalSize = 0; long fileCount = 0; var reportedFiles = new List(); var selection = new FolderFileSelection(FileNamePatterns.Parse("*.tf"), [], true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES); await new DirectoryInfo(directoryPath).DetermineContentSize( size => totalSize = size, count => fileCount = count, reportedFiles.Add, selection: selection); Assert.Multiple(() => { Assert.That(totalSize, Is.EqualTo(5)); Assert.That(fileCount, Is.EqualTo(1)); Assert.That(reportedFiles, Has.Count.EqualTo(1)); Assert.That(reportedFiles[0], Does.EndWith("included.tf")); }); } finally { Directory.Delete(directoryPath, true); } } [TestCase("*.txt;*.tf;.gitignore")] [TestCase("*")] [TestCase(":text:")] [TestCase(":TEXT:;*.pdf")] [TestCase(":documents:")] [TestCase(":DOCUMENTS:;*.tf")] [TestCase("report-??.md")] public void DataSourceValidationAcceptsSimplePatterns(string patterns) { Assert.That(new DataSourceValidation().ValidateFilePatterns(patterns), Is.Null); } [TestCase("")] [TestCase("*.txt;")] [TestCase("**/*.txt")] [TestCase("folder/*.txt")] [TestCase(":document:")] [TestCase("*.txt;:docs:")] public void DataSourceValidationRejectsInvalidPatterns(string patterns) { Assert.That(new DataSourceValidation().ValidateFilePatterns(patterns), Is.Not.Null); } }