mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-04 15:27:21 +00:00
338 lines
13 KiB
C#
338 lines
13 KiB
C#
using System.Text.Json;
|
|
|
|
using AIStudio.Settings;
|
|
using AIStudio.Settings.DataModel;
|
|
using AIStudio.Tools;
|
|
using AIStudio.Tools.Services;
|
|
|
|
namespace AIStudio.Tests.Tools;
|
|
|
|
/// <summary>
|
|
/// Checks the list of excluded files and folders, which data sources and the batch processing
|
|
/// assistant share.
|
|
/// </summary>
|
|
public sealed class FileExclusionsTests
|
|
{
|
|
[TestCase("")]
|
|
[TestCase("bin/")]
|
|
[TestCase("*.min.js")]
|
|
[TestCase("cmake-build-*/")]
|
|
[TestCase("bin/;\r\n*.min.js;;\n")]
|
|
[TestCase(FileExclusions.DEFAULT_PATTERNS)]
|
|
public void ValidExclusionListsAreAccepted(string patterns)
|
|
{
|
|
Assert.That(FileExclusions.Validate(patterns), Is.Null);
|
|
}
|
|
|
|
[TestCase("a/b/")]
|
|
[TestCase("/bin")]
|
|
[TestCase("./")]
|
|
[TestCase(":text:")]
|
|
[TestCase("bin/;:documents:")]
|
|
[TestCase("**/bin/")]
|
|
public void InvalidExclusionListsAreRejected(string patterns)
|
|
{
|
|
Assert.That(FileExclusions.Validate(patterns), Is.Not.Null);
|
|
}
|
|
|
|
[TestCase(null, true)]
|
|
[TestCase("", true)]
|
|
[TestCase(" ; \r\n", true)]
|
|
[TestCase("build/;cmake-build-*/", true)]
|
|
[TestCase(".env/;archive.v1/;*.vmwarevm/", true)]
|
|
[TestCase("*.pdf", false)]
|
|
[TestCase("Dockerfile", false)]
|
|
[TestCase("node_modules", false)]
|
|
[TestCase("build/;*.txt", false)]
|
|
public void FolderFieldRequiresExplicitFolderRules(string? patterns, bool valid)
|
|
{
|
|
using (Assert.EnterMultipleScope())
|
|
{
|
|
Assert.That(FileExclusions.IsValidFolders(patterns), Is.EqualTo(valid));
|
|
Assert.That(FileExclusions.ValidateFolders(patterns) is null, Is.EqualTo(valid));
|
|
}
|
|
}
|
|
|
|
[TestCase(null, true)]
|
|
[TestCase("", true)]
|
|
[TestCase(" ; \r\n", true)]
|
|
[TestCase("*.pdf;draft-*;report-?.md", true)]
|
|
[TestCase("Dockerfile;Caddyfile;.gitignore", true)]
|
|
[TestCase("build", true)]
|
|
[TestCase("build/", false)]
|
|
[TestCase("*.tmp;obj/", false)]
|
|
public void FileFieldRejectsFolderRulesButAcceptsExtensionlessFiles(string? patterns, bool valid)
|
|
{
|
|
using (Assert.EnterMultipleScope())
|
|
{
|
|
Assert.That(FileExclusions.IsValidFiles(patterns), Is.EqualTo(valid));
|
|
Assert.That(FileExclusions.ValidateFiles(patterns) is null, Is.EqualTo(valid));
|
|
}
|
|
}
|
|
|
|
[TestCase(":text:")]
|
|
[TestCase(":documents:")]
|
|
[TestCase("src/build/")]
|
|
[TestCase("**/*.tmp")]
|
|
[TestCase("/")]
|
|
[TestCase("./")]
|
|
public void BothFieldValidatorsReuseTheExistingSyntaxErrors(string patterns)
|
|
{
|
|
using (Assert.EnterMultipleScope())
|
|
{
|
|
Assert.That(FileExclusions.IsValidFolders(patterns), Is.False);
|
|
Assert.That(FileExclusions.IsValidFiles(patterns), Is.False);
|
|
Assert.That(FileExclusions.ValidateFolders(patterns), Is.EqualTo(FileExclusions.Validate(patterns)));
|
|
Assert.That(FileExclusions.ValidateFiles(patterns), Is.EqualTo(FileExclusions.Validate(patterns)));
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public void StoredListsStillAcceptBothKindsOfRulesTogether()
|
|
{
|
|
const string PATTERNS = "bin/;*.pdf;Dockerfile;.git/";
|
|
using (Assert.EnterMultipleScope())
|
|
{
|
|
Assert.That(FileExclusions.IsValid(PATTERNS), Is.True);
|
|
Assert.That(FileExclusions.Validate(PATTERNS), Is.Null);
|
|
Assert.That(FileExclusions.ExcludesFolder("bin", FileExclusions.Parse(PATTERNS)), Is.True);
|
|
Assert.That(FileExclusions.ExcludesFile("report.pdf", FileExclusions.Parse(PATTERNS)), Is.True);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public void EntriesAreSeparatedBySemicolonsAndLineBreaks()
|
|
{
|
|
Assert.That(FileExclusions.Parse(" bin/ ;obj/\r\n\r\n*.min.js\nBIN/ "), Is.EqualTo(new[] { "bin/", "obj/", "*.min.js" }));
|
|
}
|
|
|
|
[Test]
|
|
public void GroupingSplitsFoldersFromFilesAndDropsDuplicates()
|
|
{
|
|
var groups = FileExclusions.Group(" bin/ ;*.min.js\r\nobj/;BIN/;;\n*.MIN.js");
|
|
|
|
using (Assert.EnterMultipleScope())
|
|
{
|
|
Assert.That(groups.Folders, Is.EqualTo("bin/; obj/"));
|
|
Assert.That(groups.Files, Is.EqualTo("*.min.js"));
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public void SettingsFromBeforeTheListGetTheDefault()
|
|
{
|
|
var dataSource = JsonSerializer.Deserialize<DataSourceLocalDirectory>("{}", SettingsManager.JSON_OPTIONS);
|
|
var batchSettings = JsonSerializer.Deserialize<DataBatchProcessing>("{}", SettingsManager.JSON_OPTIONS);
|
|
|
|
using (Assert.EnterMultipleScope())
|
|
{
|
|
Assert.That(dataSource.ExcludePatterns, Is.EqualTo(FileExclusions.DEFAULT_PATTERNS));
|
|
Assert.That(batchSettings!.ExcludePatterns, Is.EqualTo(FileExclusions.DEFAULT_PATTERNS));
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public void AnInvalidStoredListFallsBackToTheDefault()
|
|
{
|
|
var dataSource = new DataSourceLocalDirectory { ExcludePatterns = "src/bin/" };
|
|
|
|
Assert.That(dataSource.EffectiveExcludePatterns, Is.EqualTo(FileExclusions.DEFAULT_PATTERNS));
|
|
}
|
|
|
|
[TestCase("node_modules")]
|
|
[TestCase("release")]
|
|
[TestCase("cmake-build-debug")]
|
|
[TestCase("Windows 11.pvm")]
|
|
[TestCase("__MACOSX")]
|
|
[TestCase("$RECYCLE.BIN")]
|
|
public void TheDefaultListExcludesTheseFolders(string folderName)
|
|
{
|
|
var folderPath = Path.Combine(Path.GetTempPath(), "test-data", folderName);
|
|
|
|
Assert.That(FileExclusions.ExcludesFolder(folderPath, FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS)), Is.True);
|
|
}
|
|
|
|
[TestCase("docs")]
|
|
[TestCase("vendor")]
|
|
[TestCase("coverage")]
|
|
[TestCase("packages")]
|
|
[TestCase("binaries")]
|
|
public void TheDefaultListKeepsTheseFolders(string folderName)
|
|
{
|
|
var folderPath = Path.Combine(Path.GetTempPath(), "test-data", folderName);
|
|
|
|
Assert.That(FileExclusions.ExcludesFolder(folderPath, FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS)), Is.False);
|
|
}
|
|
|
|
[TestCase("package-lock.json")]
|
|
[TestCase("app.min.js")]
|
|
[TestCase("app.js.map")]
|
|
[TestCase("tokenizer.json")]
|
|
[TestCase("chat_tokenizer.json")]
|
|
[TestCase("tool.dll")]
|
|
[TestCase("index.sqlite-wal")]
|
|
[TestCase("disk.vmdk")]
|
|
public void TheDefaultListExcludesTheseFiles(string fileName)
|
|
{
|
|
var filePath = Path.Combine(Path.GetTempPath(), "test-data", fileName);
|
|
|
|
Assert.That(FileExclusions.ExcludesFile(filePath, FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS)), Is.True);
|
|
}
|
|
|
|
[TestCase("package.json")]
|
|
[TestCase("notes.md")]
|
|
[TestCase("city.map")]
|
|
[TestCase("tokenizer_config.json")]
|
|
[TestCase("bin")]
|
|
public void TheDefaultListKeepsTheseFiles(string fileName)
|
|
{
|
|
var filePath = Path.Combine(Path.GetTempPath(), "test-data", fileName);
|
|
|
|
Assert.That(FileExclusions.ExcludesFile(filePath, FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS)), Is.False, "A folder entry such as bin/ names folders only.");
|
|
}
|
|
|
|
[TestCase("Thumbs.db")]
|
|
[TestCase("desktop.ini")]
|
|
[TestCase(".DS_Store")]
|
|
[TestCase("Icon\r")]
|
|
[TestCase("~$report.docx")]
|
|
[TestCase("Report.lnk")]
|
|
public void MetadataOfOperatingSystemsIsAlwaysExcluded(string fileName)
|
|
{
|
|
Assert.That(FileExclusions.ExcludesFile(fileName, []), Is.True, "An empty list must not let them in.");
|
|
}
|
|
|
|
[Test]
|
|
public async Task ExcludedFoldersAreNotEnteredAndExclusionComesBeforeSelection()
|
|
{
|
|
var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-exclusions-{Guid.NewGuid():N}");
|
|
var packagePath = Path.Combine(directoryPath, "node_modules", "left-pad");
|
|
var binPath = Path.Combine(directoryPath, "src", "bin");
|
|
Directory.CreateDirectory(packagePath);
|
|
Directory.CreateDirectory(binPath);
|
|
|
|
try
|
|
{
|
|
var manifestPath = Path.Combine(directoryPath, "package.json");
|
|
var lockFilePath = Path.Combine(directoryPath, "package-lock.json");
|
|
var sourcePath = Path.Combine(directoryPath, "src", "app.ts");
|
|
await File.WriteAllTextAsync(manifestPath, "{}");
|
|
await File.WriteAllTextAsync(lockFilePath, "{}");
|
|
await File.WriteAllTextAsync(sourcePath, "export {};");
|
|
await File.WriteAllTextAsync(Path.Combine(packagePath, "index.js"), "module.exports = {};");
|
|
await File.WriteAllTextAsync(Path.Combine(binPath, "app.txt"), "build output");
|
|
|
|
var selection = new FolderFileSelection(FileNamePatterns.Parse("*.json;*.ts;*.js;*.txt"), FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS), true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
|
var excludedFolders = new List<string>();
|
|
var candidates = selection.EnumerateCandidates(directoryPath, reportExcludedFolder: excludedFolders.Add).ToList();
|
|
|
|
using (Assert.EnterMultipleScope())
|
|
{
|
|
Assert.That(candidates, Is.EquivalentTo(new[] { manifestPath, lockFilePath, sourcePath }), "Nothing below node_modules or bin is even looked at.");
|
|
Assert.That(excludedFolders.Select(Path.GetFileName), Is.EquivalentTo(new[] { "node_modules", "bin" }));
|
|
Assert.That(candidates.Where(selection.Selects), Is.EquivalentTo(new[] { manifestPath, sourcePath }), "*.json selects the lock file, but the list excludes it.");
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(directoryPath, true);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task MetadataOfOperatingSystemsIsLeftOutEvenWhenEverythingIsSelected()
|
|
{
|
|
var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-os-metadata-{Guid.NewGuid():N}");
|
|
var archiveFolderPath = Path.Combine(directoryPath, "__MACOSX");
|
|
Directory.CreateDirectory(archiveFolderPath);
|
|
|
|
try
|
|
{
|
|
var notesPath = Path.Combine(directoryPath, "notes.md");
|
|
await File.WriteAllTextAsync(notesPath, "Notes");
|
|
|
|
// As after a copy from a ZIP file: no system attribute, only the name tells.
|
|
await File.WriteAllTextAsync(Path.Combine(directoryPath, "desktop.ini"), "[.ShellClassInfo]");
|
|
await File.WriteAllBytesAsync(Path.Combine(directoryPath, "Thumbs.db"), [0, 1, 2]);
|
|
await File.WriteAllTextAsync(Path.Combine(archiveFolderPath, "notes.md"), "Resource fork");
|
|
|
|
var selection = new FolderFileSelection(["*"], [], true, true, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
|
|
|
Assert.That(selection.EnumerateCandidates(directoryPath), Is.EqualTo(new[] { notesPath }));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(directoryPath, true);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheConfigurationOfAVirtualMachineStaysOut()
|
|
{
|
|
var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-virtual-machine-{Guid.NewGuid():N}");
|
|
var machinePath = Path.Combine(directoryPath, "Windows 11.pvm");
|
|
Directory.CreateDirectory(machinePath);
|
|
|
|
try
|
|
{
|
|
await File.WriteAllTextAsync(Path.Combine(machinePath, "config.pvs"), "<ParallelsVirtualMachine/>");
|
|
var selection = new FolderFileSelection(FileNamePatterns.Parse(FileNamePatterns.ALL_DOCUMENTS_PATTERN), FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS), true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
|
|
|
Assert.That(selection.EnumerateCandidates(directoryPath), Is.Empty, "The readable text check would select config.pvs.");
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(directoryPath, true);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheCallerCanLeaveOutFurtherFolders()
|
|
{
|
|
var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-batch-exclusions-{Guid.NewGuid():N}");
|
|
var outputPath = Path.Combine(directoryPath, "ai-results");
|
|
var binPath = Path.Combine(directoryPath, "bin");
|
|
Directory.CreateDirectory(outputPath);
|
|
Directory.CreateDirectory(binPath);
|
|
|
|
try
|
|
{
|
|
var reportPath = Path.Combine(directoryPath, "report.md");
|
|
await File.WriteAllTextAsync(reportPath, "Report");
|
|
await File.WriteAllTextAsync(Path.Combine(outputPath, "report_result.md"), "Result");
|
|
await File.WriteAllTextAsync(Path.Combine(binPath, "notes.md"), "Build output");
|
|
var selection = new FolderFileSelection(["*.md"], FileExclusions.Parse("bin/"), true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
|
|
|
var candidates = selection.EnumerateCandidates(
|
|
directoryPath,
|
|
includeDirectory: folderPath => !string.Equals(folderPath, outputPath, StringComparison.OrdinalIgnoreCase));
|
|
|
|
Assert.That(candidates, Is.EqualTo(new[] { reportPath }), "Like the batch processing assistant leaves out its output folder.");
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(directoryPath, true);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public void ExcludedFoldersAreSummarizedByName()
|
|
{
|
|
var directoryPath = Path.Combine(Path.GetTempPath(), "test-data");
|
|
var result = new FileEnumerationResult();
|
|
|
|
result.AddExcludedFolder(Path.Combine(directoryPath, "bin"));
|
|
result.AddExcludedFolder(Path.Combine(directoryPath, "src", "Bin"));
|
|
result.AddExcludedFolder(Path.Combine(directoryPath, "node_modules"));
|
|
|
|
using (Assert.EnterMultipleScope())
|
|
{
|
|
Assert.That(result.ExcludedFolderCount, Is.EqualTo(3));
|
|
Assert.That(result.ExcludedFolders, Is.EquivalentTo(new[]
|
|
{
|
|
new DataSourceEmbeddingExcludedFolderGroup("bin", 2),
|
|
new DataSourceEmbeddingExcludedFolderGroup("node_modules", 1),
|
|
}));
|
|
}
|
|
}
|
|
} |