mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-04 18:07:21 +00:00
Improved the dialogs for local data sources and the Batch Processing assistant (#1096)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and 
GitHub committed
1 parent
149395c1a0
commit
f7e6ceb915
45 files changed
+2514
-361
No files matched your search
@@ -40,6 +40,18 @@ public sealed class EmbeddingSignatureTests
|
||||
"Patterns change which files belong to the source, not how retained files are embedded; the indexing run removes and adds only the affected files.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ChangingDirectoryExclusionsKeepsTheStoredEmbeddingsOfRetainedFiles()
|
||||
{
|
||||
var allFiles = DataSource(ConfidenceLevel.LOW);
|
||||
var filteredFiles = allFiles with { ExcludePatterns = "*.mp3;draft-*" };
|
||||
|
||||
Assert.That(
|
||||
Signature(filteredFiles),
|
||||
Is.EqualTo(Signature(allFiles)),
|
||||
"Exclusions change which files belong to the source, not how retained files are embedded.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ChangingDirectoryRecursionKeepsTheStoredEmbeddingsOfRetainedFiles()
|
||||
{
|
||||
@@ -244,4 +256,4 @@ public sealed class EmbeddingSignatureTests
|
||||
|
||||
public int ChunkOverlapTokenLength { get; init; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,6 @@ using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.Validation;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
@@ -22,7 +21,7 @@ public sealed class FileExclusionsTests
|
||||
[TestCase(FileExclusions.DEFAULT_PATTERNS)]
|
||||
public void ValidExclusionListsAreAccepted(string patterns)
|
||||
{
|
||||
Assert.That(new DataSourceValidation().ValidateExcludePatterns(patterns), Is.Null);
|
||||
Assert.That(FileExclusions.Validate(patterns), Is.Null);
|
||||
}
|
||||
|
||||
[TestCase("a/b/")]
|
||||
@@ -33,7 +32,72 @@ public sealed class FileExclusionsTests
|
||||
[TestCase("**/bin/")]
|
||||
public void InvalidExclusionListsAreRejected(string patterns)
|
||||
{
|
||||
Assert.That(new DataSourceValidation().ValidateExcludePatterns(patterns), Is.Not.Null);
|
||||
Assert.That(FileExclusions.Validate(patterns), Is.Not.Null);
|
||||
}
|
||||
|
||||
[TestCase(null, true)]
|
||||
[TestCase("", true)]
|
||||
[TestCase(" ; \r\n", true)]
|
||||
[TestCase("build/;cmake-build-*/", true)]
|
||||
[TestCase(".env/;archive.v1/;*.vmwarevm/", true)]
|
||||
[TestCase("*.pdf", false)]
|
||||
[TestCase("Dockerfile", false)]
|
||||
[TestCase("node_modules", false)]
|
||||
[TestCase("build/;*.txt", false)]
|
||||
public void FolderFieldRequiresExplicitFolderRules(string? patterns, bool valid)
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileExclusions.IsValidFolders(patterns), Is.EqualTo(valid));
|
||||
Assert.That(FileExclusions.ValidateFolders(patterns) is null, Is.EqualTo(valid));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(null, true)]
|
||||
[TestCase("", true)]
|
||||
[TestCase(" ; \r\n", true)]
|
||||
[TestCase("*.pdf;draft-*;report-?.md", true)]
|
||||
[TestCase("Dockerfile;Caddyfile;.gitignore", true)]
|
||||
[TestCase("build", true)]
|
||||
[TestCase("build/", false)]
|
||||
[TestCase("*.tmp;obj/", false)]
|
||||
public void FileFieldRejectsFolderRulesButAcceptsExtensionlessFiles(string? patterns, bool valid)
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileExclusions.IsValidFiles(patterns), Is.EqualTo(valid));
|
||||
Assert.That(FileExclusions.ValidateFiles(patterns) is null, Is.EqualTo(valid));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(":text:")]
|
||||
[TestCase(":documents:")]
|
||||
[TestCase("src/build/")]
|
||||
[TestCase("**/*.tmp")]
|
||||
[TestCase("/")]
|
||||
[TestCase("./")]
|
||||
public void BothFieldValidatorsReuseTheExistingSyntaxErrors(string patterns)
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileExclusions.IsValidFolders(patterns), Is.False);
|
||||
Assert.That(FileExclusions.IsValidFiles(patterns), Is.False);
|
||||
Assert.That(FileExclusions.ValidateFolders(patterns), Is.EqualTo(FileExclusions.Validate(patterns)));
|
||||
Assert.That(FileExclusions.ValidateFiles(patterns), Is.EqualTo(FileExclusions.Validate(patterns)));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void StoredListsStillAcceptBothKindsOfRulesTogether()
|
||||
{
|
||||
const string PATTERNS = "bin/;*.pdf;Dockerfile;.git/";
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileExclusions.IsValid(PATTERNS), Is.True);
|
||||
Assert.That(FileExclusions.Validate(PATTERNS), Is.Null);
|
||||
Assert.That(FileExclusions.ExcludesFolder("bin", FileExclusions.Parse(PATTERNS)), Is.True);
|
||||
Assert.That(FileExclusions.ExcludesFile("report.pdf", FileExclusions.Parse(PATTERNS)), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
@@ -42,6 +106,18 @@ public sealed class FileExclusionsTests
|
||||
Assert.That(FileExclusions.Parse(" bin/ ;obj/\r\n\r\n*.min.js\nBIN/ "), Is.EqualTo(new[] { "bin/", "obj/", "*.min.js" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GroupingSplitsFoldersFromFilesAndDropsDuplicates()
|
||||
{
|
||||
var groups = FileExclusions.Group(" bin/ ;*.min.js\r\nobj/;BIN/;;\n*.MIN.js");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(groups.Folders, Is.EqualTo("bin/; obj/"));
|
||||
Assert.That(groups.Files, Is.EqualTo("*.min.js"));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SettingsFromBeforeTheListGetTheDefault()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
using AIStudio.Tools;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
public sealed class FilePatternExclusionTests
|
||||
{
|
||||
[TestCase("*", "*.mp3;*.webm", "recording.WEBM", false)]
|
||||
[TestCase("*", "*.mp3;*.webm", "report.pdf", true)]
|
||||
[TestCase("*.txt;*.pdf", "draft-*", "draft-report.pdf", false)]
|
||||
[TestCase("*.txt;*.pdf", "draft-*", "report.pdf", true)]
|
||||
[TestCase("*.txt", "*.mp3", "report.pdf", false)]
|
||||
[TestCase("*", "Dockerfile;Caddyfile", "Dockerfile", false)]
|
||||
[TestCase("*", ".env", ".env", false)]
|
||||
[TestCase("*", "*", "report.pdf", false)]
|
||||
[TestCase("*", "", "report.pdf", true)]
|
||||
[TestCase("*", "*-draft-*", "report-draft-final.md", false)]
|
||||
[TestCase("*", "draft-*", "draft-", false)]
|
||||
[TestCase("*", "report-?.md", "report-a.md", false)]
|
||||
[TestCase("*", "report-?.md", "report-ab.md", true)]
|
||||
public void ExclusionsOverrideIncludesButNeverSelectOtherFiles(string include, string exclude, string fileName, bool expected)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), "nested", fileName);
|
||||
var selection = new FolderFileSelection(
|
||||
FileNamePatterns.Parse(include),
|
||||
FileExclusions.Parse(exclude),
|
||||
false,
|
||||
false,
|
||||
FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
||||
|
||||
Assert.That(selection.Selects(path), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("build", true)]
|
||||
[TestCase("app-build", true)]
|
||||
[TestCase("my-build", true)]
|
||||
[TestCase("BUILD", true)]
|
||||
[TestCase("builder", false)]
|
||||
[TestCase("build-output", false)]
|
||||
public void TheFolderExampleMatchesNamesEndingInBuildButNotFiles(string name, bool expected)
|
||||
{
|
||||
var patterns = FileExclusions.Parse("*build/");
|
||||
var path = Path.Combine(Path.GetTempPath(), "test-data", name);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileExclusions.ExcludesFolder(path, patterns), Is.EqualTo(expected));
|
||||
Assert.That(FileExclusions.ExcludesFile(path, patterns), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false, "", true, false)]
|
||||
[TestCase(true, "", true, true)]
|
||||
[TestCase(false, ".gitignore", false, false)]
|
||||
[TestCase(true, ".gitignore", false, false)]
|
||||
[TestCase(false, ".hidden/", true, false)]
|
||||
[TestCase(true, ".hidden/", true, false)]
|
||||
public async Task ExplicitDotPatternsBypassOnlyTheHiddenFileRuleNotExclusionsOrHiddenFolders(
|
||||
bool includeHidden, string exclusions, bool selectsRootFile, bool selectsNestedFile)
|
||||
{
|
||||
var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-dot-pattern-priority-{Guid.NewGuid():N}");
|
||||
var hiddenDirectory = Path.Combine(directoryPath, ".hidden");
|
||||
var rootFile = Path.Combine(directoryPath, ".gitignore");
|
||||
var nestedFile = Path.Combine(hiddenDirectory, ".gitignore");
|
||||
Directory.CreateDirectory(hiddenDirectory);
|
||||
|
||||
try
|
||||
{
|
||||
await File.WriteAllTextAsync(rootFile, "bin/");
|
||||
await File.WriteAllTextAsync(nestedFile, "obj/");
|
||||
var selection = new FolderFileSelection(
|
||||
[".gitignore"], FileExclusions.Parse(exclusions), true, includeHidden,
|
||||
FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
||||
var selectedFiles = selection.EnumerateCandidates(directoryPath).Where(selection.Selects).ToArray();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(selectedFiles.Contains(rootFile), Is.EqualTo(selectsRootFile));
|
||||
Assert.That(selectedFiles.Contains(nestedFile), Is.EqualTo(selectsNestedFile));
|
||||
Assert.That(selection.LeavesOut(directoryPath, rootFile), Is.EqualTo(!selectsRootFile));
|
||||
Assert.That(selection.LeavesOut(directoryPath, nestedFile), Is.EqualTo(!selectsNestedFile));
|
||||
});
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directoryPath, true);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EnvironmentDefaultsExcludeVisibleEnvironmentFoldersRatherThanDotEnvFiles()
|
||||
{
|
||||
var defaults = FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileExclusions.ExcludesFolder("env", defaults), Is.True);
|
||||
Assert.That(FileExclusions.ExcludesFolder("venv", defaults), Is.True);
|
||||
Assert.That(FileExclusions.ExcludesFile(".env", defaults), Is.False);
|
||||
Assert.That(FileExclusions.ExcludesFolder(".env", defaults), Is.False,
|
||||
"Hidden folders are handled by the hidden-file switch, not by the default exclusion list.");
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(null)]
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
public void ExclusionsMayBeEmptyWhileIncludesAreRequired(string? patterns)
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileExclusions.Validate(patterns), Is.Null);
|
||||
Assert.That(FileExclusions.IsValid(patterns), Is.True);
|
||||
Assert.That(FileNamePatterns.IsValid(patterns), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase("**/*.txt")]
|
||||
[TestCase("folder/*.txt")]
|
||||
public void OptionalExclusionsStillRejectMalformedRules(string patterns)
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileExclusions.Validate(patterns), Is.Not.Null);
|
||||
Assert.That(FileExclusions.IsValid(patterns), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TextSelectorsCanBeCombinedWithFilenameExclusions()
|
||||
{
|
||||
var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-exclusions-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(directoryPath);
|
||||
try
|
||||
{
|
||||
var terraform = Path.Combine(directoryPath, "main.tf");
|
||||
var caddy = Path.Combine(directoryPath, "Caddyfile");
|
||||
await File.WriteAllTextAsync(terraform, "resource example {}");
|
||||
await File.WriteAllTextAsync(caddy, "localhost { respond hello }");
|
||||
var selection = new FolderFileSelection(
|
||||
[":text:"], ["*.tf"], false, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(selection.Selects(terraform), Is.False);
|
||||
Assert.That(selection.Selects(caddy), Is.True);
|
||||
});
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directoryPath, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -271,6 +271,32 @@ public sealed class FileSourceIndexerWatcherTests
|
||||
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReorderingEquivalentExclusionsDoesNotRecreateTheWatcher()
|
||||
{
|
||||
var before = DefaultConfiguration(Path.GetTempPath()) with
|
||||
{
|
||||
ExcludePatterns = FileExclusions.Parse("*.mp3;draft-*"),
|
||||
};
|
||||
var after = before with { ExcludePatterns = FileExclusions.Parse("DRAFT-*;*.MP3;*.mp3") };
|
||||
|
||||
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RenamingAnIncludedFileIntoAnExcludedNameStillRefreshesTheIndex()
|
||||
{
|
||||
var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-excluded-rename-{Guid.NewGuid():N}");
|
||||
var configuration = DefaultConfiguration(rootPath) with
|
||||
{
|
||||
FilePatterns = FileNamePatterns.Parse("*.md"),
|
||||
ExcludePatterns = FileExclusions.Parse("draft-*"),
|
||||
};
|
||||
var renamed = new RenamedEventArgs(WatcherChangeTypes.Renamed, rootPath, "draft-report.md", "report.md");
|
||||
|
||||
Assert.That(FileSourceIndexer.IsRelevantWatcherEvent(configuration, renamed), Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RelocatingToADirectoryWithDifferentCasingUsesTheLocalPathRules()
|
||||
{
|
||||
|
||||
@@ -39,7 +39,8 @@ public sealed class LocalDirectoryFilePatternsTests
|
||||
[TestCase(null)]
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
[TestCase("*.txt;")]
|
||||
[TestCase(";")]
|
||||
[TestCase(" ; ; ")]
|
||||
[TestCase("folder/*.txt")]
|
||||
public void InvalidStoredPatternsFallBackWithoutRemovingExistingDocuments(string? storedPatterns)
|
||||
{
|
||||
@@ -48,12 +49,60 @@ public sealed class LocalDirectoryFilePatternsTests
|
||||
Assert.That(dataSource.EffectiveFilePatterns, Is.EqualTo(DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void BatchExclusionsSurviveSettingsSerialization()
|
||||
{
|
||||
var written = new DataBatchProcessing { ExcludePatterns = "*.mp3;*.webm;draft-*" };
|
||||
var json = JsonSerializer.Serialize(written, SettingsManager.JSON_OPTIONS);
|
||||
var read = JsonSerializer.Deserialize<DataBatchProcessing>(json, SettingsManager.JSON_OPTIONS);
|
||||
|
||||
Assert.That(read!.ExcludePatterns, Is.EqualTo(written.ExcludePatterns));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PatternsAreSplitAndTrimmed()
|
||||
{
|
||||
Assert.That(FileNamePatterns.Parse(" *.txt ; *.tf ; .gitignore "), Is.EqualTo(new[] { "*.txt", "*.tf", ".gitignore" }));
|
||||
}
|
||||
|
||||
[TestCase("*.txt;")]
|
||||
[TestCase(";*.txt")]
|
||||
[TestCase(";; *.txt ; ; ")]
|
||||
public void EmptyEntriesAreIgnoredWithoutBroadeningTheSelection(string patterns)
|
||||
{
|
||||
var parsed = FileNamePatterns.Parse(patterns);
|
||||
var dataSource = new DataSourceLocalDirectory { FilePatterns = patterns };
|
||||
var selection = new FolderFileSelection(parsed, [], false, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
||||
var canonical = new FolderFileSelection(["*.txt"], [], false, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileNamePatterns.IsValid(patterns), Is.True);
|
||||
Assert.That(FileNamePatterns.Validate(patterns), Is.Null);
|
||||
Assert.That(parsed, Is.EqualTo(new[] { "*.txt" }));
|
||||
Assert.That(dataSource.EffectiveFilePatterns, Is.EqualTo(patterns), "Empty entries must not trigger the default document selection.");
|
||||
Assert.That(selection.Selects("report.txt"), Is.True);
|
||||
Assert.That(selection.Selects("report.md"), Is.False);
|
||||
Assert.That(selection.Selects("recording.wav"), Is.False);
|
||||
Assert.That(selection.Describe(), Is.EqualTo(canonical.Describe()), "Empty entries must not change the indexing selection signature.");
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(null)]
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
[TestCase(";")]
|
||||
[TestCase(" ; ;; ")]
|
||||
public void IncludesStillRequireAtLeastOneNonEmptyPattern(string? patterns)
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(FileNamePatterns.Parse(patterns), Is.Empty);
|
||||
Assert.That(FileNamePatterns.IsValid(patterns), Is.False);
|
||||
Assert.That(FileNamePatterns.Validate(patterns), Is.Not.Null);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NormalizingPatternsRemovesCaseInsensitiveDuplicates()
|
||||
{
|
||||
@@ -75,6 +124,7 @@ public sealed class LocalDirectoryFilePatternsTests
|
||||
{
|
||||
Type = DataSourceType.LOCAL_DIRECTORY,
|
||||
FilePatterns = "*.txt;*.tf;.gitignore",
|
||||
ExcludePatterns = "secret-*;*.backup",
|
||||
IncludeSubdirectories = false,
|
||||
IncludeHiddenFilesAndFolders = true,
|
||||
};
|
||||
@@ -85,8 +135,10 @@ public sealed class LocalDirectoryFilePatternsTests
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(json, Does.Not.Contain(nameof(DataSourceLocalDirectory.EffectiveFilePatterns)));
|
||||
Assert.That(json, Does.Not.Contain(nameof(DataSourceLocalDirectory.EffectiveExcludePatterns)));
|
||||
Assert.That(read, Is.TypeOf<DataSourceLocalDirectory>());
|
||||
Assert.That(((DataSourceLocalDirectory)read!).FilePatterns, Is.EqualTo("*.txt;*.tf;.gitignore"));
|
||||
Assert.That(((DataSourceLocalDirectory)read).ExcludePatterns, Is.EqualTo("secret-*;*.backup"));
|
||||
Assert.That(((DataSourceLocalDirectory)read).IncludeSubdirectories, Is.False);
|
||||
Assert.That(((DataSourceLocalDirectory)read).IncludeHiddenFilesAndFolders, Is.True);
|
||||
});
|
||||
@@ -419,17 +471,22 @@ public sealed class LocalDirectoryFilePatternsTests
|
||||
[TestCase(":documents:")]
|
||||
[TestCase(":DOCUMENTS:;*.tf")]
|
||||
[TestCase("report-??.md")]
|
||||
[TestCase("*.txt;")]
|
||||
[TestCase(";*.txt;; *.tf ; ;")]
|
||||
[TestCase(":documents:;;")]
|
||||
public void DataSourceValidationAcceptsSimplePatterns(string patterns)
|
||||
{
|
||||
Assert.That(new DataSourceValidation().ValidateFilePatterns(patterns), Is.Null);
|
||||
}
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase("*.txt;")]
|
||||
[TestCase(" ; ;; ")]
|
||||
[TestCase("**/*.txt")]
|
||||
[TestCase("folder/*.txt")]
|
||||
[TestCase(":document:")]
|
||||
[TestCase("*.txt;:docs:")]
|
||||
[TestCase(";*.txt;; :docs: ;")]
|
||||
[TestCase("*.txt;;folder/*.md;")]
|
||||
public void DataSourceValidationRejectsInvalidPatterns(string patterns)
|
||||
{
|
||||
Assert.That(new DataSourceValidation().ValidateFilePatterns(patterns), Is.Not.Null);
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Globalization;
|
||||
|
||||
using AIStudio.Tools;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
public sealed class LongExtensionsTests
|
||||
{
|
||||
[TestCase(0L, "0 B")]
|
||||
[TestCase(1_023L, "1023 B")]
|
||||
[TestCase(1_536L, "1.5 kB")]
|
||||
[TestCase(1_992_294L, "1.9 MB")]
|
||||
[TestCase(1_073_741_824L, "1 GB")]
|
||||
[TestCase(1_572_864_000L, "1.46 GB")]
|
||||
public void FileSizesKeepTheirFraction(long sizeBytes, string expected)
|
||||
{
|
||||
Assert.That(sizeBytes.FileSize(CultureInfo.InvariantCulture), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FileSizesUseTheDecimalSeparatorOfTheCulture()
|
||||
{
|
||||
Assert.That(1_992_294L.FileSize(CultureInfo.GetCultureInfo("de-DE")), Is.EqualTo("1,9 MB"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.Services.Indexing;
|
||||
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Checks combinations of media, transcripts, and ordinary Markdown against the current folder
|
||||
/// selection rules. Transcript reuse exercises the shared implementation used by RAG and batch.
|
||||
/// </summary>
|
||||
public sealed class MediaFileSelectionTests
|
||||
{
|
||||
private const string TRANSCRIPT_TEXT = "# Transcript\nSpoken content from the recording.";
|
||||
|
||||
private string rootPath = string.Empty;
|
||||
private string userFolder = string.Empty;
|
||||
|
||||
[SetUp]
|
||||
public void CreateDirectory()
|
||||
{
|
||||
this.rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-media-selection-{Guid.NewGuid():N}");
|
||||
this.userFolder = Path.Combine(this.rootPath, "user");
|
||||
Directory.CreateDirectory(this.userFolder);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void DeleteDirectory() => Directory.Delete(this.rootPath, true);
|
||||
|
||||
[TestCase("*.wav", "*.md")]
|
||||
[TestCase("*.wav;*.md", "*.md")]
|
||||
[TestCase("*", "*.md")]
|
||||
public async Task ExcludingMarkdownStillSelectsTheMedia(string includes, string excludes)
|
||||
{
|
||||
var mediaPath = await this.CreateMediaAndDocumentsAsync();
|
||||
|
||||
Assert.That(this.Discover(includes, excludes), Is.EqualTo(new[] { mediaPath }));
|
||||
}
|
||||
|
||||
[TestCase("*.md")]
|
||||
[TestCase("*.wav;*.md")]
|
||||
[TestCase("*")]
|
||||
public async Task ExcludingMediaKeepsOrdinaryMarkdownAndTheIndependentTranscript(string includes)
|
||||
{
|
||||
var mediaPath = await this.CreateMediaAndDocumentsAsync();
|
||||
|
||||
Assert.That(this.Discover(includes, "*.wav"), Is.EquivalentTo(new[]
|
||||
{
|
||||
MediaSidecarTranscripts.GetFilePath(mediaPath),
|
||||
Path.Combine(this.userFolder, "notes.md"),
|
||||
Path.Combine(this.userFolder, "readme.md"),
|
||||
}));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SelectingMediaAndMarkdownDoesNotProcessTheTranscriptTwice()
|
||||
{
|
||||
var mediaPath = await this.CreateMediaAndDocumentsAsync();
|
||||
|
||||
Assert.That(this.Discover("*.wav;*.md", string.Empty), Is.EquivalentTo(new[]
|
||||
{
|
||||
mediaPath,
|
||||
Path.Combine(this.userFolder, "notes.md"),
|
||||
Path.Combine(this.userFolder, "readme.md"),
|
||||
}));
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task ExistingTranscriptsAreReusedWithoutAProviderOrARewrite(bool dataSourceTranscript)
|
||||
{
|
||||
var mediaPath = await this.CreateMediaAndDocumentsAsync();
|
||||
IMediaTranscriptLocation location = dataSourceTranscript
|
||||
? this.TranscriptStore()
|
||||
: MediaSidecarTranscripts.INSTANCE;
|
||||
var slot = await location.ResolveAsync(mediaPath, CancellationToken.None);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(slot.Path)!);
|
||||
await File.WriteAllTextAsync(slot.Path, TRANSCRIPT_TEXT);
|
||||
var originalWriteTime = File.GetLastWriteTimeUtc(slot.Path);
|
||||
var calls = 0;
|
||||
Task<MediaTranscriptionResult> Transcribe(string _, TranscriptionProvider __, CancellationToken ___)
|
||||
{
|
||||
calls++;
|
||||
throw new InvalidOperationException("An existing transcript must not ask a provider.");
|
||||
}
|
||||
|
||||
for (var attempt = 0; attempt < 2; attempt++)
|
||||
{
|
||||
var result = await PersistentMediaTranscript.ReadOrCreateAsync(
|
||||
location, mediaPath, null, Transcribe, NullLogger.Instance, CancellationToken.None);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY));
|
||||
Assert.That(result.Reused, Is.True);
|
||||
Assert.That(File.ReadAllText(result.TranscriptPath), Is.EqualTo(TRANSCRIPT_TEXT));
|
||||
Assert.That(File.GetLastWriteTimeUtc(slot.Path), Is.EqualTo(originalWriteTime));
|
||||
Assert.That(calls, Is.Zero);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ChangedMediaDoesNotReuseAnOldDataSourceTranscriptEvenWithTheSameSizeAndWriteTime()
|
||||
{
|
||||
var mediaPath = await this.CreateMediaAndDocumentsAsync();
|
||||
var store = this.TranscriptStore();
|
||||
var originalSlot = await store.ResolveAsync(mediaPath, CancellationToken.None);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(originalSlot.Path)!);
|
||||
await File.WriteAllTextAsync(originalSlot.Path, TRANSCRIPT_TEXT);
|
||||
var originalWriteTime = File.GetLastWriteTimeUtc(mediaPath);
|
||||
|
||||
await File.WriteAllBytesAsync(mediaPath, [4, 5, 6]);
|
||||
File.SetLastWriteTimeUtc(mediaPath, originalWriteTime);
|
||||
var changedSlot = await store.ResolveAsync(mediaPath, CancellationToken.None);
|
||||
var result = await PersistentMediaTranscript.ReadOrCreateAsync(
|
||||
store, mediaPath, null,
|
||||
(_, _, _) => throw new InvalidOperationException("No provider was configured."),
|
||||
NullLogger.Instance, CancellationToken.None);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(changedSlot.Key, Is.Not.EqualTo(originalSlot.Key));
|
||||
Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.NO_PROVIDER));
|
||||
Assert.That(File.ReadAllText(originalSlot.Path), Is.EqualTo(TRANSCRIPT_TEXT));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase("*.wav", "*.md", false, false)]
|
||||
[TestCase("*.md", "*.wav", true, true)]
|
||||
[TestCase("*.wav;*.md", "", true, false)]
|
||||
public async Task WatcherUsesTheSelectionOfTheMediaWhenDecidingAboutItsSidecar(
|
||||
string includes, string excludes, bool markdownChangesMatter, bool transcriptChangesMatter)
|
||||
{
|
||||
var mediaPath = await this.CreateMediaAndDocumentsAsync();
|
||||
var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(
|
||||
this.userFolder,
|
||||
false,
|
||||
FilePatterns: FileNamePatterns.Parse(includes),
|
||||
ExcludePatterns: FileExclusions.Parse(excludes));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(
|
||||
FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(this.userFolder, "notes.md"), WatcherChangeTypes.Changed),
|
||||
Is.EqualTo(markdownChangesMatter));
|
||||
Assert.That(
|
||||
FileSourceIndexer.IsRelevantWatcherPath(configuration, MediaSidecarTranscripts.GetFilePath(mediaPath), WatcherChangeTypes.Changed),
|
||||
Is.EqualTo(transcriptChangesMatter));
|
||||
});
|
||||
}
|
||||
|
||||
private async Task<string> CreateMediaAndDocumentsAsync()
|
||||
{
|
||||
var mediaPath = Path.Combine(this.userFolder, "test123.wav");
|
||||
// These bytes cannot be decoded: reuse must find the transcript before asking a provider.
|
||||
await File.WriteAllBytesAsync(mediaPath, [1, 2, 3]);
|
||||
await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(mediaPath), TRANSCRIPT_TEXT);
|
||||
await File.WriteAllTextAsync(Path.Combine(this.userFolder, "notes.md"), "# Ordinary notes");
|
||||
await File.WriteAllTextAsync(Path.Combine(this.userFolder, "readme.md"), "# Ordinary readme");
|
||||
return mediaPath;
|
||||
}
|
||||
|
||||
private IReadOnlyList<string> Discover(string includes, string excludes)
|
||||
{
|
||||
using var indexer = new FileSourceIndexer(null!, null!, null!, null!, NullLogger.Instance);
|
||||
var result = indexer.GetInputFiles(new DataSourceLocalDirectory
|
||||
{
|
||||
Type = DataSourceType.LOCAL_DIRECTORY,
|
||||
Path = this.userFolder,
|
||||
FilePatterns = includes,
|
||||
ExcludePatterns = excludes,
|
||||
IncludeSubdirectories = false,
|
||||
}, new DataSourceEmbeddingManifest(),
|
||||
MediaTranscriptionAvailability.Allowed(new TranscriptionProvider { Id = "test-transcription-provider" }));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(result.Failures, Is.Empty);
|
||||
Assert.That(result.IgnoredFiles, Is.Empty);
|
||||
});
|
||||
|
||||
return result.Files.Select(file => file.FullName).ToArray();
|
||||
}
|
||||
|
||||
private DataSourceTranscriptStore TranscriptStore() => new(
|
||||
Path.Combine(this.rootPath, "data"), "d401ce54-9a42-4e78-85de-2402e07f7f7e");
|
||||
}
|
||||
Reference in new issue
Block a user