using AIStudio.Assistants.ModelComparison; using Microsoft.Extensions.Logging.Abstractions; namespace AIStudio.Tests.Assistants.ModelComparison; /// /// Checks how the Model Comparison assistant stores its presets as files below the data directory. /// [TestFixture] public sealed class ModelComparisonStorePresetTests { private string dataDirectory = string.Empty; private string PresetsDirectory => Path.Combine(this.dataDirectory, ModelComparisonStore.ROOT_DIRECTORY_NAME, "presets"); [SetUp] public void CreateDataDirectory() { this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-model-comparison-{Guid.NewGuid():N}"); Directory.CreateDirectory(this.dataDirectory); } [TearDown] public void DeleteDataDirectory() { if (Directory.Exists(this.dataDirectory)) Directory.Delete(this.dataDirectory, true); } [Test] public async Task ASavedPresetIsListedWithAllItsFields() { var store = this.CreateStore(); var preset = Preset(" Summaries "); await store.SavePresetAsync(preset); var presets = await store.ListPresetsAsync(); using (Assert.EnterMultipleScope()) { Assert.That(presets, Has.Count.EqualTo(1)); Assert.That(presets[0], Is.EqualTo(preset with { Name = "Summaries" }), "The name is trimmed, everything else stays as it was."); Assert.That(File.Exists(Path.Combine(this.PresetsDirectory, $"{preset.Id:D}.json")), Is.True); } } [Test] public async Task ANewStoreFindsThePresetsOfTheLastOne() { var preset = Preset("Summaries"); await this.CreateStore().SavePresetAsync(preset); var presets = await this.CreateStore().ListPresetsAsync(); Assert.That(presets.Select(listed => listed.Id), Is.EqualTo(new[] { preset.Id })); } [Test] public async Task SavingAgainReplacesThePreset() { var store = this.CreateStore(); var preset = Preset("Summaries"); await store.SavePresetAsync(preset); await store.SavePresetAsync(preset with { Name = "Translations", RunCount = 10 }); var presets = await store.ListPresetsAsync(); using (Assert.EnterMultipleScope()) { Assert.That(presets, Has.Count.EqualTo(1)); Assert.That(presets[0].Name, Is.EqualTo("Translations")); Assert.That(presets[0].RunCount, Is.EqualTo(10)); } } [Test] public async Task ThePresetsAreSortedByName() { var store = this.CreateStore(); await store.SavePresetAsync(Preset("Translations")); await store.SavePresetAsync(Preset("abstracts")); await store.SavePresetAsync(Preset("Summaries")); var presets = await store.ListPresetsAsync(); Assert.That(presets.Select(preset => preset.Name), Is.EqualTo(new[] { "abstracts", "Summaries", "Translations" })); } [Test] public async Task ADeletedPresetIsGone() { var store = this.CreateStore(); var kept = Preset("Kept"); var deleted = Preset("Deleted"); await store.SavePresetAsync(kept); await store.SavePresetAsync(deleted); await store.DeletePresetAsync(deleted.Id); var presets = await store.ListPresetsAsync(); using (Assert.EnterMultipleScope()) { Assert.That(presets.Select(preset => preset.Id), Is.EqualTo(new[] { kept.Id })); Assert.That(File.Exists(Path.Combine(this.PresetsDirectory, $"{deleted.Id:D}.json")), Is.False); } } [Test] public async Task APresetThisVersionCannotReadIsLeftOutButKept() { var store = this.CreateStore(); var readable = Preset("Readable"); await store.SavePresetAsync(readable); // A newer version may write a field this one does not know: var unreadablePath = Path.Combine(this.PresetsDirectory, $"{Guid.NewGuid():D}.json"); await File.WriteAllTextAsync(unreadablePath, """{ "name": "From a newer version", "fieldOfANewerVersion": true }"""); var presets = await store.ListPresetsAsync(); using (Assert.EnterMultipleScope()) { Assert.That(presets.Select(preset => preset.Id), Is.EqualTo(new[] { readable.Id })); Assert.That(File.Exists(unreadablePath), Is.True); } } [Test] public async Task TheFileNameDecidesTheIdOfAPreset() { var store = this.CreateStore(); var preset = Preset("Summaries"); await store.SavePresetAsync(preset); // A copied file keeps the ID of its original inside; its own name is what a save or delete has to reach: var copyId = Guid.NewGuid(); File.Copy(Path.Combine(this.PresetsDirectory, $"{preset.Id:D}.json"), Path.Combine(this.PresetsDirectory, $"{copyId:D}.json")); var presets = await store.ListPresetsAsync(); Assert.That(presets.Select(listed => listed.Id), Is.EquivalentTo(new[] { preset.Id, copyId })); } [Test] public async Task APresetWithoutANameIsRefused() { var store = this.CreateStore(); await Assert.ThrowsAsync(() => store.SavePresetAsync(Preset(" "))); } private ModelComparisonStore CreateStore() => new(NullLogger.Instance, new ModelComparisonStorageOptions { DataDirectory = this.dataDirectory }); private static ModelComparisonPreset Preset(string name) => new() { Id = Guid.NewGuid(), Name = name, FirstProviderId = "first", SecondProviderId = "second", JudgeEnabled = true, JudgeProviderId = "judge", JudgeInstructions = "Keep every number.", Context = "The document, with umlauts: Größe.", Question = "Summarize it.", RunCount = 5, }; }