mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-10 20:27:21 +00:00
Added material to the BeyondFacts Lab (#1180)
This commit is contained in:
1 parent
22d94eb8fd
commit
9f2280bed7
58 files changed
+3932
-120
No files matched your search
@@ -0,0 +1,406 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the BeyondFacts Lab stores the materials of a preparation.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BeyondFactsLabStoreMaterialTests
|
||||
{
|
||||
private const string HASH = "0f343b0931126a20f133d67c2b018a3b5f1d1f5c1f6a8e0a3d5a5a3e9c6a1b2c";
|
||||
|
||||
private const string OTHER_HASH = "1f343b0931126a20f133d67c2b018a3b5f1d1f5c1f6a8e0a3d5a5a3e9c6a1b2c";
|
||||
|
||||
private string dataDirectory = string.Empty;
|
||||
|
||||
private string RootDirectory => Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME);
|
||||
|
||||
[SetUp]
|
||||
public void CreateDataDirectory()
|
||||
{
|
||||
this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-materials-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.dataDirectory);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void DeleteDataDirectory()
|
||||
{
|
||||
if (Directory.Exists(this.dataDirectory))
|
||||
Directory.Delete(this.dataDirectory, true);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMaterialIsStoredWithItsText()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material("Einladung.docx", HASH), "Podium am 10. Dezember");
|
||||
var changed = await store.LoadAsync(created.ProjectId);
|
||||
var materials = await this.CreateStore().ListMaterialsAsync(created.ProjectId);
|
||||
var text = await store.ReadMaterialTextAsync(created.ProjectId, addition.Material.MaterialId, 1_000);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(addition.IsDuplicate, Is.False);
|
||||
Assert.That(materials, Has.Count.EqualTo(1));
|
||||
Assert.That(materials[0].Material!.Name, Is.EqualTo("Einladung.docx"));
|
||||
Assert.That(materials[0].Material!.TextLength, Is.EqualTo("Podium am 10. Dezember".Length));
|
||||
Assert.That(materials[0].Material!.SourceBytes, Is.EqualTo(2_048));
|
||||
Assert.That(text, Is.EqualTo("Podium am 10. Dezember"));
|
||||
Assert.That(changed!.ModifiedAtUtc, Is.GreaterThan(created.ModifiedAtUtc), "A material is work on the preparation.");
|
||||
Assert.That(File.Exists(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".json")), Is.True);
|
||||
Assert.That(File.Exists(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".md")), Is.True);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImageIsStoredWithItsCopy()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
byte[] copy = [0x89, 0x50, 0x4E, 0x47];
|
||||
|
||||
var addition = await store.AddImageMaterialAsync(created.ProjectId, ImageMaterial("Poster.tiff", HASH), copy);
|
||||
var materials = await this.CreateStore().ListMaterialsAsync(created.ProjectId);
|
||||
var path = store.MaterialImagePath(created.ProjectId, materials[0].Material!);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(addition.IsDuplicate, Is.False);
|
||||
Assert.That(materials[0].Material!.Kind, Is.EqualTo(BflMaterialKind.IMAGE));
|
||||
Assert.That(materials[0].Material!.Image!.Width, Is.EqualTo(2_560));
|
||||
Assert.That(materials[0].Material!.TextLength, Is.Zero);
|
||||
Assert.That(path, Is.EqualTo(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".png")));
|
||||
Assert.That(await File.ReadAllBytesAsync(path), Is.EqualTo(copy));
|
||||
Assert.That(File.Exists(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".md")), Is.False, "An image has no text.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImageWithoutItsCopyCannotBeOpened()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddImageMaterialAsync(created.ProjectId, ImageMaterial("Poster.png", HASH), new byte[] { 1 });
|
||||
|
||||
File.Delete(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".png"));
|
||||
var materials = await store.ListMaterialsAsync(created.ProjectId);
|
||||
|
||||
Assert.That(materials.Single().IsAvailable, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DeletingAnImageDeletesItsCopy()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddImageMaterialAsync(created.ProjectId, ImageMaterial("Poster.png", HASH), new byte[] { 1 });
|
||||
|
||||
await store.DeleteMaterialAsync(created.ProjectId, addition.Material.MaterialId);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(await store.ListMaterialsAsync(created.ProjectId), Is.Empty);
|
||||
Assert.That(File.Exists(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".png")), Is.False);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARecordingIsStoredWithItsTranscript()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var material = Material("Interview.mp3", HASH);
|
||||
material.Kind = BflMaterialKind.RECORDING;
|
||||
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, material, "Guten Tag.");
|
||||
var materials = await this.CreateStore().ListMaterialsAsync(created.ProjectId);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(materials[0].Material!.Kind, Is.EqualTo(BflMaterialKind.RECORDING));
|
||||
Assert.That(await store.ReadMaterialTextAsync(created.ProjectId, addition.Material.MaterialId, 100), Is.EqualTo("Guten Tag."));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMaterialStoredBeforeTheKindsIsADocument()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
var json = this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".json");
|
||||
|
||||
var lines = (await File.ReadAllLinesAsync(json)).Where(line => !line.Contains("\"kind\"", StringComparison.Ordinal) && !line.Contains("\"image\"", StringComparison.Ordinal));
|
||||
await File.WriteAllLinesAsync(json, lines);
|
||||
var materials = await this.CreateStore().ListMaterialsAsync(created.ProjectId);
|
||||
|
||||
Assert.That(materials.Single().Material!.Kind, Is.EqualTo(BflMaterialKind.DOCUMENT));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImageWithoutTheDescriptionOfItsCopyIsDamaged()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddImageMaterialAsync(created.ProjectId, ImageMaterial("Poster.png", HASH), new byte[] { 1 });
|
||||
var json = this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".json");
|
||||
|
||||
await File.WriteAllTextAsync(json, (await File.ReadAllTextAsync(json)).Replace("\"png\"", "\"../../x\"", StringComparison.Ordinal));
|
||||
var materials = await this.CreateStore().ListMaterialsAsync(created.ProjectId);
|
||||
|
||||
Assert.That(materials.Single().IsAvailable, Is.False, "An unknown extension could point anywhere.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImageCannotBeAddedAsText()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(async () => await store.AddMaterialAsync(Guid.NewGuid(), ImageMaterial("Poster.png", HASH), "Text"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADocumentCannotBeAddedAsAnImage()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(async () => await store.AddImageMaterialAsync(Guid.NewGuid(), Material("Paper.pdf", HASH), new byte[] { 1 }));
|
||||
}
|
||||
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public async Task AMaterialKeepsWhetherItsTextIsOnlyTheBeginning(bool isShortened)
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var material = Material("Buch.pdf", HASH);
|
||||
material.IsShortened = isShortened;
|
||||
|
||||
await store.AddMaterialAsync(created.ProjectId, material, "Kapitel 1");
|
||||
var materials = await this.CreateStore().ListMaterialsAsync(created.ProjectId);
|
||||
|
||||
Assert.That(materials[0].Material!.IsShortened, Is.EqualTo(isShortened));
|
||||
}
|
||||
|
||||
[TestCase("/Users/person/Documents/Talk.pdf")]
|
||||
[TestCase(@"C:\Users\person\Documents\Talk.pdf")]
|
||||
[TestCase("Talk.pdf")]
|
||||
public async Task TheStoreKeepsNoPath(string name)
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material(name, HASH), "Text");
|
||||
var json = await File.ReadAllTextAsync(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".json"));
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(addition.Material.Name, Is.EqualTo("Talk.pdf"));
|
||||
Assert.That(json, Does.Not.Contain("Users"));
|
||||
Assert.That(json, Does.Not.Contain("Documents"));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderAloneIsNoName()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(async () => await store.AddMaterialAsync(created.ProjectId, Material("/Users/person/", HASH), "Text"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADocumentWithTheSameContentIsStoredOnce()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
|
||||
var first = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
var second = await store.AddMaterialAsync(created.ProjectId, Material("Paper (Kopie).pdf", HASH.ToUpperInvariant()), "Text");
|
||||
var materials = await store.ListMaterialsAsync(created.ProjectId);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(second.IsDuplicate, Is.True);
|
||||
Assert.That(second.Material.MaterialId, Is.EqualTo(first.Material.MaterialId));
|
||||
Assert.That(second.Material.Name, Is.EqualTo("Paper.pdf"));
|
||||
Assert.That(materials, Has.Count.EqualTo(1));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task MaterialsAreListedInTheOrderTheyWereAdded()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
|
||||
await store.AddMaterialAsync(created.ProjectId, Material("Erstes.pdf", HASH), "Eins");
|
||||
await Task.Delay(20);
|
||||
await store.AddMaterialAsync(created.ProjectId, Material("Zweites.pdf", OTHER_HASH), "Zwei");
|
||||
var materials = await store.ListMaterialsAsync(created.ProjectId);
|
||||
|
||||
Assert.That(materials.Select(entry => entry.Material!.Name), Is.EqualTo(new[] { "Erstes.pdf", "Zweites.pdf" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APreparationWithoutMaterialsHasNone()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
|
||||
Assert.That(await store.ListMaterialsAsync(created.ProjectId), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMaterialWithoutTextCannotBeOpened()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
|
||||
File.Delete(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".md"));
|
||||
var materials = await store.ListMaterialsAsync(created.ProjectId);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(materials, Has.Count.EqualTo(1));
|
||||
Assert.That(materials[0].IsAvailable, Is.False);
|
||||
Assert.That(materials[0].MaterialId, Is.EqualTo(addition.Material.MaterialId));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedMaterialIsListedAndCanBeDeleted()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
var json = this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".json");
|
||||
|
||||
await File.WriteAllTextAsync(json, """{ "schemaVersion": 1, "fieldOfANewerVersion": true }""");
|
||||
var damaged = await store.ListMaterialsAsync(created.ProjectId);
|
||||
await store.DeleteMaterialAsync(created.ProjectId, addition.Material.MaterialId);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(damaged.Single().IsAvailable, Is.False);
|
||||
Assert.That(await store.ListMaterialsAsync(created.ProjectId), Is.Empty);
|
||||
Assert.That(File.Exists(json), Is.False);
|
||||
Assert.That(File.Exists(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".md")), Is.False);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedMaterialIsNoDuplicate()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
await File.WriteAllTextAsync(this.MaterialFile(created.ProjectId, addition.Material.MaterialId, ".json"), "{");
|
||||
|
||||
var again = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
|
||||
Assert.That(again.IsDuplicate, Is.False, "The person cannot see the damaged one, so the document has to come back.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DeletingAMaterialCountsAsAChange()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
var afterAdding = await store.LoadAsync(created.ProjectId);
|
||||
await Task.Delay(20);
|
||||
|
||||
var afterDeleting = await store.DeleteMaterialAsync(created.ProjectId, addition.Material.MaterialId);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(afterDeleting.ModifiedAtUtc, Is.GreaterThan(afterAdding!.ModifiedAtUtc));
|
||||
Assert.That(await store.ListMaterialsAsync(created.ProjectId), Is.Empty);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ReadingStopsAtTheLimit()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var text = string.Concat(Enumerable.Repeat("Ein Satz über Wissenschaft. ", 10_000));
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), text);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(await store.ReadMaterialTextAsync(created.ProjectId, addition.Material.MaterialId, 100_000), Is.EqualTo(text[..100_000]));
|
||||
Assert.That(await store.ReadMaterialTextAsync(created.ProjectId, addition.Material.MaterialId, text.Length + 10), Is.EqualTo(text));
|
||||
Assert.That(await store.ReadMaterialTextAsync(created.ProjectId, addition.Material.MaterialId, 0), Is.Empty);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ReadingADeletedMaterialFails()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var addition = await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
await store.DeleteMaterialAsync(created.ProjectId, addition.Material.MaterialId);
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(async () => await store.ReadMaterialTextAsync(created.ProjectId, addition.Material.MaterialId, 100));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMaterialOfAMissingPreparationIsNotStored()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
|
||||
await Assert.ThrowsAsync<FileNotFoundException>(async () => await store.AddMaterialAsync(Guid.NewGuid(), Material("Paper.pdf", HASH), "Text"));
|
||||
}
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase(" \n")]
|
||||
public async Task AMaterialWithoutTextIsRefused(string text)
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(async () => await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), text));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DeletingAPreparationDeletesItsMaterials()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
await store.AddMaterialAsync(created.ProjectId, Material("Paper.pdf", HASH), "Text");
|
||||
|
||||
await store.DeleteAsync(created.ProjectId);
|
||||
|
||||
Assert.That(Directory.Exists(Path.Combine(this.RootDirectory, created.ProjectId.ToString("D"))), Is.False);
|
||||
}
|
||||
|
||||
private static BflMaterial Material(string name, string hash) => new()
|
||||
{
|
||||
Name = name,
|
||||
SourceHash = hash,
|
||||
SourceBytes = 2_048,
|
||||
};
|
||||
|
||||
private static BflMaterial ImageMaterial(string name, string hash) => new()
|
||||
{
|
||||
Name = name,
|
||||
Kind = BflMaterialKind.IMAGE,
|
||||
Image = new() { Extension = "png", Width = 2_560, Height = 1_440 },
|
||||
SourceHash = hash,
|
||||
SourceBytes = 4_096,
|
||||
};
|
||||
|
||||
private string MaterialFile(Guid projectId, Guid materialId, string extension) => Path.Combine(this.RootDirectory, projectId.ToString("D"), "materials", $"{materialId:D}{extension}");
|
||||
|
||||
private BeyondFactsLabStore CreateStore() => new(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory });
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tests.Tools;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.AssistantSessions;
|
||||
using AIStudio.Tools.Security;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the BeyondFacts Lab adds an image to the material of a preparation: as a copy which the runtime
|
||||
/// prepared for models, never as a path to the file of the person.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// FakeImageRuntime plays the runtime and hands back the same three bytes for every image.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class BflImageMaterialTests
|
||||
{
|
||||
private string dataDirectory = string.Empty;
|
||||
private FakeImageRuntime runtime = null!;
|
||||
private RustService rustService = null!;
|
||||
private MediaTranscriptionService transcriptionService = null!;
|
||||
private BeyondFactsLabStore store = null!;
|
||||
private AssistantSessionService sessionService = null!;
|
||||
private BflMaterialService service = null!;
|
||||
|
||||
[SetUp]
|
||||
public void PrepareService()
|
||||
{
|
||||
this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-images-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.dataDirectory);
|
||||
|
||||
this.runtime = new FakeImageRuntime();
|
||||
this.rustService = new RustService(this.runtime);
|
||||
var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
|
||||
var guardService = new PromptInjectionGuardService(this.rustService, settingsManager, NullLogger<PromptInjectionGuardService>.Instance, NullLoggerFactory.Instance);
|
||||
this.transcriptionService = new MediaTranscriptionService(null!, settingsManager, null!, [], NullLogger<MediaTranscriptionService>.Instance);
|
||||
this.store = new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory });
|
||||
this.sessionService = new AssistantSessionService(MessageBus.INSTANCE);
|
||||
this.service = new BflMaterialService(this.store, this.rustService, guardService, MessageBus.INSTANCE, settingsManager, this.transcriptionService, this.sessionService, NullLogger<BflMaterialService>.Instance);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void CleanUp()
|
||||
{
|
||||
this.transcriptionService.Dispose();
|
||||
this.rustService.Dispose();
|
||||
this.runtime.Dispose();
|
||||
if (Directory.Exists(this.dataDirectory))
|
||||
Directory.Delete(this.dataDirectory, true);
|
||||
}
|
||||
|
||||
[TestCase("image/jpeg", "jpg")]
|
||||
[TestCase("image/png", "png")]
|
||||
public async Task AnImageBecomesACopyPreparedForModels(string mimeType, string extension)
|
||||
{
|
||||
this.runtime.MimeType = mimeType;
|
||||
var project = await this.store.CreateAsync("Panel");
|
||||
var poster = await this.WriteFileAsync("Poster.tiff");
|
||||
|
||||
await this.service.ImportAsync(project.ProjectId, [poster], null!);
|
||||
var material = (await this.store.ListMaterialsAsync(project.ProjectId)).Single().Material!;
|
||||
var copy = this.store.MaterialImagePath(project.ProjectId, material);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(material.Kind, Is.EqualTo(BflMaterialKind.IMAGE));
|
||||
Assert.That(material.Name, Is.EqualTo("Poster.tiff"));
|
||||
Assert.That(material.Image!.Extension, Is.EqualTo(extension), "The runtime tells the format of the copy by its extension.");
|
||||
Assert.That(await File.ReadAllBytesAsync(copy), Is.EqualTo(Convert.FromBase64String(FakeImageRuntime.IMAGE_BASE64)));
|
||||
Assert.That(copy, Does.StartWith(Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME)), "The material must not depend on the file of the person.");
|
||||
Assert.That(this.runtime.RequestedPaths, Is.EqualTo(new[] { poster }));
|
||||
Assert.That(this.runtime.RequestedBudgets, Is.EqualTo(new[] { IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES }), "The copy must be as large as any provider takes.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheLabShowsThatItWorksWhileItReadsMaterial()
|
||||
{
|
||||
var project = await this.store.CreateAsync("Panel");
|
||||
var poster = await this.WriteFileAsync("Poster.png");
|
||||
var key = BflMaterialService.ImportSessionKey(project.ProjectId);
|
||||
List<AssistantSessionStatus?> statusesWhileReading = [];
|
||||
this.service.MaterialsChanged += _ =>
|
||||
{
|
||||
if (this.service.MaterialInProgress(project.ProjectId) is not null)
|
||||
statusesWhileReading.Add(this.sessionService.TryGetSnapshot(key)?.Status);
|
||||
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
|
||||
await this.service.ImportAsync(project.ProjectId, [poster], null!);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(statusesWhileReading, Is.EqualTo(new AssistantSessionStatus?[] { AssistantSessionStatus.RUNNING }), "The navigation and the tile of the lab show a running session.");
|
||||
Assert.That(this.sessionService.TryGetSnapshot(key), Is.Null, "Once the material is there, nothing is left to show.");
|
||||
Assert.That(this.sessionService.HasActiveSessions, Is.False);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImageTheRuntimeCannotPrepareIsLeftOut()
|
||||
{
|
||||
this.runtime.ErrorCode = "NOT_DECODABLE";
|
||||
var project = await this.store.CreateAsync("Panel");
|
||||
|
||||
await this.service.ImportAsync(project.ProjectId, [await this.WriteFileAsync("Poster.png")], null!);
|
||||
|
||||
Assert.That(await this.store.ListMaterialsAsync(project.ProjectId), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheSameImageIsAddedOnce()
|
||||
{
|
||||
var project = await this.store.CreateAsync("Panel");
|
||||
var poster = await this.WriteFileAsync("Poster.png");
|
||||
|
||||
await this.service.ImportAsync(project.ProjectId, [poster], null!);
|
||||
await this.service.ImportAsync(project.ProjectId, [poster], null!);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(await this.store.ListMaterialsAsync(project.ProjectId), Has.Count.EqualTo(1));
|
||||
Assert.That(this.runtime.RequestedPaths, Has.Count.EqualTo(1), "The lab prepared an image the preparation had already.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImageModelsCannotTakeIsNoMaterial()
|
||||
{
|
||||
var project = await this.store.CreateAsync("Panel");
|
||||
|
||||
await this.service.ImportAsync(project.ProjectId, [await this.WriteFileAsync("Poster.heic")], null!);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(await this.store.ListMaterialsAsync(project.ProjectId), Is.Empty);
|
||||
Assert.That(this.runtime.RequestedPaths, Is.Empty);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> WriteFileAsync(string name)
|
||||
{
|
||||
var folder = Path.Combine(this.dataDirectory, "person", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(folder);
|
||||
var path = Path.Combine(folder, name);
|
||||
await File.WriteAllBytesAsync(path, [1, 2, 3, 4]);
|
||||
return path;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Models;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how much of the material of a preparation the BeyondFacts Lab puts into the prompt of a step.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BflMaterialBudgetTests
|
||||
{
|
||||
[Test]
|
||||
public void TheMaterialGetsHalfOfWhatThePromptLeavesFree()
|
||||
{
|
||||
var characters = BflMaterialBudget.CharactersFor(ContextWindow.Of(128_000), 90_000);
|
||||
|
||||
Assert.That(characters, Is.EqualTo(((128_000 - BflTokenBudget.RESERVE_TOKENS) * 4 - 90_000) / 2));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EachImageTakesItsRoomBeforeTheText()
|
||||
{
|
||||
var characters = BflMaterialBudget.CharactersFor(ContextWindow.Of(128_000), 90_000, sentImages: 2);
|
||||
|
||||
Assert.That(characters, Is.EqualTo(((128_000 - BflTokenBudget.RESERVE_TOKENS - 2 * BflMaterialBudget.TOKENS_PER_IMAGE) * 4 - 90_000) / 2));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ImagesBeyondWhatTheModelTakesStayBehind()
|
||||
{
|
||||
List<BflMaterial> images = [Image("A.png"), Image("B.png"), Image("C.png")];
|
||||
|
||||
var shares = BflMaterialBudget.ShareImages(images, 2);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(shares.Select(share => share.Image), Is.EqualTo(new[] { BflImageDelivery.SENT, BflImageDelivery.SENT, BflImageDelivery.OVER_THE_LIMIT }), "The images the person added first go along.");
|
||||
Assert.That(shares.Select(share => share.Material.Name), Is.EqualTo(new[] { "A.png", "B.png", "C.png" }));
|
||||
Assert.That(shares.All(share => share.Characters is 0), Is.True, "An image takes no characters.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModelWhichSeesNoImagesGetsNone()
|
||||
{
|
||||
var shares = BflMaterialBudget.ShareImages([Image("A.png"), Image("B.png")], 0);
|
||||
|
||||
Assert.That(shares.Select(share => share.Image), Is.All.EqualTo(BflImageDelivery.MODEL_SEES_NO_IMAGES));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AHugeWindowStillCarriesNoBook()
|
||||
{
|
||||
Assert.That(BflMaterialBudget.CharactersFor(ContextWindow.Of(1_000_000), 90_000), Is.EqualTo(BflMaterialBudget.MAX_CHARACTERS));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASmallWindowLeavesNoRoom()
|
||||
{
|
||||
Assert.That(BflMaterialBudget.CharactersFor(ContextWindow.Of(24_000), 90_000), Is.Zero);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnUnknownWindowGetsAFixedAmount()
|
||||
{
|
||||
Assert.That(BflMaterialBudget.CharactersFor(ContextWindow.UNKNOWN, 90_000), Is.EqualTo(BflMaterialBudget.CHARACTERS_WITHOUT_A_KNOWN_CONTEXT_WINDOW));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ShortMaterialComesWholeAndLongMaterialIsShortenedAlike()
|
||||
{
|
||||
var shares = BflMaterialBudget.Share([Material(10_000), Material(100), Material(50_000)], 20_100);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(shares.Select(share => share.Characters), Is.EqualTo(new[] { 10_000, 100, 10_000 }), "The order stays the order the person added them in.");
|
||||
Assert.That(shares.Select(share => share.IsComplete), Is.EqualTo(new[] { true, true, false }));
|
||||
Assert.That(shares[2].Percent, Is.EqualTo(20));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WhatAShortMaterialLeavesGoesToTheOthers()
|
||||
{
|
||||
var shares = BflMaterialBudget.Share([Material(30_000), Material(1_000), Material(30_000)], 21_000);
|
||||
|
||||
Assert.That(shares.Select(share => share.Characters), Is.EqualTo(new[] { 10_000, 1_000, 10_000 }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NoRoomMeansNothingOfAnyMaterial()
|
||||
{
|
||||
var shares = BflMaterialBudget.Share([Material(500), Material(5_000)], 0);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(shares.Select(share => share.Characters), Is.EqualTo(new[] { 0, 0 }));
|
||||
Assert.That(shares.Select(share => share.Percent), Is.EqualTo(new[] { 0, 0 }));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheSharesNeverExceedTheBudget()
|
||||
{
|
||||
var materials = Enumerable.Range(1, 17).Select(index => Material(index * 7_919)).ToList();
|
||||
|
||||
var shares = BflMaterialBudget.Share(materials, 123_457);
|
||||
|
||||
Assert.That(shares.Sum(share => share.Characters), Is.LessThanOrEqualTo(123_457));
|
||||
}
|
||||
|
||||
[TestCase(1_000, 1, 1)]
|
||||
[TestCase(1_000, 999, 99)]
|
||||
[TestCase(1_000, 1_000, 100)]
|
||||
public void APartIsNeitherNothingNorEverything(int textLength, int characters, int percent)
|
||||
{
|
||||
Assert.That(new BflMaterialShare(Material(textLength), characters).Percent, Is.EqualTo(percent));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AShortenedMaterialEndsAtALineEnd()
|
||||
{
|
||||
var beginning = "Erste Zeile\n" + new string('x', 95) + "\nletzt";
|
||||
|
||||
Assert.That(BflMaterialBudget.EndAtLineEnd(beginning), Is.EqualTo("Erste Zeile\n" + new string('x', 95)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ALineEndFarAwayIsNotWorthTheLoss()
|
||||
{
|
||||
var beginning = "Kurz\n" + new string('x', 200);
|
||||
|
||||
Assert.That(BflMaterialBudget.EndAtLineEnd(beginning), Is.EqualTo(beginning));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ACutNeverSplitsACharacter()
|
||||
{
|
||||
var beginning = new string('x', 50) + "😀"[..1];
|
||||
|
||||
Assert.That(BflMaterialBudget.EndAtLineEnd(beginning), Is.EqualTo(new string('x', 50)));
|
||||
}
|
||||
|
||||
private static BflMaterial Material(int textLength) => new()
|
||||
{
|
||||
MaterialId = Guid.NewGuid(),
|
||||
Name = $"{textLength}.pdf",
|
||||
TextLength = textLength,
|
||||
};
|
||||
|
||||
private static BflMaterial Image(string name) => new()
|
||||
{
|
||||
MaterialId = Guid.NewGuid(),
|
||||
Name = name,
|
||||
Kind = BflMaterialKind.IMAGE,
|
||||
Image = new() { Extension = "png", Width = 100, Height = 100 },
|
||||
};
|
||||
}
|
||||
@@ -9,7 +9,7 @@ namespace AIStudio.Tests.BeyondFactsLab;
|
||||
[TestFixture]
|
||||
public sealed class BflPromptComposerTests
|
||||
{
|
||||
private const string CORE = "Sprache: {{RESPONSE_LANGUAGE}}\n{{ADDRESS_FORM}}\n{{KNOWLEDGE}}\n{{STEP}}\n{{DETAILS}}\n{{MISSING_DETAILS}}";
|
||||
private const string CORE = "Sprache: {{RESPONSE_LANGUAGE}}\n{{ADDRESS_FORM}}\n{{KNOWLEDGE}}\n{{STEP}}\n{{MATERIALS}}\n{{DETAILS}}\n{{MISSING_DETAILS}}";
|
||||
|
||||
private static readonly BflPromptParts PARTS = new(CORE, "Leitfaden", "Schritt", "Anrede");
|
||||
|
||||
@@ -41,7 +41,7 @@ public sealed class BflPromptComposerTests
|
||||
foreach (var step in BflStep.All)
|
||||
{
|
||||
var parts = await BflPromptLibrary.LoadPartsAsync(step, AddressForm.FORMAL);
|
||||
var prompt = BflPromptComposer.Compose(parts, step, new Dictionary<BflStep, BflStepFormData>(), "Deutsch");
|
||||
var prompt = BflPromptComposer.Compose(parts, step, new Dictionary<BflStep, BflStepFormData>(), [], "Deutsch");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
@@ -55,8 +55,8 @@ public sealed class BflPromptComposerTests
|
||||
[Test]
|
||||
public async Task TheFormOfAddressDecidesTheInstruction()
|
||||
{
|
||||
var formal = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.FORMAL), BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), "Deutsch");
|
||||
var informal = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.INFORMAL), BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), "Deutsch");
|
||||
var formal = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.FORMAL), BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [], "Deutsch");
|
||||
var informal = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.INFORMAL), BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [], "Deutsch");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
@@ -119,7 +119,7 @@ public sealed class BflPromptComposerTests
|
||||
[Test]
|
||||
public void TheLanguageOfTheAnswersIsNamed()
|
||||
{
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), "English (United States)");
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [], "English (United States)");
|
||||
|
||||
Assert.That(prompt, Does.StartWith("Sprache: English (United States)\n"));
|
||||
}
|
||||
@@ -127,7 +127,7 @@ public sealed class BflPromptComposerTests
|
||||
[Test]
|
||||
public void TheMissingKeyDetailsAreListed()
|
||||
{
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), "Deutsch");
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [], "Deutsch");
|
||||
|
||||
foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.IsKeyDetail))
|
||||
Assert.That(prompt, Does.Contain(field.Label), field.Id);
|
||||
@@ -140,7 +140,7 @@ public sealed class BflPromptComposerTests
|
||||
foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.IsKeyDetail))
|
||||
forms[field.Step].Set(field.Id, "vorhanden");
|
||||
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, forms, "Deutsch");
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, forms, [], "Deutsch");
|
||||
|
||||
Assert.That(prompt, Does.EndWith("Es fehlen keine wichtigen Angaben."));
|
||||
}
|
||||
@@ -155,7 +155,7 @@ public sealed class BflPromptComposerTests
|
||||
goals.SetChoices(directions, [BflFieldCatalog.GOAL_INFORM, BflFieldCatalog.GOAL_SHOW_OPTIONS]);
|
||||
var forms = new Dictionary<BflStep, BflStepFormData> { [BflStep.FRAMEWORK] = framework, [BflStep.GOALS_AND_ROLE] = goals };
|
||||
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, forms, "Deutsch");
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, forms, [], "Deutsch");
|
||||
var inform = directions.Options.Single(option => option.Id == BflFieldCatalog.GOAL_INFORM).Label;
|
||||
var showOptions = directions.Options.Single(option => option.Id == BflFieldCatalog.GOAL_SHOW_OPTIONS).Label;
|
||||
var format = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.FORMAT).Label;
|
||||
@@ -175,7 +175,7 @@ public sealed class BflPromptComposerTests
|
||||
var knowledge = new BflStepFormData();
|
||||
knowledge.Set(BflFieldCatalog.CORE_THESES, "Erste These\r\nZweite These");
|
||||
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.KNOWLEDGE, new Dictionary<BflStep, BflStepFormData> { [BflStep.KNOWLEDGE] = knowledge }, "Deutsch");
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.KNOWLEDGE, new Dictionary<BflStep, BflStepFormData> { [BflStep.KNOWLEDGE] = knowledge }, [], "Deutsch");
|
||||
|
||||
Assert.That(prompt, Does.Contain(": Erste These\n Zweite These"));
|
||||
}
|
||||
@@ -190,7 +190,7 @@ public sealed class BflPromptComposerTests
|
||||
|
||||
foreach (var step in BflStep.All)
|
||||
{
|
||||
var prompt = BflPromptComposer.Compose(PARTS, step, forms, "Deutsch");
|
||||
var prompt = BflPromptComposer.Compose(PARTS, step, forms, [], "Deutsch");
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt.Contains("VERTRAULICH-4711", StringComparison.Ordinal), Is.EqualTo(step is BflStep.VALUES_AND_PERSPECTIVES), step.ToString());
|
||||
@@ -208,8 +208,8 @@ public sealed class BflPromptComposerTests
|
||||
var strategy = new BflStepFormData();
|
||||
strategy.Set(BflFieldCatalog.COMMUNICATION_WORRIES, "Zeitdruck");
|
||||
|
||||
var first = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, new Dictionary<BflStep, BflStepFormData> { [BflStep.FRAMEWORK] = framework, [BflStep.STRATEGY] = strategy }, "Deutsch");
|
||||
var second = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, new Dictionary<BflStep, BflStepFormData> { [BflStep.STRATEGY] = strategy, [BflStep.FRAMEWORK] = framework }, "Deutsch");
|
||||
var first = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, new Dictionary<BflStep, BflStepFormData> { [BflStep.FRAMEWORK] = framework, [BflStep.STRATEGY] = strategy }, [], "Deutsch");
|
||||
var second = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, new Dictionary<BflStep, BflStepFormData> { [BflStep.STRATEGY] = strategy, [BflStep.FRAMEWORK] = framework }, [], "Deutsch");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
@@ -218,4 +218,116 @@ public sealed class BflPromptComposerTests
|
||||
Assert.That(first.IndexOf("Interview", StringComparison.Ordinal), Is.LessThan(first.IndexOf("Öffentlichkeit", StringComparison.Ordinal)), "The details follow the order of the forms.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void APreparationWithoutMaterialSaysSo()
|
||||
{
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [], "Deutsch");
|
||||
|
||||
Assert.That(prompt, Does.Contain("Die Person hat kein Material hinzugefügt."));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheMaterialComesWithItsNameAndText()
|
||||
{
|
||||
var invitation = Excerpt("Einladung.docx", "Podium am 10. Dezember\r\nim Audimax", 10_000);
|
||||
var paper = Excerpt("Paper.pdf", "Abstract", 10_000);
|
||||
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [invitation, paper], "Deutsch");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt, Does.Contain("## Material 1: „Einladung.docx“\n\nPodium am 10. Dezember\nim Audimax"));
|
||||
Assert.That(prompt, Does.Contain("## Material 2: „Paper.pdf“\n\nAbstract"));
|
||||
Assert.That(prompt, Does.Not.Contain("Gekürzt"));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AShortenedMaterialSaysHowMuchOfItIsThere()
|
||||
{
|
||||
var material = new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Paper.pdf", TextLength = 1_000 };
|
||||
var shortened = new BflMaterialExcerpt(new(material, 400), "Der Anfang");
|
||||
var missing = new BflMaterialExcerpt(new(new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Buch.pdf", TextLength = 1_000 }, 0), string.Empty);
|
||||
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [shortened, missing], "Deutsch");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt, Does.Contain("(Gekürzt: Hier steht nur der Anfang, etwa 40 % des Materials."));
|
||||
Assert.That(prompt, Does.Contain("## Material 2: „Buch.pdf“\n(Für dieses Material reicht der Platz im Kontext des Modells nicht. Du kennst nur seinen Namen.)"));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void APlaceholderInTheMaterialStaysText()
|
||||
{
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [Excerpt("Notizen.md", "Siehe {{KNOWLEDGE}}", 100)], "Deutsch");
|
||||
|
||||
Assert.That(prompt, Does.Contain("Siehe {{KNOWLEDGE}}"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheMaterialStandsBetweenTheStepAndTheDetails()
|
||||
{
|
||||
var parts = await BflPromptLibrary.LoadPartsAsync(BflStep.KNOWLEDGE, AddressForm.FORMAL);
|
||||
var knowledge = new BflStepFormData();
|
||||
knowledge.Set(BflFieldCatalog.CORE_THESES, "THESE-4711");
|
||||
|
||||
var prompt = BflPromptComposer.Compose(parts, BflStep.KNOWLEDGE, new Dictionary<BflStep, BflStepFormData> { [BflStep.KNOWLEDGE] = knowledge }, [Excerpt("Paper.pdf", "MATERIAL-4711", 100)], "Deutsch");
|
||||
var material = prompt.IndexOf("MATERIAL-4711", StringComparison.Ordinal);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt.IndexOf(parts.Step.Trim(), StringComparison.Ordinal), Is.LessThan(material), "The material changes less often than the details, so it comes first for the prompt caching.");
|
||||
Assert.That(material, Is.LessThan(prompt.IndexOf("THESE-4711", StringComparison.Ordinal)));
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(BflImageDelivery.SENT, "(Ein Bild. Es liegt der ersten Nachricht dieses Gesprächs bei.)")]
|
||||
[TestCase(BflImageDelivery.MODEL_SEES_NO_IMAGES, "(Ein Bild. Das Modell kann keine Bilder sehen. Du kennst nur seinen Namen.)")]
|
||||
[TestCase(BflImageDelivery.OVER_THE_LIMIT, "(Ein Bild. Das Modell nimmt nicht so viele Bilder auf einmal an. Du kennst nur seinen Namen.)")]
|
||||
public void AnImageSaysWhetherItGoesAlong(BflImageDelivery delivery, string expected)
|
||||
{
|
||||
var image = new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Poster.png", Kind = BflMaterialKind.IMAGE };
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [new(new(image, 0, delivery), string.Empty)], "Deutsch");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt, Does.Contain($"## Material 1: „Poster.png“\n{expected}"));
|
||||
Assert.That(prompt, Does.Not.Contain("reicht der Platz"), "An image takes no room in the prompt.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARecordingIsMarkedAsATranscript()
|
||||
{
|
||||
var recording = new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Interview.mp3", Kind = BflMaterialKind.RECORDING, TextLength = 10 };
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [new(new(recording, 10), "Guten Tag.")], "Deutsch");
|
||||
|
||||
Assert.That(prompt, Does.Contain("## Material 1: „Interview.mp3“\n(Das Transkript einer Aufnahme."));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMaterialWhoseEndTheLabLeftOutSaysSo()
|
||||
{
|
||||
var book = new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Buch.pdf", TextLength = 10, IsShortened = true };
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [new(new(book, 10), "Kapitel 1.")], "Deutsch");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt, Does.Contain("(Das Lab hat nur den Anfang dieses Materials übernommen. Den Rest kennst du nicht.)"));
|
||||
Assert.That(prompt, Does.Not.Contain("Gekürzt:"), "The whole stored text fits.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheOnboardingKnowsNoMaterial()
|
||||
{
|
||||
var parts = new BflOnboardingPromptParts("{{MATERIALS}}", "Anrede");
|
||||
|
||||
Assert.Throws<FormatException>(() => BflPromptComposer.ComposeOnboarding(parts, "Deutsch"));
|
||||
}
|
||||
|
||||
private static BflMaterialExcerpt Excerpt(string name, string text, int textLength) => new(new(new BflMaterial { MaterialId = Guid.NewGuid(), Name = name, TextLength = textLength }, textLength), text);
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the BeyondFacts Lab turns recordings into material: who may hear them, and where their transcripts wait.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BflRecordingTests
|
||||
{
|
||||
private string dataDirectory = string.Empty;
|
||||
private SettingsManager settingsManager = null!;
|
||||
private MediaTranscriptionService transcriptionService = null!;
|
||||
|
||||
[SetUp]
|
||||
public void PrepareSettings()
|
||||
{
|
||||
this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-recordings-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.dataDirectory);
|
||||
|
||||
this.settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
|
||||
var transcriptionProvider = new TranscriptionProvider { Id = Guid.NewGuid().ToString(), Name = "Transcription", UsedLLMProvider = LLMProviders.OPEN_AI };
|
||||
this.settingsManager.ConfigurationData.TranscriptionProviders.Add(transcriptionProvider);
|
||||
this.settingsManager.ConfigurationData.App.UseTranscriptionProvider = transcriptionProvider.Id;
|
||||
this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.CUSTOM;
|
||||
|
||||
// Finding the provider reads only the settings, so the other services stay out of these tests:
|
||||
this.transcriptionService = new MediaTranscriptionService(null!, this.settingsManager, null!, [], NullLogger<MediaTranscriptionService>.Instance);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void CleanUp()
|
||||
{
|
||||
this.transcriptionService.Dispose();
|
||||
if (Directory.Exists(this.dataDirectory))
|
||||
Directory.Delete(this.dataDirectory, true);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ATranscriptionProviderBelowTheConfidenceOfTheLabHearsNoRecording()
|
||||
{
|
||||
this.settingsManager.ConfigurationData.Confidence.CustomConfidenceScheme[LLMProviders.OPEN_AI] = ConfidenceLevel.LOW;
|
||||
var service = this.CreateService();
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(this.transcriptionService.GetUsableProvider(), Is.Not.Null, "The chat may still use the provider.");
|
||||
Assert.That(service.UsableTranscriptionProvider(), Is.Null, "A recording of the lab is as confidential as a paper.");
|
||||
Assert.That(service.MinimumTranscriptionConfidence, Is.GreaterThanOrEqualTo(ConfidenceLevel.MEDIUM));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ATranscriptionProviderWithTheConfidenceOfTheLabHearsRecordings()
|
||||
{
|
||||
this.settingsManager.ConfigurationData.Confidence.CustomConfidenceScheme[LLMProviders.OPEN_AI] = ConfidenceLevel.MEDIUM;
|
||||
|
||||
Assert.That(this.CreateService().UsableTranscriptionProvider(), Is.Not.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AProhibitedTranscriptionProviderHearsNoRecording()
|
||||
{
|
||||
this.settingsManager.ConfigurationData.Confidence.CustomConfidenceScheme[LLMProviders.OPEN_AI] = ConfidenceLevel.HIGH;
|
||||
this.settingsManager.ConfigurationData.App.ProhibitedProviders.Add(LLMProviders.OPEN_AI);
|
||||
|
||||
Assert.That(this.CreateService().UsableTranscriptionProvider(), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ATranscriptWaitsInAFolderOfItsOwnBelowTheStaging()
|
||||
{
|
||||
var first = new BflTranscriptLocation(this.dataDirectory);
|
||||
var second = new BflTranscriptLocation(this.dataDirectory);
|
||||
|
||||
var slot = await first.ResolveAsync("/Users/person/Interview.mp3", CancellationToken.None);
|
||||
var stagingDirectory = Path.Combine(this.dataDirectory, "media-staging");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(Path.GetDirectoryName(slot.Path), Is.EqualTo(first.FolderPath));
|
||||
Assert.That(Path.GetDirectoryName(first.FolderPath), Is.EqualTo(stagingDirectory), "The cleanup at the start of AI Studio removes only what lies below the staging.");
|
||||
Assert.That(second.FolderPath, Is.Not.EqualTo(first.FolderPath), "Two imports would share one transcript.");
|
||||
Assert.That(slot.Path, Does.Not.Contain("Interview"), "The folder of the person decides nothing in the data directory.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DeletingTheLocationRemovesTheTranscript()
|
||||
{
|
||||
var location = new BflTranscriptLocation(this.dataDirectory);
|
||||
var slot = await location.ResolveAsync("/Users/person/Interview.mp3", CancellationToken.None);
|
||||
Directory.CreateDirectory(location.FolderPath);
|
||||
await File.WriteAllTextAsync(slot.Path, "Guten Tag.");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(location.TryDelete(), Is.True);
|
||||
Assert.That(Directory.Exists(location.FolderPath), Is.False);
|
||||
Assert.That(new BflTranscriptLocation(this.dataDirectory).TryDelete(), Is.True, "A location without a transcript is no failure.");
|
||||
}
|
||||
}
|
||||
|
||||
private BflMaterialService CreateService() => new(new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory }), null!, null!, null!, this.settingsManager, this.transcriptionService, null!, NullLogger<BflMaterialService>.Instance);
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Chat;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the images of the material go along with a request of the BeyondFacts Lab.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The images belong to the request only: the stored chat must never hold a path to them, the answer streams into
|
||||
/// the chat while the request runs, and the tokens a provider reports belong to a chat with as many messages.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class BflRequestImageTests
|
||||
{
|
||||
[Test]
|
||||
public void TheImagesGoAlongWithTheFirstMessage()
|
||||
{
|
||||
var thread = Conversation();
|
||||
var image = new FileAttachmentImage("Poster.png", "/data/beyondFactsLab/p/materials/m.png", 1_024);
|
||||
|
||||
var request = BflConversationService.WithImages(thread, [image]);
|
||||
var firstMessage = (ContentText)request.Blocks[0].Content!;
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(firstMessage.FileAttachments, Is.EqualTo(new FileAttachment[] { image }));
|
||||
Assert.That(firstMessage.Text, Is.EqualTo("Eröffnung"));
|
||||
Assert.That(request.Blocks[0].HideFromUser, Is.True, "The opening request stays hidden.");
|
||||
Assert.That(request.ChatId, Is.EqualTo(thread.ChatId));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheStoredChatHoldsNoImage()
|
||||
{
|
||||
var thread = Conversation();
|
||||
|
||||
BflConversationService.WithImages(thread, [new FileAttachmentImage("Poster.png", "/data/m.png", 1_024)]);
|
||||
|
||||
Assert.That(thread.Blocks.Select(block => (block.Content as ContentText)?.FileAttachments.Count ?? 0), Is.All.Zero);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryOtherMessageStaysTheSame()
|
||||
{
|
||||
var thread = Conversation();
|
||||
|
||||
var request = BflConversationService.WithImages(thread, [new FileAttachmentImage("Poster.png", "/data/m.png", 1_024)]);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(request.Blocks, Has.Count.EqualTo(thread.Blocks.Count), "The tokens a provider reports would no longer belong to the chat.");
|
||||
Assert.That(request.Blocks[0], Is.Not.SameAs(thread.Blocks[0]));
|
||||
for (var index = 1; index < thread.Blocks.Count; index++)
|
||||
Assert.That(request.Blocks[index], Is.SameAs(thread.Blocks[index]), "The answer must stream into the chat itself.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WithoutImagesTheRequestIsTheChat()
|
||||
{
|
||||
var thread = Conversation();
|
||||
|
||||
Assert.That(BflConversationService.WithImages(thread, []), Is.SameAs(thread));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A conversation of a step: the hidden opening request, an answer, a message of the person, and the answer which
|
||||
/// streams in.
|
||||
/// </summary>
|
||||
private static ChatThread Conversation() => new()
|
||||
{
|
||||
ChatId = Guid.NewGuid(),
|
||||
Blocks =
|
||||
[
|
||||
new() { Role = ChatRole.USER, ContentType = ContentType.TEXT, Content = new ContentText { Text = "Eröffnung" }, HideFromUser = true },
|
||||
new() { Role = ChatRole.AI, ContentType = ContentType.TEXT, Content = new ContentText { Text = "Worum geht es?" } },
|
||||
new() { Role = ChatRole.USER, ContentType = ContentType.TEXT, Content = new ContentText { Text = "Ein Podium." } },
|
||||
new() { Role = ChatRole.AI, ContentType = ContentType.TEXT, Content = new ContentText { InitialRemoteWait = true } },
|
||||
],
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user