Moved the chats of the BeyondFacts Lab to the embedded chat of assistants (#1203)

app/Tests/BeyondFactsLab/BflChatRequestTests.cs

@@ -0,0 +1,165 @@
+using AIStudio.Assistants.BeyondFactsLab;
+using AIStudio.Chat;
+using AIStudio.Chat.Embedded;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.AssistantSessions;
+using AIStudio.Tools.ToolCallingSystem;
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.BeyondFactsLab;
+
+/// <summary>
+/// Checks what a request of a chat of the BeyondFacts Lab carries when it runs in the embedded chat, without asking any
+/// provider.
+/// </summary>
+[TestFixture]
+public sealed class BflChatRequestTests
+{
+    private const AIStudio.Tools.Components LAB = AIStudio.Tools.Components.BEYOND_FACTS_LAB_ASSISTANT;
+
+    private const string SYSTEM_PROMPT = "Der Prompt des Schritts.";
+
+    private string dataDirectory = string.Empty;
+    private SettingsManager settingsManager = null!;
+    private ServiceProvider serviceProvider = null!;
+    private IServiceProvider previousServiceProvider = null!;
+    private AIStudio.Settings.Provider provider = null!;
+
+    [SetUp]
+    public void CreateSettings()
+    {
+        this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-chat-requests-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(this.dataDirectory);
+
+        // The scheme which trusts Europe gives Mistral MEDIUM, the floor of the lab:
+        this.settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
+        this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_EUROPE;
+        this.previousServiceProvider = Program.SERVICE_PROVIDER;
+        this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider();
+        Program.SERVICE_PROVIDER = this.serviceProvider;
+
+        this.provider = new AIStudio.Settings.Provider(1, Guid.NewGuid().ToString(), "Instance 1", LLMProviders.MISTRAL, new("test-model", null));
+    }
+
+    [TearDown]
+    public void RestoreApplicationState()
+    {
+        Program.SERVICE_PROVIDER = this.previousServiceProvider;
+        this.serviceProvider.Dispose();
+        if (Directory.Exists(this.dataDirectory))
+            Directory.Delete(this.dataDirectory, true);
+    }
+
+    [Test]
+    public async Task AStepAsksWithThePromptOfTheLabAndReadsOnlyWebPagesGivenToIt()
+    {
+        var store = this.CreateStore();
+        var created = await store.CreateAsync("Panel");
+        var location = new BflStepThreadLocation(created.ProjectId, BflStep.FRAMEWORK);
+        ChatThread? sent = null;
+        var service = this.CreateService(thread => sent = thread);
+
+        await service.GetAsync(new BflChatStore(store, location), LAB, location.ChatId, () => BflThreadFactory.Create(location));
+        var isSent = await service.SendAsync(location.ChatId, this.Request(location, "Ich spreche auf einem Podium.", new(SYSTEM_PROMPT, [])));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(isSent, Is.True);
+            Assert.That(sent!.SystemPrompt, Is.EqualTo(SYSTEM_PROMPT));
+            Assert.That(sent.RuntimeSelectedToolIds, Is.EquivalentTo(new[] { ToolSelectionRules.READ_WEB_PAGE_TOOL_ID }));
+            Assert.That(sent.RuntimeToolsAreAssistantManaged, Is.True);
+            Assert.That(sent.RuntimeComponent, Is.EqualTo(LAB));
+            Assert.That(sent.DataSourceOptions.DisableDataSources, Is.True);
+            Assert.That(sent.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT));
+            Assert.That((await store.LoadThreadAsync(location))!.Blocks.Select(block => block.Role), Is.EqualTo(new[] { ChatRole.USER, ChatRole.AI }));
+        }
+    }
+
+    [Test]
+    public async Task TheOnboardingAsksWithoutTools()
+    {
+        var store = this.CreateStore();
+        var location = BflOnboardingThreadLocation.INSTANCE;
+        ChatThread? sent = null;
+        var service = this.CreateService(thread => sent = thread);
+
+        await service.GetAsync(new BflChatStore(store, location), LAB, location.ChatId, () => BflThreadFactory.Create(location));
+        await service.SendAsync(location.ChatId, this.Request(location, "Passt das Lab zu meinem Interview?", new(SYSTEM_PROMPT, [])));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(sent!.RuntimeSelectedToolIds, Is.Empty);
+            Assert.That(sent.RequiredOutboundDataRestriction, Is.EqualTo(OutboundDataRequirement.NONE));
+            Assert.That(File.Exists(Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, "onboarding", "thread.json")), Is.True);
+        }
+    }
+
+    [Test]
+    public async Task TheImagesOfTheMaterialGoAlongWithTheRequestOnly()
+    {
+        var store = this.CreateStore();
+        var created = await store.CreateAsync("Panel");
+        var location = new BflStepThreadLocation(created.ProjectId, BflStep.KNOWLEDGE);
+        var image = new FileAttachmentImage("Poster.png", Path.Combine(this.dataDirectory, "poster-copy.png"), 1_024);
+        ChatThread? sent = null;
+        var service = this.CreateService(thread => sent = thread);
+
+        await service.GetAsync(new BflChatStore(store, location), LAB, location.ChatId, () => BflThreadFactory.Create(location));
+        await service.SendAsync(location.ChatId, this.Request(location, BflConversationService.OPENING_REQUEST, new(SYSTEM_PROMPT, [image]), isHidden: true));
+
+        var stored = await store.LoadThreadAsync(location);
+        var storedFile = await File.ReadAllTextAsync(Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, created.ProjectId.ToString("D"), "steps", BflStep.KNOWLEDGE.FolderName, "thread.json"));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(sent!.Blocks[0].Content!.FileAttachments, Is.EqualTo(new FileAttachment[] { image }), "The image goes along with the first message.");
+            Assert.That(stored!.Blocks[0].Content!.FileAttachments, Is.Empty, "The stored chat never holds an image.");
+            Assert.That(stored.Blocks[0].HideFromUser, Is.True);
+            Assert.That(storedFile, Does.Not.Contain("poster-copy.png"));
+        }
+    }
+
+    [Test]
+    public async Task AChatStoredBeforeTheLabOfferedToolsIsRestrictedBeforeItsRequest()
+    {
+        var store = this.CreateStore();
+        var created = await store.CreateAsync("Panel");
+        var location = new BflStepThreadLocation(created.ProjectId, BflStep.STRATEGY);
+        await store.SaveThreadAsync(location, new ChatThread { ChatId = location.ChatId, Name = location.ChatName, Blocks = [] });
+        ChatThread? sent = null;
+        var service = this.CreateService(thread => sent = thread);
+
+        var conversation = await service.GetAsync(new BflChatStore(store, location), LAB, location.ChatId, () => BflThreadFactory.Create(location));
+        var restrictionBefore = conversation.Thread!.RequiredOutboundDataRestriction;
+        await service.SendAsync(location.ChatId, this.Request(location, "Wer sitzt mit auf dem Podium?", new(SYSTEM_PROMPT, [])));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(restrictionBefore, Is.EqualTo(OutboundDataRequirement.NONE));
+            Assert.That(sent!.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT));
+            Assert.That((await store.LoadThreadAsync(location))!.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT), "The chat stores the restriction.");
+        }
+    }
+
+    private EmbeddedChatRequest Request(BflThreadLocation location, string text, BflRequestMaterial requestMaterial, bool isHidden = false) => new()
+    {
+        Provider = this.provider,
+        Message = new ContentText { Text = text },
+        IsHidden = isHidden,
+        PrepareAsync = thread => Task.FromResult(BflConversationService.Prepare(location, this.provider, thread, requestMaterial))
+    };
+
+    private EmbeddedChatService CreateService(Action<ChatThread> sent) => new(this.settingsManager, new AssistantSessionService(MessageBus.INSTANCE), MessageBus.INSTANCE, NullLogger<EmbeddedChatService>.Instance, (_, answer, thread, _) =>
+    {
+        sent(thread);
+        answer.Text = "Worum geht es Ihnen?";
+        return Task.CompletedTask;
+    });
+
+    private BeyondFactsLabStore CreateStore() => new(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory });
+}

app/Tests/BeyondFactsLab/BflChatStoreTests.cs

@@ -0,0 +1,147 @@
+using AIStudio.Assistants.BeyondFactsLab;
+using AIStudio.Chat;
+using AIStudio.Chat.Embedded;
+using AIStudio.Settings;
+using AIStudio.Tools;
+using AIStudio.Tools.AssistantSessions;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.BeyondFactsLab;
+
+/// <summary>
+/// Checks that the embedded chats of the BeyondFacts Lab live where the lab keeps its chats, so that every chat stored
+/// before stays readable.
+/// </summary>
+[TestFixture]
+public sealed class BflChatStoreTests
+{
+    private string dataDirectory = string.Empty;
+
+    private string RootDirectory => Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME);
+
+    [SetUp]
+    public void CreateDataDirectory()
+    {
+        this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-chat-store-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(this.dataDirectory);
+    }
+
+    [TearDown]
+    public void DeleteDataDirectory()
+    {
+        if (Directory.Exists(this.dataDirectory))
+            Directory.Delete(this.dataDirectory, true);
+    }
+
+    [Test]
+    public async Task AChatStoredBeforeOpensInTheEmbeddedChatWithAllItsMessages()
+    {
+        var store = this.CreateStore();
+        var created = await store.CreateAsync("Panel");
+        var location = new BflStepThreadLocation(created.ProjectId, BflStep.KNOWLEDGE);
+        var thread = BflThreadFactory.Create(location);
+        AddMessage(thread, ChatRole.USER, BflConversationService.OPENING_REQUEST, isHidden: true);
+        AddMessage(thread, ChatRole.AI, "Worauf beruht Ihre Expertise?", isHidden: false);
+        await store.SaveThreadAsync(location, thread);
+
+        var conversation = await CreateService().GetAsync(new BflChatStore(this.CreateStore(), location), AIStudio.Tools.Components.BEYOND_FACTS_LAB_ASSISTANT, location.ChatId, () => BflThreadFactory.Create(location));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(conversation.IsAvailable, Is.True);
+            Assert.That(conversation.Thread!.ChatId, Is.EqualTo(location.ChatId));
+            Assert.That(conversation.Thread.Blocks.Select(block => block.HideFromUser), Is.EqualTo(new[] { true, false }), "The hidden request stays hidden.");
+            Assert.That(conversation.UserMessageCount, Is.Zero);
+            Assert.That(conversation.Thread.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT), "The chat keeps the restriction of its step.");
+        }
+    }
+
+    [Test]
+    public async Task AChatOfAStepIsStoredInTheFolderOfTheStep()
+    {
+        var store = this.CreateStore();
+        var created = await store.CreateAsync("Panel");
+        var location = new BflStepThreadLocation(created.ProjectId, BflStep.STRATEGY);
+        var thread = BflThreadFactory.Create(location);
+        AddMessage(thread, ChatRole.USER, "Wer sitzt mit auf dem Podium?", isHidden: false);
+
+        await new BflChatStore(store, location).SaveThreadAsync(thread);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(File.Exists(Path.Combine(this.RootDirectory, created.ProjectId.ToString("D"), "steps", BflStep.STRATEGY.FolderName, "thread.json")), Is.True);
+            Assert.That((await store.LoadThreadAsync(location))!.Blocks, Has.Count.EqualTo(1));
+            Assert.That(store.ReadAllThreadLocations(), Does.ContainKey(location.ChatId), "The cleanup of the provenance store finds the chat.");
+        }
+    }
+
+    [Test]
+    public async Task TheOnboardingIsStoredInItsOwnFolder()
+    {
+        var store = this.CreateStore();
+        var location = BflOnboardingThreadLocation.INSTANCE;
+        var thread = BflThreadFactory.Create(location);
+        AddMessage(thread, ChatRole.USER, "Passt das Lab zu meinem Interview?", isHidden: false);
+
+        await new BflChatStore(store, location).SaveThreadAsync(thread);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(File.Exists(Path.Combine(this.RootDirectory, "onboarding", "thread.json")), Is.True);
+            Assert.That(store.ReadAllThreadLocations(), Does.ContainKey(location.ChatId));
+        }
+    }
+
+    [Test]
+    public async Task AChatOfAnotherPlaceIsRefused()
+    {
+        var store = this.CreateStore();
+        var created = await store.CreateAsync("Panel");
+        var chatStore = new BflChatStore(store, new BflStepThreadLocation(created.ProjectId, BflStep.FRAMEWORK));
+        var otherLocation = new BflStepThreadLocation(created.ProjectId, BflStep.STRATEGY);
+
+        using (Assert.EnterMultipleScope())
+        {
+            await Assert.ThrowsAsync<ArgumentException>(async () => await chatStore.LoadThreadAsync(otherLocation.ChatId));
+            await Assert.ThrowsAsync<ArgumentException>(async () => await chatStore.SaveThreadAsync(BflThreadFactory.Create(otherLocation)));
+            await Assert.ThrowsAsync<ArgumentException>(async () => await chatStore.DeleteThreadAsync(otherLocation.ChatId));
+        }
+    }
+
+    [Test]
+    public async Task StartingOverDeletesTheFileOfTheChat()
+    {
+        var store = this.CreateStore();
+        var created = await store.CreateAsync("Panel");
+        var location = new BflStepThreadLocation(created.ProjectId, BflStep.FRAMEWORK);
+        var thread = BflThreadFactory.Create(location);
+        AddMessage(thread, ChatRole.USER, "Hallo", isHidden: false);
+        await store.SaveThreadAsync(location, thread);
+
+        var service = CreateService();
+        var conversation = await service.GetAsync(new BflChatStore(store, location), AIStudio.Tools.Components.BEYOND_FACTS_LAB_ASSISTANT, location.ChatId, () => BflThreadFactory.Create(location));
+        var reset = await service.ResetAsync(location.ChatId, () => BflThreadFactory.Create(location));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(reset, Is.True);
+            Assert.That(await store.LoadThreadAsync(location), Is.Null);
+            Assert.That(conversation.Thread!.Blocks, Is.Empty);
+            Assert.That(conversation.Thread.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT), "The new conversation starts restricted as well.");
+        }
+    }
+
+    private static void AddMessage(ChatThread thread, ChatRole role, string text, bool isHidden) => thread.Blocks.Add(new ContentBlock
+    {
+        Time = DateTimeOffset.Now,
+        ContentType = ContentType.TEXT,
+        Role = role,
+        Content = new ContentText { Text = text },
+        HideFromUser = isHidden
+    });
+
+    private static EmbeddedChatService CreateService() => new(new SettingsManager(NullLogger<SettingsManager>.Instance, null!), new AssistantSessionService(MessageBus.INSTANCE), MessageBus.INSTANCE, NullLogger<EmbeddedChatService>.Instance, (_, _, _, _) => Task.CompletedTask);
+
+    private BeyondFactsLabStore CreateStore() => new(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory });
+}

app/Tests/BeyondFactsLab/BflMaterialBudgetTests.cs

@@ -1,4 +1,5 @@
 using AIStudio.Assistants.BeyondFactsLab;
+using AIStudio.Chat.Embedded;
 using AIStudio.Models;
 
 namespace AIStudio.Tests.BeyondFactsLab;
@@ -14,7 +15,7 @@ public sealed class BflMaterialBudgetTests
     {
         var characters = BflMaterialBudget.CharactersFor(ContextWindow.Of(128_000), 90_000);
 
-        Assert.That(characters, Is.EqualTo(((128_000 - BflTokenBudget.RESERVE_TOKENS) * 4 - 90_000) / 2));
+        Assert.That(characters, Is.EqualTo(((128_000 - EmbeddedChatTokenBudget.RESERVE_TOKENS) * 4 - 90_000) / 2));
     }
 
     [Test]
@@ -22,7 +23,7 @@ public sealed class BflMaterialBudgetTests
     {
         var characters = BflMaterialBudget.CharactersFor(ContextWindow.Of(128_000), 90_000, sentImages: 2);
 
-        Assert.That(characters, Is.EqualTo(((128_000 - BflTokenBudget.RESERVE_TOKENS - 2 * BflMaterialBudget.TOKENS_PER_IMAGE) * 4 - 90_000) / 2));
+        Assert.That(characters, Is.EqualTo(((128_000 - EmbeddedChatTokenBudget.RESERVE_TOKENS - 2 * BflMaterialBudget.TOKENS_PER_IMAGE) * 4 - 90_000) / 2));
     }
 
     [Test]

app/Tests/BeyondFactsLab/BflPromptOverridesTests.cs

@@ -198,7 +198,7 @@ public sealed class BflPromptOverridesTests
     {
         var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
         var store = new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.directory });
-        var service = new BflConversationService(store, settingsManager, null!, null!, null!, NullLogger<BflConversationService>.Instance);
+        var service = new BflConversationService(store, null!, settingsManager, null!, null!, NullLogger<BflConversationService>.Instance);
         await BflPromptOverrides.CopyBuiltInAsync(store.PromptOverridesDirectory);
 
         settingsManager.ConfigurationData.BeyondFactsLab.AllowPromptOverrides = true;

app/Tests/BeyondFactsLab/BflTokenBudgetTests.cs

@@ -1,49 +0,0 @@
-using AIStudio.Assistants.BeyondFactsLab;
-using AIStudio.Chat;
-using AIStudio.Models;
-
-namespace AIStudio.Tests.BeyondFactsLab;
-
-/// <summary>
-/// Checks when the BeyondFacts Lab warns about a full context, and when it stops sending.
-/// </summary>
-[TestFixture]
-public sealed class BflTokenBudgetTests
-{
-    private const int WINDOW = 32_000;
-
-    [Test]
-    public void ACountWhichFailedTellsNothing()
-    {
-        Assert.That(BflTokenBudget.Of(ConversationTokens.UNAVAILABLE), Is.EqualTo(BflTokenBudgetState.UNKNOWN));
-    }
-
-    [Test]
-    public void AModelWithoutAStatementAboutItsWindowTellsNothing()
-    {
-        var tokens = new ConversationTokens { IsKnown = true, HistoryTokens = 500_000, Window = ContextWindow.UNKNOWN };
-
-        Assert.That(BflTokenBudget.Of(tokens), Is.EqualTo(BflTokenBudgetState.UNKNOWN), "The lab cannot know when such a model is full, so it never stops sending.");
-    }
-
-    [TestCase(0, BflTokenBudgetState.OK)]
-    [TestCase(25_599, BflTokenBudgetState.OK)]
-    [TestCase(25_600, BflTokenBudgetState.WARNING)]
-    [TestCase(WINDOW - BflTokenBudget.RESERVE_TOKENS, BflTokenBudgetState.WARNING)]
-    [TestCase(WINDOW - BflTokenBudget.RESERVE_TOKENS + 1, BflTokenBudgetState.FULL)]
-    [TestCase(WINDOW * 2, BflTokenBudgetState.FULL)]
-    public void TheShareOfTheWindowDecides(int historyTokens, BflTokenBudgetState expected)
-    {
-        var tokens = new ConversationTokens { IsKnown = true, HistoryTokens = historyTokens, Window = ContextWindow.Of(WINDOW) };
-
-        Assert.That(BflTokenBudget.Of(tokens), Is.EqualTo(expected));
-    }
-
-    [Test]
-    public void TheMessageBeingWrittenCountsAsWell()
-    {
-        var tokens = new ConversationTokens { IsKnown = true, HistoryTokens = 20_000, DraftTokens = 6_000, Window = ContextWindow.Of(WINDOW) };
-
-        Assert.That(BflTokenBudget.Of(tokens), Is.EqualTo(BflTokenBudgetState.WARNING));
-    }
-}