using AIStudio.Assistants.BeyondFactsLab; using AIStudio.Chat; using AIStudio.Chat.Embedded; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools; using AIStudio.Tools.AssistantSessions; using AIStudio.Tools.ToolCallingSystem; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; namespace AIStudio.Tests.BeyondFactsLab; /// /// Checks what a request of a chat of the BeyondFacts Lab carries when it runs in the embedded chat, without asking any /// provider. /// [TestFixture] public sealed class BflChatRequestTests { private const AIStudio.Tools.Components LAB = AIStudio.Tools.Components.BEYOND_FACTS_LAB_ASSISTANT; private const string SYSTEM_PROMPT = "Der Prompt des Schritts."; private string dataDirectory = string.Empty; private SettingsManager settingsManager = null!; private ServiceProvider serviceProvider = null!; private IServiceProvider previousServiceProvider = null!; private AIStudio.Settings.Provider provider = null!; [SetUp] public void CreateSettings() { this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-chat-requests-{Guid.NewGuid():N}"); Directory.CreateDirectory(this.dataDirectory); // The scheme which trusts Europe gives Mistral MEDIUM, the floor of the lab: this.settingsManager = new SettingsManager(NullLogger.Instance, null!); this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_EUROPE; this.previousServiceProvider = Program.SERVICE_PROVIDER; this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider(); Program.SERVICE_PROVIDER = this.serviceProvider; this.provider = new AIStudio.Settings.Provider(1, Guid.NewGuid().ToString(), "Instance 1", LLMProviders.MISTRAL, new("test-model", null)); } [TearDown] public void RestoreApplicationState() { Program.SERVICE_PROVIDER = this.previousServiceProvider; this.serviceProvider.Dispose(); if (Directory.Exists(this.dataDirectory)) Directory.Delete(this.dataDirectory, true); } [Test] public async Task AStepAsksWithThePromptOfTheLabAndReadsOnlyWebPagesGivenToIt() { var store = this.CreateStore(); var created = await store.CreateAsync("Panel"); var location = new BflStepThreadLocation(created.ProjectId, BflStep.FRAMEWORK); ChatThread? sent = null; var service = this.CreateService(thread => sent = thread); await service.GetAsync(new BflChatStore(store, location), LAB, location.ChatId, () => BflThreadFactory.Create(location)); var isSent = await service.SendAsync(location.ChatId, this.Request(location, "Ich spreche auf einem Podium.", new(SYSTEM_PROMPT, []))); using (Assert.EnterMultipleScope()) { Assert.That(isSent, Is.True); Assert.That(sent!.SystemPrompt, Is.EqualTo(SYSTEM_PROMPT)); Assert.That(sent.RuntimeSelectedToolIds, Is.EquivalentTo(new[] { ToolSelectionRules.READ_WEB_PAGE_TOOL_ID })); Assert.That(sent.RuntimeToolsAreAssistantManaged, Is.True); Assert.That(sent.RuntimeComponent, Is.EqualTo(LAB)); Assert.That(sent.DataSourceOptions.DisableDataSources, Is.True); Assert.That(sent.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT)); Assert.That((await store.LoadThreadAsync(location))!.Blocks.Select(block => block.Role), Is.EqualTo(new[] { ChatRole.USER, ChatRole.AI })); } } [Test] public async Task TheOnboardingAsksWithoutTools() { var store = this.CreateStore(); var location = BflOnboardingThreadLocation.INSTANCE; ChatThread? sent = null; var service = this.CreateService(thread => sent = thread); await service.GetAsync(new BflChatStore(store, location), LAB, location.ChatId, () => BflThreadFactory.Create(location)); await service.SendAsync(location.ChatId, this.Request(location, "Passt das Lab zu meinem Interview?", new(SYSTEM_PROMPT, []))); using (Assert.EnterMultipleScope()) { Assert.That(sent!.RuntimeSelectedToolIds, Is.Empty); Assert.That(sent.RequiredOutboundDataRestriction, Is.EqualTo(OutboundDataRequirement.NONE)); Assert.That(File.Exists(Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, "onboarding", "thread.json")), Is.True); } } [Test] public async Task TheImagesOfTheMaterialGoAlongWithTheRequestOnly() { var store = this.CreateStore(); var created = await store.CreateAsync("Panel"); var location = new BflStepThreadLocation(created.ProjectId, BflStep.KNOWLEDGE); var image = new FileAttachmentImage("Poster.png", Path.Combine(this.dataDirectory, "poster-copy.png"), 1_024); ChatThread? sent = null; var service = this.CreateService(thread => sent = thread); await service.GetAsync(new BflChatStore(store, location), LAB, location.ChatId, () => BflThreadFactory.Create(location)); await service.SendAsync(location.ChatId, this.Request(location, BflConversationService.OPENING_REQUEST, new(SYSTEM_PROMPT, [image]), isHidden: true)); var stored = await store.LoadThreadAsync(location); var storedFile = await File.ReadAllTextAsync(Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, created.ProjectId.ToString("D"), "steps", BflStep.KNOWLEDGE.FolderName, "thread.json")); using (Assert.EnterMultipleScope()) { Assert.That(sent!.Blocks[0].Content!.FileAttachments, Is.EqualTo(new FileAttachment[] { image }), "The image goes along with the first message."); Assert.That(stored!.Blocks[0].Content!.FileAttachments, Is.Empty, "The stored chat never holds an image."); Assert.That(stored.Blocks[0].HideFromUser, Is.True); Assert.That(storedFile, Does.Not.Contain("poster-copy.png")); } } [Test] public async Task AChatStoredBeforeTheLabOfferedToolsIsRestrictedBeforeItsRequest() { var store = this.CreateStore(); var created = await store.CreateAsync("Panel"); var location = new BflStepThreadLocation(created.ProjectId, BflStep.STRATEGY); await store.SaveThreadAsync(location, new ChatThread { ChatId = location.ChatId, Name = location.ChatName, Blocks = [] }); ChatThread? sent = null; var service = this.CreateService(thread => sent = thread); var conversation = await service.GetAsync(new BflChatStore(store, location), LAB, location.ChatId, () => BflThreadFactory.Create(location)); var restrictionBefore = conversation.Thread!.RequiredOutboundDataRestriction; await service.SendAsync(location.ChatId, this.Request(location, "Wer sitzt mit auf dem Podium?", new(SYSTEM_PROMPT, []))); using (Assert.EnterMultipleScope()) { Assert.That(restrictionBefore, Is.EqualTo(OutboundDataRequirement.NONE)); Assert.That(sent!.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT)); Assert.That((await store.LoadThreadAsync(location))!.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT), "The chat stores the restriction."); } } private EmbeddedChatRequest Request(BflThreadLocation location, string text, BflRequestMaterial requestMaterial, bool isHidden = false) => new() { Provider = this.provider, Message = new ContentText { Text = text }, IsHidden = isHidden, PrepareAsync = thread => Task.FromResult(BflConversationService.Prepare(location, this.provider, thread, requestMaterial)) }; private EmbeddedChatService CreateService(Action sent) => new(this.settingsManager, new AssistantSessionService(MessageBus.INSTANCE), MessageBus.INSTANCE, NullLogger.Instance, (_, answer, thread, _) => { sent(thread); answer.Text = "Worum geht es Ihnen?"; return Task.CompletedTask; }); private BeyondFactsLabStore CreateStore() => new(NullLogger.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory }); }