Added research on the web to the BeyondFacts Lab (#1186)

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-11-10 18:17:44 +01:00
1 parent e5c0931a60
commit 9a676101f7
66 files changed
+3609 -70

No files matched your search

@@ -0,0 +1,118 @@
using AIStudio.Assistants.BeyondFactsLab;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.BeyondFactsLab;
/// <summary>
/// Checks how the BeyondFacts Lab stores the research of a preparation.
/// </summary>
[TestFixture]
public sealed class BeyondFactsLabStoreResearchTests
{
private string dataDirectory = string.Empty;
[SetUp]
public void CreateDataDirectory()
{
this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.dataDirectory);
}
[TearDown]
public void DeleteDataDirectory()
{
if (Directory.Exists(this.dataDirectory))
Directory.Delete(this.dataDirectory, true);
}
[Test]
public async Task APreparationWithoutResearchHasNone()
{
var store = this.CreateStore();
var created = await store.CreateAsync("Panel");
using (Assert.EnterMultipleScope())
{
Assert.That(await store.LoadResearchAsync(created.ProjectId, BflResearchTopic.EVENT), Is.Null);
Assert.That(await store.LoadAllResearchAsync(created.ProjectId), Is.Empty);
}
}
[Test]
public async Task AResearchIsStoredForItsTopic()
{
var store = this.CreateStore();
var created = await store.CreateAsync("Panel");
var research = new BflResearch
{
Text = "## Teilnehmende\n\n### Person A",
Details = [new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Klimapolitik" }],
SearchQueries = ["Podium Klimapolitik"],
Pages = [new() { Title = "Programm", Url = "https://example.org/programm" }],
CreatedAtUtc = new DateTimeOffset(2026, 11, 10, 14, 0, 0, TimeSpan.Zero),
};
var changed = await store.SaveResearchAsync(created.ProjectId, BflResearchTopic.PARTICIPANTS, research);
var loaded = await this.CreateStore().LoadResearchAsync(created.ProjectId, BflResearchTopic.PARTICIPANTS);
var all = await this.CreateStore().LoadAllResearchAsync(created.ProjectId);
using (Assert.EnterMultipleScope())
{
Assert.That(loaded, Is.Not.Null);
Assert.That(loaded!.Text, Is.EqualTo(research.Text));
Assert.That(loaded.Details.Single().Value, Is.EqualTo("Klimapolitik"));
Assert.That(loaded.SearchQueries, Is.EqualTo(new[] { "Podium Klimapolitik" }));
Assert.That(loaded.Pages.Single().Url, Is.EqualTo("https://example.org/programm"));
Assert.That(loaded.CreatedAtUtc, Is.EqualTo(research.CreatedAtUtc));
Assert.That(all.Keys, Is.EquivalentTo(new[] { BflResearchTopic.PARTICIPANTS }));
Assert.That(changed.ModifiedAtUtc, Is.GreaterThanOrEqualTo(created.ModifiedAtUtc));
Assert.That(File.Exists(Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, created.ProjectId.ToString("D"), "steps", "05-strategy", "research", "participants.json")), Is.True);
}
}
[Test]
public async Task ADamagedResearchStaysAsItIs()
{
var store = this.CreateStore();
var created = await store.CreateAsync("Panel");
var path = await this.WriteResearchFileAsync(created.ProjectId, "event.json", "{ not json");
using (Assert.EnterMultipleScope())
{
await Assert.ThrowsAsync<InvalidDataException>(async () => await store.LoadResearchAsync(created.ProjectId, BflResearchTopic.EVENT));
await Assert.ThrowsAsync<InvalidDataException>(async () => await store.SaveResearchAsync(created.ProjectId, BflResearchTopic.EVENT, new BflResearch { Text = "Neu" }));
Assert.That(await store.LoadAllResearchAsync(created.ProjectId), Is.Empty);
Assert.That(await File.ReadAllTextAsync(path), Is.EqualTo("{ not json"));
}
}
[Test]
public async Task AResearchOfANewerVersionStaysAsItIs()
{
var store = this.CreateStore();
var created = await store.CreateAsync("Panel");
await this.WriteResearchFileAsync(created.ProjectId, "debate.json", $$"""{ "schemaVersion": {{BflResearch.CURRENT_SCHEMA_VERSION + 1}}, "text": "Neu" }""");
await Assert.ThrowsAsync<InvalidDataException>(async () => await store.LoadResearchAsync(created.ProjectId, BflResearchTopic.DEBATE));
}
[Test]
public async Task ResearchForAMissingPreparationIsNotStored()
{
var store = this.CreateStore();
await Assert.ThrowsAsync<FileNotFoundException>(async () => await store.SaveResearchAsync(Guid.NewGuid(), BflResearchTopic.EVENT, new BflResearch { Text = "Text" }));
}
private async Task<string> WriteResearchFileAsync(Guid projectId, string fileName, string content)
{
var directory = Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, projectId.ToString("D"), "steps", "05-strategy", "research");
Directory.CreateDirectory(directory);
var path = Path.Combine(directory, fileName);
await File.WriteAllTextAsync(path, content);
return path;
}
private BeyondFactsLabStore CreateStore() => new(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory });
}
@@ -0,0 +1,131 @@
using AIStudio.Assistants.BeyondFactsLab;
using AIStudio.Chat;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ToolCallingSystem;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.BeyondFactsLab;
/// <summary>
/// Checks the tools of the chats of the BeyondFacts Lab, and that the chat of a step never searches the web.
/// </summary>
/// <remarks>
/// That the level ONLY_LINKS_FROM_CHAT keeps a web search from running, while Read Web Page still reads the addresses
/// given to the model, is checked by ToolOutboundDataTests for every chat alike.
/// </remarks>
[TestFixture]
public sealed class BflChatToolsTests
{
private static readonly Guid PROJECT_ID = Guid.Parse("6f1c2b0e-3a51-4c3e-9d2f-0b7a4e5c8d91");
private string dataDirectory = string.Empty;
[SetUp]
public void CreateDataDirectory()
{
this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.dataDirectory);
}
[TearDown]
public void DeleteDataDirectory()
{
if (Directory.Exists(this.dataDirectory))
Directory.Delete(this.dataDirectory, true);
}
[Test]
public void TheChatOfEveryStepReadsWebPagesButNeverSearches()
{
using (Assert.EnterMultipleScope())
{
foreach (var step in BflStep.All)
{
var tools = BflChatTools.For(new BflStepThreadLocation(PROJECT_ID, step));
Assert.That(tools, Is.EquivalentTo(new[] { ToolSelectionRules.READ_WEB_PAGE_TOOL_ID }), step.ToString());
Assert.That(tools, Does.Not.Contain(ToolSelectionRules.WEB_SEARCH_TOOL_ID), step.ToString());
}
}
}
[Test]
public void TheOnboardingHasNoTools()
{
Assert.That(BflChatTools.For(BflOnboardingThreadLocation.INSTANCE), Is.Empty);
}
[Test]
public void TheChatOfAStepReadsOnlyTheAddressesGivenToItFromTheStart()
{
var stepChat = BflThreadFactory.Create(new BflStepThreadLocation(PROJECT_ID, BflStep.STRATEGY));
var onboarding = BflThreadFactory.Create(BflOnboardingThreadLocation.INSTANCE);
using (Assert.EnterMultipleScope())
{
Assert.That(stepChat.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT));
Assert.That(BflChatTools.STEP_REQUIREMENT.Restriction, Is.EqualTo(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT));
Assert.That(BflChatTools.STEP_REQUIREMENT.Origin, Is.EqualTo(OutboundDataOrigin.BEYOND_FACTS_LAB));
Assert.That(onboarding.RequiredOutboundDataRestriction, Is.EqualTo(OutboundDataRequirement.NONE));
}
}
[Test]
public void AnEarlierChatGetsTheRestrictionAndAStricterOneStays()
{
var earlier = new ChatThread();
var stricter = new ChatThread();
var strictest = new OutboundDataRequirement(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, OutboundDataOrigin.INTERNAL_WEB_PAGE, "intranet.example.org");
stricter.RequireOutboundDataRestriction(strictest);
BflChatTools.Restrict(earlier, new BflStepThreadLocation(PROJECT_ID, BflStep.FRAMEWORK));
BflChatTools.Restrict(stricter, new BflStepThreadLocation(PROJECT_ID, BflStep.FRAMEWORK));
using (Assert.EnterMultipleScope())
{
Assert.That(earlier.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT));
Assert.That(stricter.RequiredOutboundDataRestriction, Is.EqualTo(strictest));
}
}
[Test]
public async Task TheRestrictionIsStoredWithTheChat()
{
var store = new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory });
var created = await store.CreateAsync("Panel");
var location = new BflStepThreadLocation(created.ProjectId, BflStep.KNOWLEDGE);
await store.SaveThreadAsync(location, BflThreadFactory.Create(location));
var loaded = await new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory }).LoadThreadAsync(location);
Assert.That(loaded!.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT));
}
[Test]
public void AToolKeptBackNamesTheLab()
{
var message = BflChatTools.STEP_REQUIREMENT.GetToolBlockedMessage(new Data());
using (Assert.EnterMultipleScope())
{
Assert.That(message, Does.Contain("BeyondFacts Lab"));
Assert.That(message, Is.Not.EqualTo(new OutboundDataRequirement(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, (OutboundDataOrigin)999, string.Empty).GetToolBlockedMessage(new Data())));
}
}
[Test]
public void ARequestOfAStepOffersTheToolsOfItsPlace()
{
var location = new BflStepThreadLocation(PROJECT_ID, BflStep.TEST_SITUATION);
var thread = BflThreadFactory.Create(location);
var provider = new AIStudio.Settings.Provider(1, "provider-1", "Test", AIStudio.Provider.LLMProviders.SELF_HOSTED, new AIStudio.Provider.Model("model", null));
BflThreadFactory.PrepareForRequest(thread, provider, "Prompt", BflChatTools.For(location));
using (Assert.EnterMultipleScope())
{
Assert.That(thread.RuntimeSelectedToolIds, Is.EquivalentTo(new[] { ToolSelectionRules.READ_WEB_PAGE_TOOL_ID }));
Assert.That(thread.RuntimeToolsAreAssistantManaged, Is.True);
}
}
}
@@ -0,0 +1,76 @@
using AIStudio.Assistants.BeyondFactsLab;
namespace AIStudio.Tests.BeyondFactsLab;
/// <summary>
/// Checks when the BeyondFacts Lab takes an event for not public, and when a suggestion about it keeps the research
/// from starting on its own.
/// </summary>
[TestFixture]
public sealed class BflEventVisibilityTests
{
[TestCase(null, false, TestName = "AnEmptyFieldLetsTheLabResearch")]
[TestCase(BflFieldCatalog.EVENT_PUBLIC, false, TestName = "APublicEventIsResearched")]
[TestCase(BflFieldCatalog.EVENT_NOT_PUBLIC, true, TestName = "AnEventWhichIsNotPublicIsNeverResearched")]
[TestCase("event_unknown", false, TestName = "AnOptionThisVersionDoesNotKnowLetsTheLabResearch")]
public void OnlyTheOptionNotPublicStopsTheResearch(string? visibility, bool isNotPublic)
{
var forms = Forms(visibility);
using (Assert.EnterMultipleScope())
{
Assert.That(BflEventVisibility.IsNotPublic(forms), Is.EqualTo(isNotPublic));
Assert.That(BflEventVisibility.IsNotPublic(forms[BflStep.FRAMEWORK]), Is.EqualTo(isNotPublic));
}
}
[Test]
public void APreparationWithoutEventDetailsIsResearchedAsBefore()
{
Assert.That(BflEventVisibility.IsNotPublic(new Dictionary<BflStep, BflStepFormData>()), Is.False);
}
[TestCase(BflFieldCatalog.EVENT_PUBLIC)]
[TestCase(BflFieldCatalog.EVENT_NOT_PUBLIC)]
public void ASuggestionWaitsWhileThePersonHasNotDecided(string suggestion)
{
Assert.That(BflEventVisibility.IsSuggestionPending(WorkStates(suggestion), Forms(null)), Is.True);
}
[Test]
public void ASuggestionNoLongerWaitsOnceThePersonDecided()
{
Assert.That(BflEventVisibility.IsSuggestionPending(WorkStates(BflFieldCatalog.EVENT_NOT_PUBLIC), Forms(BflFieldCatalog.EVENT_PUBLIC)), Is.False);
}
[Test]
public void WithoutASuggestionNothingWaits()
{
using (Assert.EnterMultipleScope())
{
Assert.That(BflEventVisibility.IsSuggestionPending(new Dictionary<BflStep, BflWorkState>(), Forms(null)), Is.False);
Assert.That(BflEventVisibility.IsSuggestionPending(new Dictionary<BflStep, BflWorkState> { [BflStep.FRAMEWORK] = new() }, Forms(null)), Is.False);
}
}
[Test]
public void ASuggestionWhichThisVersionDoesNotShowKeepsNothingWaiting()
{
Assert.That(BflEventVisibility.IsSuggestionPending(WorkStates("event_unknown"), Forms(null)), Is.False);
}
private static Dictionary<BflStep, BflStepFormData> Forms(string? visibility)
{
var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData());
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_TOPIC, "Klimapolitik");
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_VISIBILITY, visibility);
return forms;
}
private static Dictionary<BflStep, BflWorkState> WorkStates(string suggestion)
{
var workState = new BflWorkState();
workState.FieldSuggestions[BflFieldCatalog.EVENT_VISIBILITY] = suggestion;
return new Dictionary<BflStep, BflWorkState> { [BflStep.FRAMEWORK] = workState };
}
}
@@ -35,7 +35,7 @@ public sealed class BflFieldCatalogTests
{
Assert.That(BflFieldCatalog.FIELDS.Select(field => field.Id), Is.EqualTo(new[]
{
"format", "event_topic", "audience", "contribution_topic", "event_date", "event_location", "event_duration", "contribution_duration", "other_participants", "event_link",
"format", "event_topic", "audience", "event_visibility", "contribution_topic", "event_date", "event_location", "event_duration", "contribution_duration", "other_participants", "event_link",
"contribution_summary", "core_theses", "uncertainties", "material_notes",
"goals", "goal_directions",
"values_and_concerns", "personal_concern", "other_perspectives",
@@ -50,6 +50,7 @@ public sealed class BflFieldCatalogTests
{
Assert.That(BflFieldCatalog.FIELDS.SelectMany(field => field.Options).Select(option => option.Id), Is.EqualTo(new[]
{
"event_public", "event_not_public",
"goal_inform", "goal_moderate", "goal_take_position", "goal_raise_awareness", "goal_show_options", "goal_influence_policy",
"questions_mixed", "questions_scientific_criticism", "questions_emotional_criticism", "questions_manipulative", "questions_political_instrumentalization",
"difficulty_moderate", "difficulty_demanding", "difficulty_very_demanding",
@@ -164,7 +165,21 @@ public sealed class BflFieldCatalogTests
Assert.That(keyDetails, Is.EquivalentTo(new[]
{
BflFieldCatalog.FORMAT, BflFieldCatalog.EVENT_TOPIC, BflFieldCatalog.AUDIENCE, BflFieldCatalog.CONTRIBUTION_TOPIC, BflFieldCatalog.CONTRIBUTION_SUMMARY,
BflFieldCatalog.FORMAT, BflFieldCatalog.EVENT_TOPIC, BflFieldCatalog.AUDIENCE, BflFieldCatalog.EVENT_VISIBILITY, BflFieldCatalog.CONTRIBUTION_TOPIC, BflFieldCatalog.CONTRIBUTION_SUMMARY,
}));
}
[Test]
public void WhetherTheEventIsPublicIsASingleChoiceWhichNeverReachesTheResearch()
{
var visibility = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.EVENT_VISIBILITY);
using (Assert.EnterMultipleScope())
{
Assert.That(visibility.Step, Is.EqualTo(BflStep.FRAMEWORK));
Assert.That(visibility.Kind, Is.EqualTo(BflFieldKind.CHOICE));
Assert.That(visibility.Options.Select(option => option.Id), Is.EqualTo(new[] { BflFieldCatalog.EVENT_PUBLIC, BflFieldCatalog.EVENT_NOT_PUBLIC }));
Assert.That(visibility.IsSharedWithResearch, Is.False, "The field decides about the research; it is no detail about the event.");
}
}
}
@@ -20,7 +20,7 @@ public sealed class BflMarkdownSectionsTests
"KNOWLEDGE/ESTABLISHED", "KNOWLEDGE/UNCERTAINTIES", "KNOWLEDGE/WEAK_POINTS", "KNOWLEDGE/CONTROVERSIES", "KNOWLEDGE/EXPERTISE_LIMITS", "KNOWLEDGE/KEY_MESSAGES", "KNOWLEDGE/OPEN_POINTS",
"GOALS_AND_ROLE/GOAL", "GOALS_AND_ROLE/ROLE", "GOALS_AND_ROLE/REASONS", "GOALS_AND_ROLE/CONSEQUENCES", "GOALS_AND_ROLE/OPEN_POINTS",
"VALUES_AND_PERSPECTIVES/VALUES", "VALUES_AND_PERSPECTIVES/VALUE_CONFLICTS", "VALUES_AND_PERSPECTIVES/PERSONAL_INVOLVEMENT", "VALUES_AND_PERSPECTIVES/PERSPECTIVES", "VALUES_AND_PERSPECTIVES/CONSEQUENCES", "VALUES_AND_PERSPECTIVES/OPEN_POINTS",
"STRATEGY/KEY_MESSAGES", "STRATEGY/STRUCTURE", "STRATEGY/LANGUAGE", "STRATEGY/UNCERTAINTY", "STRATEGY/FRAMING", "STRATEGY/COUNTERARGUMENTS", "STRATEGY/WORRIES", "STRATEGY/OPEN_POINTS",
"STRATEGY/KEY_MESSAGES", "STRATEGY/STRUCTURE", "STRATEGY/LANGUAGE", "STRATEGY/UNCERTAINTY", "STRATEGY/FRAMING", "STRATEGY/PARTICIPANTS", "STRATEGY/COUNTERARGUMENTS", "STRATEGY/WORRIES", "STRATEGY/OPEN_POINTS",
"TEST_SITUATION/STRENGTHS", "TEST_SITUATION/PRACTICE", "TEST_SITUATION/HARD_QUESTIONS", "TEST_SITUATION/POSITION", "TEST_SITUATION/OPEN_POINTS",
"RESULTS/CHANGED", "RESULTS/KEPT", "RESULTS/LEFT_OPEN", "RESULTS/NEXT_STEPS",
}));
@@ -199,18 +199,59 @@ public sealed class BflMarkdownSectionsTests
}
[Test]
public void OnlyEmptyTextFieldsWhichAreNotPersonalGetSuggestions()
public void OnlyEmptyFieldsWithATextOrASingleChoiceWhichAreNotPersonalGetSuggestions()
{
var goals = BflMarkdownSections.SuggestableFields(BflStep.GOALS_AND_ROLE, new BflStepFormData()).Select(field => field.Id);
var values = BflMarkdownSections.SuggestableFields(BflStep.VALUES_AND_PERSPECTIVES, new BflStepFormData()).Select(field => field.Id);
var test = BflMarkdownSections.SuggestableFields(BflStep.TEST_SITUATION, new BflStepFormData()).Select(field => field.Id);
using (Assert.EnterMultipleScope())
{
Assert.That(goals, Is.EqualTo(new[] { BflFieldCatalog.GOALS }), "A field with several choices gets no suggestion.");
Assert.That(values, Is.EqualTo(new[] { BflFieldCatalog.VALUES_AND_CONCERNS, BflFieldCatalog.OTHER_PERSPECTIVES }));
Assert.That(test, Is.EqualTo(new[] { BflFieldCatalog.TEST_NOTES }));
Assert.That(test, Is.EqualTo(new[] { BflFieldCatalog.QUESTION_TYPE, BflFieldCatalog.DIFFICULTY, BflFieldCatalog.TEST_NOTES }));
}
}
[Test]
public void ASuggestionForAChoiceNamesItsOptionByLabelOrIdAndKeepsTheId()
{
var label = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.DIFFICULTY).Options.Single(option => option.Id == BflFieldCatalog.DIFFICULTY_DEMANDING).Label;
string[] suggestions = [label, label.ToUpperInvariant(), $"„{label}“", $"\"{label}\"", $"**{label}**", $"{label}.", BflFieldCatalog.DIFFICULTY_DEMANDING];
using (Assert.EnterMultipleScope())
{
foreach (var suggestion in suggestions)
{
var answer = BflMarkdownSections.Read($"## [FIELD:difficulty]\n{suggestion}", BflStep.TEST_SITUATION, new BflStepFormData());
Assert.That(answer.FieldSuggestions, Is.EqualTo(new Dictionary<string, string> { [BflFieldCatalog.DIFFICULTY] = BflFieldCatalog.DIFFICULTY_DEMANDING }), suggestion);
}
}
}
[TestCase("## [FIELD:difficulty]\nSomewhat demanding", TestName = "ASuggestionWhichNamesNoOptionIsLeftOut")]
[TestCase("## [FIELD:difficulty]\ndifficulty_moderate, difficulty_demanding", TestName = "ASuggestionWithSeveralOptionsIsLeftOut")]
[TestCase("## [FIELD:goal_directions]\ngoal_inform", TestName = "AFieldWithSeveralChoicesGetsNoSuggestion")]
public void ASuggestionForAChoiceWhichDoesNotFitIsLeftOut(string suggestion)
{
var step = suggestion.Contains("goal_", StringComparison.Ordinal) ? BflStep.GOALS_AND_ROLE : BflStep.TEST_SITUATION;
var answer = BflMarkdownSections.Read(suggestion, step, new BflStepFormData());
Assert.That(answer.FieldSuggestions, Is.Empty);
}
[Test]
public void AChoiceWhichHoldsAnOptionGetsNoSuggestion()
{
var form = new BflStepFormData();
form.Set(BflFieldCatalog.DIFFICULTY, BflFieldCatalog.DIFFICULTY_MODERATE);
var answer = BflMarkdownSections.Read($"## [FIELD:difficulty]\n{BflFieldCatalog.DIFFICULTY_DEMANDING}", BflStep.TEST_SITUATION, form);
Assert.That(answer.FieldSuggestions, Is.Empty);
}
private static BflWorkStateAnswer Read(string answer) => BflMarkdownSections.Read(answer, BflStep.GOALS_AND_ROLE, new BflStepFormData());
private static string FrameworkWithSuggestions(string suggestions) => $"## [FORMAT]\n## [AUDIENCE]\n## [SPEAKING_TIME]\n## [PARTICIPANTS]\n## [CONFLICTS]\n## [OPEN_POINTS]\n{suggestions}";
@@ -24,7 +24,7 @@ public sealed class BflMissingDetailsTests
var missing = BflMissingDetails.Of(BflStep.FRAMEWORK, form);
Assert.That(missing.Select(field => field.Id), Is.EqualTo(new[] { BflFieldCatalog.FORMAT, BflFieldCatalog.AUDIENCE, BflFieldCatalog.CONTRIBUTION_TOPIC }));
Assert.That(missing.Select(field => field.Id), Is.EqualTo(new[] { BflFieldCatalog.FORMAT, BflFieldCatalog.AUDIENCE, BflFieldCatalog.EVENT_VISIBILITY, BflFieldCatalog.CONTRIBUTION_TOPIC }));
}
[Test]
@@ -69,6 +69,7 @@ public sealed class BflPromptLibraryTests
var onboarding = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.ONBOARDING_PATH).Select(name => $"{{{{{name}}}}}"));
var workState = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.WORK_STATE_PATH).Select(name => $"{{{{{name}}}}}"));
var result = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.ResultPath(BflResultKind.POSITION_PAPER)).Select(name => $"{{{{{name}}}}}"));
var research = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.RESEARCH_PATH).Select(name => $"{{{{{name}}}}}"));
using (Assert.EnterMultipleScope())
{
@@ -76,6 +77,7 @@ public sealed class BflPromptLibraryTests
Assert.That(BflPromptComposer.ComposeOnboarding(new(onboarding, "Anrede"), "Deutsch"), Does.Not.Contain("{{"));
Assert.That(BflPromptComposer.ComposeWorkStateRequest(workState, BflStep.FRAMEWORK, new BflStepFormData()), Does.Not.Contain("{{"));
Assert.That(BflPromptComposer.ComposeResultRequest(result, new Dictionary<BflStep, BflWorkState>()), Does.Not.Contain("{{"));
Assert.That(BflPromptComposer.ComposeResearch(new(research, "Auftrag"), [], "Deutsch"), Does.Not.Contain("{{"));
}
}
@@ -198,7 +198,7 @@ public sealed class BflPromptOverridesTests
{
var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
var store = new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.directory });
var service = new BflConversationService(store, settingsManager, null!, null!, NullLogger<BflConversationService>.Instance);
var service = new BflConversationService(store, settingsManager, null!, null!, null!, NullLogger<BflConversationService>.Instance);
await BflPromptOverrides.CopyBuiltInAsync(store.PromptOverridesDirectory);
settingsManager.ConfigurationData.BeyondFactsLab.AllowPromptOverrides = true;
@@ -0,0 +1,54 @@
using AIStudio.Assistants.BeyondFactsLab;
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Tests.BeyondFactsLab;
/// <summary>
/// Checks what the BeyondFacts Lab tells the person when its research cannot search the web.
/// </summary>
[TestFixture]
public sealed class BflResearchAvailabilityTests
{
[Test]
public void EveryReasonHasAMessageOfItsOwn()
{
var reasons = Enum.GetValues<ToolOfferBlockReason>().Where(reason => reason is not ToolOfferBlockReason.NONE).ToList();
var messages = reasons.Select(BflResearchAvailability.MessageFor).ToList();
var unknown = BflResearchAvailability.MessageFor((ToolOfferBlockReason)int.MaxValue);
using (Assert.EnterMultipleScope())
{
Assert.That(messages, Has.All.Not.Empty);
Assert.That(messages, Is.Unique);
Assert.That(messages, Has.None.EqualTo(unknown), "Every reason this version knows has a text of its own.");
Assert.That(unknown, Is.Not.Empty);
}
}
[Test]
public void NothingInTheWayNeedsNoMessage()
{
var availability = new BflResearchAvailability(ToolOfferBlockReason.NONE, ToolOfferBlockReason.NONE);
using (Assert.EnterMultipleScope())
{
Assert.That(availability.CanResearch, Is.True);
Assert.That(availability.Message, Is.Empty);
}
}
[Test]
public void TheWebSearchDecidesWhetherTheLabCanResearch()
{
var withoutReadingPages = new BflResearchAvailability(ToolOfferBlockReason.NONE, ToolOfferBlockReason.TOOL_SWITCHED_OFF);
var withoutSearching = new BflResearchAvailability(ToolOfferBlockReason.NOT_CONFIGURED, ToolOfferBlockReason.NONE);
using (Assert.EnterMultipleScope())
{
Assert.That(withoutReadingPages.CanResearch, Is.True);
Assert.That(withoutReadingPages.Message, Is.Empty);
Assert.That(withoutSearching.CanResearch, Is.False);
Assert.That(withoutSearching.Message, Is.EqualTo(BflResearchAvailability.MessageFor(ToolOfferBlockReason.NOT_CONFIGURED)));
}
}
}
@@ -0,0 +1,144 @@
using AIStudio.Assistants.BeyondFactsLab;
namespace AIStudio.Tests.BeyondFactsLab;
/// <summary>
/// Checks how the research reaches the step "Strategy" of the BeyondFacts Lab, and when the lab starts it on its own.
/// </summary>
[TestFixture]
public sealed class BflResearchInStrategyTests
{
private static readonly BflPromptParts PARTS = new("{{RESEARCH}}", "Leitfaden", "Schritt", "Anrede");
private static readonly DateTimeOffset CREATED_AT_UTC = new(2026, 11, 10, 14, 0, 0, TimeSpan.Zero);
[Test]
public void OnlyTheStepStrategyCarriesTheResearch()
{
var forms = Forms("Klimapolitik");
var research = Research(forms, BflResearchTopic.PARTICIPANTS, "## Teilnehmende\n\n### Person A");
using (Assert.EnterMultipleScope())
{
foreach (var step in BflStep.All)
{
var prompt = BflPromptComposer.Compose(PARTS, step, forms, [], "Deutsch", null, null, research);
if (step is BflStep.STRATEGY)
{
Assert.That(prompt, Does.Contain("### Person A"));
Assert.That(prompt, Does.Contain($"<thema titel=\"{BflResearchTopic.PARTICIPANTS.Title}\" stand=\"2026-11-10\">"));
}
else
{
Assert.That(prompt, Does.Not.Contain("Person A"), step.ToString());
Assert.That(prompt, Does.Contain("„Strategie“"), step.ToString());
}
}
}
}
[Test]
public void TheResearchComesInTheOrderOfItsTopics()
{
var forms = Forms("Klimapolitik");
var research = new Dictionary<BflResearchTopic, BflResearch>
{
[BflResearchTopic.DEBATE] = BflResearch.Of("Debatte", [], [], BflPromptComposer.PublicDetailsOf(forms), CREATED_AT_UTC),
[BflResearchTopic.EVENT] = BflResearch.Of("Veranstaltung", [], [], BflPromptComposer.PublicDetailsOf(forms), CREATED_AT_UTC),
};
var prompt = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, forms, [], "Deutsch", null, null, research);
Assert.That(prompt.IndexOf("Veranstaltung", StringComparison.Ordinal), Is.LessThan(prompt.IndexOf("Debatte", StringComparison.Ordinal)));
}
[Test]
public void AResearchWhosePublicDetailsChangedIsMarked()
{
var research = Research(Forms("Klimapolitik"), BflResearchTopic.EVENT, "## Veranstaltung");
var current = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, Forms("Klimapolitik"), [], "Deutsch", null, null, research);
var changed = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, Forms("Energiepolitik"), [], "Deutsch", null, null, research);
using (Assert.EnterMultipleScope())
{
Assert.That(current, Does.Not.Contain("haben sich seit dieser Recherche geändert"));
Assert.That(changed, Does.Contain("haben sich seit dieser Recherche geändert"));
}
}
[Test]
public void ForAnEventWhichIsNotPublicTheStepLearnsThatThereIsNoResearch()
{
var forms = Forms("Klimapolitik");
var research = Research(forms, BflResearchTopic.PARTICIPANTS, "## Teilnehmende\n\n### Person A");
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_VISIBILITY, BflFieldCatalog.EVENT_NOT_PUBLIC);
var prompt = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, forms, [], "Deutsch", null, null, research);
using (Assert.EnterMultipleScope())
{
Assert.That(prompt, Does.Contain("nicht öffentlich"));
Assert.That(prompt, Does.Not.Contain("Person A"), "An earlier research no longer counts once the person said that the event is not public.");
}
}
[Test]
public void ForAPublicEventTheStepGetsTheResearch()
{
var forms = Forms("Klimapolitik");
var research = Research(forms, BflResearchTopic.PARTICIPANTS, "## Teilnehmende\n\n### Person A");
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_VISIBILITY, BflFieldCatalog.EVENT_PUBLIC);
var prompt = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, forms, [], "Deutsch", null, null, research);
using (Assert.EnterMultipleScope())
{
Assert.That(prompt, Does.Contain("### Person A"));
Assert.That(prompt, Does.Not.Contain("haben sich seit dieser Recherche geändert"), "Whether the event is public is no detail which the research receives.");
}
}
[Test]
public void WithoutResearchTheStepLearnsThatThereIsNone()
{
var prompt = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, Forms("Klimapolitik"), [], "Deutsch");
Assert.That(prompt, Is.EqualTo("Es gibt noch keine Recherche."));
}
[Test]
public void EveryTopicWithoutResearchIsDue()
{
var details = BflPromptComposer.PublicDetailsOf(Forms("Klimapolitik"));
Assert.That(BflResearchRunner.TopicsDue(new Dictionary<BflResearchTopic, BflResearch>(), details), Is.EqualTo(BflResearchTopic.All));
}
[Test]
public void OnlyTheTopicsWithoutAFittingResearchAreDue()
{
var forms = Forms("Klimapolitik");
var details = BflPromptComposer.PublicDetailsOf(forms);
var research = new Dictionary<BflResearchTopic, BflResearch>
{
[BflResearchTopic.EVENT] = BflResearch.Of("Veranstaltung", [], [], details, CREATED_AT_UTC),
[BflResearchTopic.PARTICIPANTS] = BflResearch.Of("Teilnehmende", [], [], BflPromptComposer.PublicDetailsOf(Forms("Energiepolitik")), CREATED_AT_UTC),
};
Assert.That(BflResearchRunner.TopicsDue(research, details), Is.EqualTo(new[] { BflResearchTopic.PARTICIPANTS, BflResearchTopic.DEBATE }));
}
private static Dictionary<BflStep, BflStepFormData> Forms(string eventTopic)
{
var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData());
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.FORMAT, "Podium");
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_TOPIC, eventTopic);
return forms;
}
private static Dictionary<BflResearchTopic, BflResearch> Research(IReadOnlyDictionary<BflStep, BflStepFormData> forms, BflResearchTopic topic, string text) => new()
{
[topic] = BflResearch.Of(text, [], [], BflPromptComposer.PublicDetailsOf(forms), CREATED_AT_UTC),
};
}
@@ -0,0 +1,177 @@
using AIStudio.Assistants.BeyondFactsLab;
using AIStudio.Provider;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Tests.BeyondFactsLab;
/// <summary>
/// Checks what the research agents of the BeyondFacts Lab receive: the public details about the event, and nothing
/// else of a preparation.
/// </summary>
[TestFixture]
public sealed class BflResearchPromptTests
{
[Test]
public async Task OnlyThePublicDetailsReachTheResearch()
{
var forms = FormsWithAMarkerInEveryField();
var details = BflPromptComposer.PublicDetailsOf(forms);
foreach (var topic in BflResearchTopic.All)
{
var prompt = BflPromptComposer.ComposeResearch(await BflPromptLibrary.LoadResearchPartsAsync(topic), details, "Deutsch");
using (Assert.EnterMultipleScope())
{
foreach (var field in TextFields())
{
if (field.IsSharedWithResearch)
Assert.That(prompt, Does.Contain(Marker(field)), $"{topic}: {field.Id}");
else
Assert.That(prompt, Does.Not.Contain(Marker(field)), $"{topic}: {field.Id}");
}
}
}
}
[Test]
public void ThePublicDetailsAreTheFieldsMarkedForTheResearch()
{
var details = BflPromptComposer.PublicDetailsOf(FormsWithAMarkerInEveryField());
var expected = BflFieldCatalog.FIELDS.Where(field => field.IsSharedWithResearch).Select(field => field.Id).ToList();
using (Assert.EnterMultipleScope())
{
Assert.That(details.Select(detail => detail.FieldId), Is.EqualTo(expected));
Assert.That(details.Select(detail => detail.Value), Is.EqualTo(expected.Select(id => $"MARKER-{id}")));
}
}
[Test]
public void TheTopicOfTheContributionNeverReachesTheResearch()
{
var field = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.CONTRIBUTION_TOPIC);
Assert.That(field.IsSharedWithResearch, Is.False, "The topic of the contribution may be unpublished.");
}
[Test]
public void ADetailOfAFieldWhichIsNotPublicIsLeftOut()
{
BflResearchDetail[] details =
[
new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Klimapolitik" },
new() { FieldId = BflFieldCatalog.CONTRIBUTION_TOPIC, Value = "Unveröffentlichte Studie" },
new() { FieldId = "unknown_field", Value = "Unbekannt" },
];
var prompt = BflPromptComposer.ComposeResearch(new("{{PUBLIC_DETAILS}}", "Auftrag"), details, "Deutsch");
using (Assert.EnterMultipleScope())
{
Assert.That(prompt, Does.Contain("Klimapolitik"));
Assert.That(prompt, Does.Not.Contain("Unveröffentlichte Studie"));
Assert.That(prompt, Does.Not.Contain("Unbekannt"));
}
}
[Test]
public void TheResearchHasNoPlaceForAnythingButThePublicDetails()
{
var placeholders = BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.RESEARCH_PATH);
using (Assert.EnterMultipleScope())
{
Assert.That(placeholders, Is.EquivalentTo(new[] { BflPromptTemplate.RESPONSE_LANGUAGE, BflPromptTemplate.TOPIC, BflPromptTemplate.TOOL_CALLS, BflPromptTemplate.PUBLIC_DETAILS }));
foreach (var topic in BflResearchTopic.All)
Assert.That(BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.ResearchTopicPath(topic)), Is.Empty, topic.ToString());
}
}
[Test]
public async Task EveryTopicHasATaskOfItsOwn()
{
var paths = BflResearchTopic.All.Select(BflPromptLibrary.ResearchTopicPath).ToList();
var fileNames = BflResearchTopic.All.Select(topic => topic.FileName).ToList();
using (Assert.EnterMultipleScope())
{
Assert.That(paths, Is.Unique);
Assert.That(fileNames, Is.Unique);
Assert.That(BflPromptLibrary.AllPaths, Is.SupersetOf(paths).And.Contains(BflPromptLibrary.RESEARCH_PATH));
foreach (var path in paths)
Assert.That(await BflPromptLibrary.ReadAsync(path), Is.Not.Empty, path);
}
}
[Test]
public async Task TheAgentKnowsItsBudgetOfToolCalls()
{
var prompt = BflPromptComposer.ComposeResearch(await BflPromptLibrary.LoadResearchPartsAsync(BflResearchTopic.PARTICIPANTS), [], "Deutsch");
Assert.That(prompt, Does.Contain($"höchstens {ToolSelectionRules.MAX_TOOL_CALLS} Werkzeugaufrufe"));
}
[Test]
public async Task TheRequestNamesSectionsOfThePrompt()
{
var research = (await BflPromptLibrary.ReadAsync(BflPromptLibrary.RESEARCH_PATH)).ReplaceLineEndings("\n");
var request = BflResearchRunner.REQUEST;
using (Assert.EnterMultipleScope())
{
foreach (var section in new[] { "Dein Auftrag", "Vorgehen", "Form" })
{
Assert.That(request, Does.Contain($"„{section}“"));
Assert.That(research, Does.Contain($"\n# {section}\n"));
}
}
}
[Test]
public async Task TheSamePublicDetailsGiveTheSamePrompt()
{
var parts = await BflPromptLibrary.LoadResearchPartsAsync(BflResearchTopic.EVENT);
var first = BflPromptComposer.ComposeResearch(parts, BflPromptComposer.PublicDetailsOf(FormsWithAMarkerInEveryField()), "Deutsch");
var second = BflPromptComposer.ComposeResearch(parts, BflPromptComposer.PublicDetailsOf(FormsWithAMarkerInEveryField()), "Deutsch");
Assert.That(second, Is.EqualTo(first));
}
[Test]
public void AResearchAgentSearchesTheWebAndReadsPagesInAChatOfItsOwn()
{
var provider = new AIStudio.Settings.Provider(1, "provider-1", "Test", LLMProviders.SELF_HOSTED, new Model("model", null));
var first = BflThreadFactory.CreateHidden("research-event");
var second = BflThreadFactory.CreateHidden("research-event");
BflThreadFactory.PrepareForRequest(first, provider, "Prompt", BflResearchRunner.TOOL_IDS);
using (Assert.EnterMultipleScope())
{
Assert.That(BflResearchRunner.TOOL_IDS, Is.EquivalentTo(new[] { ToolSelectionRules.WEB_SEARCH_TOOL_ID, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID }));
Assert.That(first.RuntimeSelectedToolIds, Is.EquivalentTo(BflResearchRunner.TOOL_IDS));
Assert.That(first.RuntimeToolsAreAssistantManaged, Is.True);
Assert.That(first.RequiredOutboundDataRestriction.Restriction, Is.EqualTo(OutboundDataRestriction.UNRESTRICTED));
Assert.That(first.DataSourceOptions.DisableDataSources, Is.True);
Assert.That(first.Blocks, Is.Empty);
Assert.That(first.ChatId, Is.Not.EqualTo(Guid.Empty).And.Not.EqualTo(second.ChatId));
}
}
private static IEnumerable<BflFieldDefinition> TextFields() => BflFieldCatalog.FIELDS.Where(field => field.Kind is not (BflFieldKind.CHOICE or BflFieldKind.MULTI_CHOICE));
private static string Marker(BflFieldDefinition field) => $"MARKER-{field.Id}";
private static Dictionary<BflStep, BflStepFormData> FormsWithAMarkerInEveryField()
{
var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData());
foreach (var field in TextFields())
forms[field.Step].Set(field.Id, Marker(field));
foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.Kind is BflFieldKind.CHOICE or BflFieldKind.MULTI_CHOICE))
forms[field.Step].SetChoices(field, [field.Options[0].Id]);
return forms;
}
}
@@ -0,0 +1,117 @@
using AIStudio.Assistants.BeyondFactsLab;
using AIStudio.Tools;
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Tests.BeyondFactsLab;
/// <summary>
/// Checks how the BeyondFacts Lab reads a research from the answer of an agent and the calls of its tools.
/// </summary>
[TestFixture]
public sealed class BflResearchTests
{
private static readonly DateTimeOffset CREATED_AT_UTC = new(2026, 11, 10, 14, 0, 0, TimeSpan.Zero);
private static readonly BflResearchDetail[] DETAILS =
[
new() { FieldId = BflFieldCatalog.FORMAT, Value = "Podium" },
new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Klimapolitik" },
];
[Test]
public void AResearchKeepsWhatLeftAIStudio()
{
ToolInvocationTrace[] invocations =
[
Call(2, ToolSelectionRules.WEB_SEARCH_TOOL_ID, BflResearch.QUERY_ARGUMENT, "Podium Klimapolitik 2026"),
Call(1, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, BflResearch.URL_ARGUMENT, "https://example.org/event"),
Call(3, ToolSelectionRules.WEB_SEARCH_TOOL_ID, BflResearch.QUERY_ARGUMENT, "Podium Klimapolitik 2026"),
Call(4, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, BflResearch.URL_ARGUMENT, "https://example.org/unknown-title"),
Call(5, ToolSelectionRules.WEB_SEARCH_TOOL_ID, BflResearch.QUERY_ARGUMENT, "Blocked query", wasExecuted: false),
Call(6, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, BflResearch.URL_ARGUMENT, "not an address"),
Call(7, "calculate", "expression", "1 + 1"),
];
Source[] sources =
[
new("Event page", "https://example.org/event", SourceOrigin.TOOL),
new("Written by the model", "https://example.org/unknown-title", SourceOrigin.LLM),
];
var research = BflResearch.Of("## Veranstaltung\n\nEin Podium.", invocations, sources, DETAILS, CREATED_AT_UTC);
using (Assert.EnterMultipleScope())
{
Assert.That(research.Text, Is.EqualTo("## Veranstaltung\n\nEin Podium."));
Assert.That(research.SearchQueries, Is.EqualTo(new[] { "Podium Klimapolitik 2026" }));
Assert.That(research.Pages.Select(page => page.Url), Is.EqualTo(new[] { "https://example.org/event", "https://example.org/unknown-title" }));
Assert.That(research.Pages.Select(page => page.Title), Is.EqualTo(new[] { "Event page", "https://example.org/unknown-title" }));
Assert.That(research.Details.Select(detail => detail.Value), Is.EqualTo(new[] { "Podium", "Klimapolitik" }));
Assert.That(research.Details, Has.None.SameAs(DETAILS[0]), "The research keeps a copy of the details.");
Assert.That(research.CreatedAtUtc, Is.EqualTo(CREATED_AT_UTC));
}
}
[Test]
public void ACodeBlockAroundTheAnswerIsTakenOff()
{
var research = BflResearch.Of("```markdown\n## Debatte\n\nText\n```", [], [], DETAILS, CREATED_AT_UTC);
Assert.That(research.Text, Is.EqualTo("## Debatte\n\nText"));
}
[Test]
public void AVeryLongAnswerIsShortenedAtTheEndOfALine()
{
var line = new string('x', 99);
var answer = string.Join("\n", Enumerable.Repeat(line, 1_000));
var research = BflResearch.Of(answer, [], [], DETAILS, CREATED_AT_UTC);
using (Assert.EnterMultipleScope())
{
Assert.That(research.Text.Length, Is.LessThanOrEqualTo(BflResearch.MAX_TEXT_CHARACTERS));
Assert.That(research.Text, Does.EndWith(line));
}
}
[Test]
public void ALongTitleIsShortened()
{
var title = new string('T', BflResearch.MAX_TITLE_CHARACTERS + 50);
var research = BflResearch.Of("Text", [Call(1, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, BflResearch.URL_ARGUMENT, "https://example.org/")], [new(title, "https://example.org/", SourceOrigin.TOOL)], DETAILS, CREATED_AT_UTC);
Assert.That(research.Pages.Single().Title, Has.Length.EqualTo(BflResearch.MAX_TITLE_CHARACTERS).And.EndWith("…"));
}
[Test]
public void AnAnswerWithoutTextGivesAnEmptyResearch()
{
var research = BflResearch.Of(" \n ", [], [], DETAILS, CREATED_AT_UTC);
Assert.That(research.Text, Is.Empty);
}
[Test]
public void AResearchIsOutdatedOnceThePublicDetailsChanged()
{
var research = BflResearch.Of("Text", [], [], DETAILS, CREATED_AT_UTC);
using (Assert.EnterMultipleScope())
{
Assert.That(research.IsOutdated([new() { FieldId = BflFieldCatalog.FORMAT, Value = "Podium" }, new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Klimapolitik" }]), Is.False);
Assert.That(research.IsOutdated([new() { FieldId = BflFieldCatalog.FORMAT, Value = "Podium" }, new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Energiepolitik" }]), Is.True);
Assert.That(research.IsOutdated([new() { FieldId = BflFieldCatalog.FORMAT, Value = "Podium" }]), Is.True);
Assert.That(research.IsOutdated([.. DETAILS, new() { FieldId = BflFieldCatalog.EVENT_LINK, Value = "https://example.org/" }]), Is.True);
}
}
private static ToolInvocationTrace Call(int order, string toolId, string argument, string value, bool wasExecuted = true) => new()
{
Order = order,
ToolId = toolId,
WasExecuted = wasExecuted,
Status = wasExecuted ? ToolInvocationTraceStatus.SUCCESS : ToolInvocationTraceStatus.BLOCKED,
Arguments = new() { [argument] = value },
};
}
@@ -0,0 +1,121 @@
using AIStudio.Assistants.BeyondFactsLab;
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Tests.BeyondFactsLab;
/// <summary>
/// Checks what the chats of the BeyondFacts Lab learn about their tools with every request.
/// </summary>
[TestFixture]
public sealed class BflToolStatusTests
{
private static readonly BflResearchAvailability EVERYTHING = new(ToolOfferBlockReason.NONE, ToolOfferBlockReason.NONE);
private static readonly BflResearchAvailability NOTHING = new(ToolOfferBlockReason.NOT_CONFIGURED, ToolOfferBlockReason.TOOL_SWITCHED_OFF_BY_USER);
[Test]
public void EveryReasonHasASentenceOfItsOwn()
{
var sentences = Enum.GetValues<ToolOfferBlockReason>().Select(BflToolStatus.ReasonFor).ToList();
using (Assert.EnterMultipleScope())
{
Assert.That(sentences, Has.All.Not.Empty);
Assert.That(sentences, Is.Unique);
Assert.That(sentences, Has.None.EqualTo(BflToolStatus.ReasonFor((ToolOfferBlockReason)int.MaxValue)), "Every reason this version knows has a sentence of its own.");
}
}
[Test]
public void AStepWithItsToolsReadsPagesAndKnowsTheResearch()
{
var text = BflToolStatus.ForStep(EVERYTHING);
using (Assert.EnterMultipleScope())
{
Assert.That(text, Does.Contain("Du kannst Webseiten lesen"));
Assert.That(text, Does.Contain("Im Web suchen kannst du in diesem Gespräch nie"));
Assert.That(text, Does.Contain("Mit der aktuellen Einrichtung ist die Recherche verfügbar."));
}
}
[Test]
public void AStepWithoutItsToolsLearnsWhy()
{
var text = BflToolStatus.ForStep(NOTHING);
using (Assert.EnterMultipleScope())
{
Assert.That(text, Does.Contain("Webseiten lesen kannst du in diesem Gespräch nicht."));
Assert.That(text, Does.Contain(BflToolStatus.ReasonFor(ToolOfferBlockReason.TOOL_SWITCHED_OFF_BY_USER)));
Assert.That(text, Does.Contain("Mit der aktuellen Einrichtung ist die Recherche nicht verfügbar."));
Assert.That(text, Does.Contain(BflToolStatus.ReasonFor(ToolOfferBlockReason.NOT_CONFIGURED)));
Assert.That(text, Does.Not.Contain("Du kannst Webseiten lesen"));
}
}
[Test]
public void TheOnboardingTellsWhatTheLabCanDoWithTheCurrentSetup()
{
var everything = BflToolStatus.ForOnboarding(EVERYTHING);
var nothing = BflToolStatus.ForOnboarding(NOTHING);
using (Assert.EnterMultipleScope())
{
Assert.That(everything, Does.Contain("kann das Lab im Web recherchieren"));
Assert.That(everything, Does.Contain("können außerdem Webseiten lesen"));
Assert.That(nothing, Does.Contain("kann das Lab nicht im Web recherchieren"));
Assert.That(nothing, Does.Contain(BflToolStatus.ReasonFor(ToolOfferBlockReason.NOT_CONFIGURED)));
Assert.That(nothing, Does.Contain(BflToolStatus.ReasonFor(ToolOfferBlockReason.TOOL_SWITCHED_OFF_BY_USER)));
}
}
[Test]
public void TheStepsAndTheOnboardingKnowThatTheLabNeverResearchesAnEventWhichIsNotPublic()
{
using (Assert.EnterMultipleScope())
{
Assert.That(BflToolStatus.ForStep(EVERYTHING), Does.Contain("Ist die Veranstaltung nicht öffentlich, recherchiert das Lab nie im Web"));
Assert.That(BflToolStatus.ForStep(NOTHING), Does.Contain("Ist die Veranstaltung nicht öffentlich, recherchiert das Lab nie im Web"));
Assert.That(BflToolStatus.ForOnboarding(EVERYTHING), Does.Contain("Ist die Veranstaltung nicht öffentlich, recherchiert das Lab nie im Web"));
}
}
[Test]
public void EveryTextNamesExactlyTheDetailsTheResearchReceives()
{
var texts = new[] { BflToolStatus.ForStep(EVERYTHING), BflToolStatus.ForStep(NOTHING), BflToolStatus.ForOnboarding(EVERYTHING), BflToolStatus.ForOnboarding(NOTHING) };
using (Assert.EnterMultipleScope())
{
foreach (var field in BflFieldCatalog.FIELDS)
{
foreach (var text in texts)
{
if (field.IsSharedWithResearch)
Assert.That(text, Does.Contain($"„{field.Label}“"), field.Id);
else
Assert.That(text, Does.Not.Contain($"„{field.Label}“"), field.Id);
}
}
}
}
[Test]
public void TheSameSetupGivesTheSamePrompt()
{
var parts = new BflPromptParts("{{TOOLS}}", "Leitfaden", "Schritt", "Anrede");
var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData());
var first = BflPromptComposer.Compose(parts, BflStep.STRATEGY, forms, [], "Deutsch", null, new BflResearchAvailability(ToolOfferBlockReason.NONE, ToolOfferBlockReason.NONE));
var second = BflPromptComposer.Compose(parts, BflStep.STRATEGY, forms, [], "Deutsch", null, new BflResearchAvailability(ToolOfferBlockReason.NONE, ToolOfferBlockReason.NONE));
var other = BflPromptComposer.Compose(parts, BflStep.STRATEGY, forms, [], "Deutsch", null, NOTHING);
using (Assert.EnterMultipleScope())
{
Assert.That(second, Is.EqualTo(first));
Assert.That(other, Is.Not.EqualTo(first));
Assert.That(first, Does.Not.Contain("\r"));
}
}
}
@@ -26,7 +26,7 @@ public sealed class BflWorkStateRequestTests
}
[Test]
public void TheRequestOffersSuggestionsOnlyForEmptyTextFields()
public void TheRequestOffersSuggestionsOnlyForEmptyFields()
{
var form = new BflStepFormData();
form.Set(BflFieldCatalog.FORMAT, "Podium");
@@ -41,6 +41,21 @@ public sealed class BflWorkStateRequestTests
}
}
[Test]
public void TheRequestNamesTheOptionsOfAnEmptyChoice()
{
var difficulty = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.DIFFICULTY);
var request = BflPromptComposer.ComposeWorkStateRequest("{{FIELDS}}", BflStep.TEST_SITUATION, new BflStepFormData());
using (Assert.EnterMultipleScope())
{
Assert.That(request, Does.Contain($"[{BflMarkdownSections.FIELD_PREFIX}{BflFieldCatalog.DIFFICULTY}]"));
foreach (var option in difficulty.Options)
Assert.That(request, Does.Contain($"„{option.Label}“"), option.Id);
}
}
[Test]
public void ARequestWithoutEmptyFieldsAsksForNoSuggestions()
{
+2 -1
View File
@@ -61,12 +61,13 @@ public static partial class JsonTestRoots
yield return Case(new(typeof(IReadOnlyList<string>), SqliteProvenanceStoreClientImplementation.JSON_OPTIONS));
yield return Case(new(typeof(List<string>), SqliteProvenanceStoreClientImplementation.JSON_OPTIONS));
// The preparations of the BeyondFacts Lab, their forms, materials, work states, and results, and the preparation selected last:
// The preparations of the BeyondFacts Lab, their forms, materials, work states, results, and research, and the preparation selected last:
yield return Case(new(typeof(BflProject), BeyondFactsLabJson.PERSISTENCE));
yield return Case(new(typeof(BflStepFormData), BeyondFactsLabJson.PERSISTENCE));
yield return Case(new(typeof(BflMaterial), BeyondFactsLabJson.PERSISTENCE));
yield return Case(new(typeof(BflWorkState), BeyondFactsLabJson.PERSISTENCE));
yield return Case(new(typeof(BflResult), BeyondFactsLabJson.PERSISTENCE));
yield return Case(new(typeof(BflResearch), BeyondFactsLabJson.PERSISTENCE));
yield return Case(new(typeof(Guid?), BeyondFactsLabJson.PERSISTENCE), "selected preparation");
}