mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-11 19:47:21 +00:00
Added research on the web to the BeyondFacts Lab (#1186)
This commit is contained in:
1 parent
e5c0931a60
commit
9a676101f7
66 files changed
+3609
-70
No files matched your search
@@ -0,0 +1,118 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the BeyondFacts Lab stores the research of a preparation.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BeyondFactsLabStoreResearchTests
|
||||
{
|
||||
private string dataDirectory = string.Empty;
|
||||
|
||||
[SetUp]
|
||||
public void CreateDataDirectory()
|
||||
{
|
||||
this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.dataDirectory);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void DeleteDataDirectory()
|
||||
{
|
||||
if (Directory.Exists(this.dataDirectory))
|
||||
Directory.Delete(this.dataDirectory, true);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APreparationWithoutResearchHasNone()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(await store.LoadResearchAsync(created.ProjectId, BflResearchTopic.EVENT), Is.Null);
|
||||
Assert.That(await store.LoadAllResearchAsync(created.ProjectId), Is.Empty);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AResearchIsStoredForItsTopic()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var research = new BflResearch
|
||||
{
|
||||
Text = "## Teilnehmende\n\n### Person A",
|
||||
Details = [new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Klimapolitik" }],
|
||||
SearchQueries = ["Podium Klimapolitik"],
|
||||
Pages = [new() { Title = "Programm", Url = "https://example.org/programm" }],
|
||||
CreatedAtUtc = new DateTimeOffset(2026, 11, 10, 14, 0, 0, TimeSpan.Zero),
|
||||
};
|
||||
|
||||
var changed = await store.SaveResearchAsync(created.ProjectId, BflResearchTopic.PARTICIPANTS, research);
|
||||
var loaded = await this.CreateStore().LoadResearchAsync(created.ProjectId, BflResearchTopic.PARTICIPANTS);
|
||||
var all = await this.CreateStore().LoadAllResearchAsync(created.ProjectId);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(loaded, Is.Not.Null);
|
||||
Assert.That(loaded!.Text, Is.EqualTo(research.Text));
|
||||
Assert.That(loaded.Details.Single().Value, Is.EqualTo("Klimapolitik"));
|
||||
Assert.That(loaded.SearchQueries, Is.EqualTo(new[] { "Podium Klimapolitik" }));
|
||||
Assert.That(loaded.Pages.Single().Url, Is.EqualTo("https://example.org/programm"));
|
||||
Assert.That(loaded.CreatedAtUtc, Is.EqualTo(research.CreatedAtUtc));
|
||||
Assert.That(all.Keys, Is.EquivalentTo(new[] { BflResearchTopic.PARTICIPANTS }));
|
||||
Assert.That(changed.ModifiedAtUtc, Is.GreaterThanOrEqualTo(created.ModifiedAtUtc));
|
||||
Assert.That(File.Exists(Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, created.ProjectId.ToString("D"), "steps", "05-strategy", "research", "participants.json")), Is.True);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedResearchStaysAsItIs()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var path = await this.WriteResearchFileAsync(created.ProjectId, "event.json", "{ not json");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
await Assert.ThrowsAsync<InvalidDataException>(async () => await store.LoadResearchAsync(created.ProjectId, BflResearchTopic.EVENT));
|
||||
await Assert.ThrowsAsync<InvalidDataException>(async () => await store.SaveResearchAsync(created.ProjectId, BflResearchTopic.EVENT, new BflResearch { Text = "Neu" }));
|
||||
Assert.That(await store.LoadAllResearchAsync(created.ProjectId), Is.Empty);
|
||||
Assert.That(await File.ReadAllTextAsync(path), Is.EqualTo("{ not json"));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AResearchOfANewerVersionStaysAsItIs()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
var created = await store.CreateAsync("Panel");
|
||||
await this.WriteResearchFileAsync(created.ProjectId, "debate.json", $$"""{ "schemaVersion": {{BflResearch.CURRENT_SCHEMA_VERSION + 1}}, "text": "Neu" }""");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(async () => await store.LoadResearchAsync(created.ProjectId, BflResearchTopic.DEBATE));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ResearchForAMissingPreparationIsNotStored()
|
||||
{
|
||||
var store = this.CreateStore();
|
||||
|
||||
await Assert.ThrowsAsync<FileNotFoundException>(async () => await store.SaveResearchAsync(Guid.NewGuid(), BflResearchTopic.EVENT, new BflResearch { Text = "Text" }));
|
||||
}
|
||||
|
||||
private async Task<string> WriteResearchFileAsync(Guid projectId, string fileName, string content)
|
||||
{
|
||||
var directory = Path.Combine(this.dataDirectory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, projectId.ToString("D"), "steps", "05-strategy", "research");
|
||||
Directory.CreateDirectory(directory);
|
||||
var path = Path.Combine(directory, fileName);
|
||||
await File.WriteAllTextAsync(path, content);
|
||||
return path;
|
||||
}
|
||||
|
||||
private BeyondFactsLabStore CreateStore() => new(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory });
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks the tools of the chats of the BeyondFacts Lab, and that the chat of a step never searches the web.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// That the level ONLY_LINKS_FROM_CHAT keeps a web search from running, while Read Web Page still reads the addresses
|
||||
/// given to the model, is checked by ToolOutboundDataTests for every chat alike.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class BflChatToolsTests
|
||||
{
|
||||
private static readonly Guid PROJECT_ID = Guid.Parse("6f1c2b0e-3a51-4c3e-9d2f-0b7a4e5c8d91");
|
||||
|
||||
private string dataDirectory = string.Empty;
|
||||
|
||||
[SetUp]
|
||||
public void CreateDataDirectory()
|
||||
{
|
||||
this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.dataDirectory);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void DeleteDataDirectory()
|
||||
{
|
||||
if (Directory.Exists(this.dataDirectory))
|
||||
Directory.Delete(this.dataDirectory, true);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheChatOfEveryStepReadsWebPagesButNeverSearches()
|
||||
{
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
foreach (var step in BflStep.All)
|
||||
{
|
||||
var tools = BflChatTools.For(new BflStepThreadLocation(PROJECT_ID, step));
|
||||
Assert.That(tools, Is.EquivalentTo(new[] { ToolSelectionRules.READ_WEB_PAGE_TOOL_ID }), step.ToString());
|
||||
Assert.That(tools, Does.Not.Contain(ToolSelectionRules.WEB_SEARCH_TOOL_ID), step.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheOnboardingHasNoTools()
|
||||
{
|
||||
Assert.That(BflChatTools.For(BflOnboardingThreadLocation.INSTANCE), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheChatOfAStepReadsOnlyTheAddressesGivenToItFromTheStart()
|
||||
{
|
||||
var stepChat = BflThreadFactory.Create(new BflStepThreadLocation(PROJECT_ID, BflStep.STRATEGY));
|
||||
var onboarding = BflThreadFactory.Create(BflOnboardingThreadLocation.INSTANCE);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(stepChat.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT));
|
||||
Assert.That(BflChatTools.STEP_REQUIREMENT.Restriction, Is.EqualTo(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT));
|
||||
Assert.That(BflChatTools.STEP_REQUIREMENT.Origin, Is.EqualTo(OutboundDataOrigin.BEYOND_FACTS_LAB));
|
||||
Assert.That(onboarding.RequiredOutboundDataRestriction, Is.EqualTo(OutboundDataRequirement.NONE));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnEarlierChatGetsTheRestrictionAndAStricterOneStays()
|
||||
{
|
||||
var earlier = new ChatThread();
|
||||
var stricter = new ChatThread();
|
||||
var strictest = new OutboundDataRequirement(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, OutboundDataOrigin.INTERNAL_WEB_PAGE, "intranet.example.org");
|
||||
stricter.RequireOutboundDataRestriction(strictest);
|
||||
|
||||
BflChatTools.Restrict(earlier, new BflStepThreadLocation(PROJECT_ID, BflStep.FRAMEWORK));
|
||||
BflChatTools.Restrict(stricter, new BflStepThreadLocation(PROJECT_ID, BflStep.FRAMEWORK));
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(earlier.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT));
|
||||
Assert.That(stricter.RequiredOutboundDataRestriction, Is.EqualTo(strictest));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheRestrictionIsStoredWithTheChat()
|
||||
{
|
||||
var store = new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory });
|
||||
var created = await store.CreateAsync("Panel");
|
||||
var location = new BflStepThreadLocation(created.ProjectId, BflStep.KNOWLEDGE);
|
||||
|
||||
await store.SaveThreadAsync(location, BflThreadFactory.Create(location));
|
||||
var loaded = await new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.dataDirectory }).LoadThreadAsync(location);
|
||||
|
||||
Assert.That(loaded!.RequiredOutboundDataRestriction, Is.EqualTo(BflChatTools.STEP_REQUIREMENT));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AToolKeptBackNamesTheLab()
|
||||
{
|
||||
var message = BflChatTools.STEP_REQUIREMENT.GetToolBlockedMessage(new Data());
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(message, Does.Contain("BeyondFacts Lab"));
|
||||
Assert.That(message, Is.Not.EqualTo(new OutboundDataRequirement(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, (OutboundDataOrigin)999, string.Empty).GetToolBlockedMessage(new Data())));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARequestOfAStepOffersTheToolsOfItsPlace()
|
||||
{
|
||||
var location = new BflStepThreadLocation(PROJECT_ID, BflStep.TEST_SITUATION);
|
||||
var thread = BflThreadFactory.Create(location);
|
||||
var provider = new AIStudio.Settings.Provider(1, "provider-1", "Test", AIStudio.Provider.LLMProviders.SELF_HOSTED, new AIStudio.Provider.Model("model", null));
|
||||
|
||||
BflThreadFactory.PrepareForRequest(thread, provider, "Prompt", BflChatTools.For(location));
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(thread.RuntimeSelectedToolIds, Is.EquivalentTo(new[] { ToolSelectionRules.READ_WEB_PAGE_TOOL_ID }));
|
||||
Assert.That(thread.RuntimeToolsAreAssistantManaged, Is.True);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks when the BeyondFacts Lab takes an event for not public, and when a suggestion about it keeps the research
|
||||
/// from starting on its own.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BflEventVisibilityTests
|
||||
{
|
||||
[TestCase(null, false, TestName = "AnEmptyFieldLetsTheLabResearch")]
|
||||
[TestCase(BflFieldCatalog.EVENT_PUBLIC, false, TestName = "APublicEventIsResearched")]
|
||||
[TestCase(BflFieldCatalog.EVENT_NOT_PUBLIC, true, TestName = "AnEventWhichIsNotPublicIsNeverResearched")]
|
||||
[TestCase("event_unknown", false, TestName = "AnOptionThisVersionDoesNotKnowLetsTheLabResearch")]
|
||||
public void OnlyTheOptionNotPublicStopsTheResearch(string? visibility, bool isNotPublic)
|
||||
{
|
||||
var forms = Forms(visibility);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(BflEventVisibility.IsNotPublic(forms), Is.EqualTo(isNotPublic));
|
||||
Assert.That(BflEventVisibility.IsNotPublic(forms[BflStep.FRAMEWORK]), Is.EqualTo(isNotPublic));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void APreparationWithoutEventDetailsIsResearchedAsBefore()
|
||||
{
|
||||
Assert.That(BflEventVisibility.IsNotPublic(new Dictionary<BflStep, BflStepFormData>()), Is.False);
|
||||
}
|
||||
|
||||
[TestCase(BflFieldCatalog.EVENT_PUBLIC)]
|
||||
[TestCase(BflFieldCatalog.EVENT_NOT_PUBLIC)]
|
||||
public void ASuggestionWaitsWhileThePersonHasNotDecided(string suggestion)
|
||||
{
|
||||
Assert.That(BflEventVisibility.IsSuggestionPending(WorkStates(suggestion), Forms(null)), Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASuggestionNoLongerWaitsOnceThePersonDecided()
|
||||
{
|
||||
Assert.That(BflEventVisibility.IsSuggestionPending(WorkStates(BflFieldCatalog.EVENT_NOT_PUBLIC), Forms(BflFieldCatalog.EVENT_PUBLIC)), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WithoutASuggestionNothingWaits()
|
||||
{
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(BflEventVisibility.IsSuggestionPending(new Dictionary<BflStep, BflWorkState>(), Forms(null)), Is.False);
|
||||
Assert.That(BflEventVisibility.IsSuggestionPending(new Dictionary<BflStep, BflWorkState> { [BflStep.FRAMEWORK] = new() }, Forms(null)), Is.False);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASuggestionWhichThisVersionDoesNotShowKeepsNothingWaiting()
|
||||
{
|
||||
Assert.That(BflEventVisibility.IsSuggestionPending(WorkStates("event_unknown"), Forms(null)), Is.False);
|
||||
}
|
||||
|
||||
private static Dictionary<BflStep, BflStepFormData> Forms(string? visibility)
|
||||
{
|
||||
var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData());
|
||||
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_TOPIC, "Klimapolitik");
|
||||
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_VISIBILITY, visibility);
|
||||
return forms;
|
||||
}
|
||||
|
||||
private static Dictionary<BflStep, BflWorkState> WorkStates(string suggestion)
|
||||
{
|
||||
var workState = new BflWorkState();
|
||||
workState.FieldSuggestions[BflFieldCatalog.EVENT_VISIBILITY] = suggestion;
|
||||
return new Dictionary<BflStep, BflWorkState> { [BflStep.FRAMEWORK] = workState };
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,7 @@ public sealed class BflFieldCatalogTests
|
||||
{
|
||||
Assert.That(BflFieldCatalog.FIELDS.Select(field => field.Id), Is.EqualTo(new[]
|
||||
{
|
||||
"format", "event_topic", "audience", "contribution_topic", "event_date", "event_location", "event_duration", "contribution_duration", "other_participants", "event_link",
|
||||
"format", "event_topic", "audience", "event_visibility", "contribution_topic", "event_date", "event_location", "event_duration", "contribution_duration", "other_participants", "event_link",
|
||||
"contribution_summary", "core_theses", "uncertainties", "material_notes",
|
||||
"goals", "goal_directions",
|
||||
"values_and_concerns", "personal_concern", "other_perspectives",
|
||||
@@ -50,6 +50,7 @@ public sealed class BflFieldCatalogTests
|
||||
{
|
||||
Assert.That(BflFieldCatalog.FIELDS.SelectMany(field => field.Options).Select(option => option.Id), Is.EqualTo(new[]
|
||||
{
|
||||
"event_public", "event_not_public",
|
||||
"goal_inform", "goal_moderate", "goal_take_position", "goal_raise_awareness", "goal_show_options", "goal_influence_policy",
|
||||
"questions_mixed", "questions_scientific_criticism", "questions_emotional_criticism", "questions_manipulative", "questions_political_instrumentalization",
|
||||
"difficulty_moderate", "difficulty_demanding", "difficulty_very_demanding",
|
||||
@@ -164,7 +165,21 @@ public sealed class BflFieldCatalogTests
|
||||
|
||||
Assert.That(keyDetails, Is.EquivalentTo(new[]
|
||||
{
|
||||
BflFieldCatalog.FORMAT, BflFieldCatalog.EVENT_TOPIC, BflFieldCatalog.AUDIENCE, BflFieldCatalog.CONTRIBUTION_TOPIC, BflFieldCatalog.CONTRIBUTION_SUMMARY,
|
||||
BflFieldCatalog.FORMAT, BflFieldCatalog.EVENT_TOPIC, BflFieldCatalog.AUDIENCE, BflFieldCatalog.EVENT_VISIBILITY, BflFieldCatalog.CONTRIBUTION_TOPIC, BflFieldCatalog.CONTRIBUTION_SUMMARY,
|
||||
}));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WhetherTheEventIsPublicIsASingleChoiceWhichNeverReachesTheResearch()
|
||||
{
|
||||
var visibility = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.EVENT_VISIBILITY);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(visibility.Step, Is.EqualTo(BflStep.FRAMEWORK));
|
||||
Assert.That(visibility.Kind, Is.EqualTo(BflFieldKind.CHOICE));
|
||||
Assert.That(visibility.Options.Select(option => option.Id), Is.EqualTo(new[] { BflFieldCatalog.EVENT_PUBLIC, BflFieldCatalog.EVENT_NOT_PUBLIC }));
|
||||
Assert.That(visibility.IsSharedWithResearch, Is.False, "The field decides about the research; it is no detail about the event.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@ public sealed class BflMarkdownSectionsTests
|
||||
"KNOWLEDGE/ESTABLISHED", "KNOWLEDGE/UNCERTAINTIES", "KNOWLEDGE/WEAK_POINTS", "KNOWLEDGE/CONTROVERSIES", "KNOWLEDGE/EXPERTISE_LIMITS", "KNOWLEDGE/KEY_MESSAGES", "KNOWLEDGE/OPEN_POINTS",
|
||||
"GOALS_AND_ROLE/GOAL", "GOALS_AND_ROLE/ROLE", "GOALS_AND_ROLE/REASONS", "GOALS_AND_ROLE/CONSEQUENCES", "GOALS_AND_ROLE/OPEN_POINTS",
|
||||
"VALUES_AND_PERSPECTIVES/VALUES", "VALUES_AND_PERSPECTIVES/VALUE_CONFLICTS", "VALUES_AND_PERSPECTIVES/PERSONAL_INVOLVEMENT", "VALUES_AND_PERSPECTIVES/PERSPECTIVES", "VALUES_AND_PERSPECTIVES/CONSEQUENCES", "VALUES_AND_PERSPECTIVES/OPEN_POINTS",
|
||||
"STRATEGY/KEY_MESSAGES", "STRATEGY/STRUCTURE", "STRATEGY/LANGUAGE", "STRATEGY/UNCERTAINTY", "STRATEGY/FRAMING", "STRATEGY/COUNTERARGUMENTS", "STRATEGY/WORRIES", "STRATEGY/OPEN_POINTS",
|
||||
"STRATEGY/KEY_MESSAGES", "STRATEGY/STRUCTURE", "STRATEGY/LANGUAGE", "STRATEGY/UNCERTAINTY", "STRATEGY/FRAMING", "STRATEGY/PARTICIPANTS", "STRATEGY/COUNTERARGUMENTS", "STRATEGY/WORRIES", "STRATEGY/OPEN_POINTS",
|
||||
"TEST_SITUATION/STRENGTHS", "TEST_SITUATION/PRACTICE", "TEST_SITUATION/HARD_QUESTIONS", "TEST_SITUATION/POSITION", "TEST_SITUATION/OPEN_POINTS",
|
||||
"RESULTS/CHANGED", "RESULTS/KEPT", "RESULTS/LEFT_OPEN", "RESULTS/NEXT_STEPS",
|
||||
}));
|
||||
@@ -199,18 +199,59 @@ public sealed class BflMarkdownSectionsTests
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void OnlyEmptyTextFieldsWhichAreNotPersonalGetSuggestions()
|
||||
public void OnlyEmptyFieldsWithATextOrASingleChoiceWhichAreNotPersonalGetSuggestions()
|
||||
{
|
||||
var goals = BflMarkdownSections.SuggestableFields(BflStep.GOALS_AND_ROLE, new BflStepFormData()).Select(field => field.Id);
|
||||
var values = BflMarkdownSections.SuggestableFields(BflStep.VALUES_AND_PERSPECTIVES, new BflStepFormData()).Select(field => field.Id);
|
||||
var test = BflMarkdownSections.SuggestableFields(BflStep.TEST_SITUATION, new BflStepFormData()).Select(field => field.Id);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(goals, Is.EqualTo(new[] { BflFieldCatalog.GOALS }), "A field with several choices gets no suggestion.");
|
||||
Assert.That(values, Is.EqualTo(new[] { BflFieldCatalog.VALUES_AND_CONCERNS, BflFieldCatalog.OTHER_PERSPECTIVES }));
|
||||
Assert.That(test, Is.EqualTo(new[] { BflFieldCatalog.TEST_NOTES }));
|
||||
Assert.That(test, Is.EqualTo(new[] { BflFieldCatalog.QUESTION_TYPE, BflFieldCatalog.DIFFICULTY, BflFieldCatalog.TEST_NOTES }));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASuggestionForAChoiceNamesItsOptionByLabelOrIdAndKeepsTheId()
|
||||
{
|
||||
var label = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.DIFFICULTY).Options.Single(option => option.Id == BflFieldCatalog.DIFFICULTY_DEMANDING).Label;
|
||||
string[] suggestions = [label, label.ToUpperInvariant(), $"„{label}“", $"\"{label}\"", $"**{label}**", $"{label}.", BflFieldCatalog.DIFFICULTY_DEMANDING];
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
foreach (var suggestion in suggestions)
|
||||
{
|
||||
var answer = BflMarkdownSections.Read($"## [FIELD:difficulty]\n{suggestion}", BflStep.TEST_SITUATION, new BflStepFormData());
|
||||
Assert.That(answer.FieldSuggestions, Is.EqualTo(new Dictionary<string, string> { [BflFieldCatalog.DIFFICULTY] = BflFieldCatalog.DIFFICULTY_DEMANDING }), suggestion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase("## [FIELD:difficulty]\nSomewhat demanding", TestName = "ASuggestionWhichNamesNoOptionIsLeftOut")]
|
||||
[TestCase("## [FIELD:difficulty]\ndifficulty_moderate, difficulty_demanding", TestName = "ASuggestionWithSeveralOptionsIsLeftOut")]
|
||||
[TestCase("## [FIELD:goal_directions]\ngoal_inform", TestName = "AFieldWithSeveralChoicesGetsNoSuggestion")]
|
||||
public void ASuggestionForAChoiceWhichDoesNotFitIsLeftOut(string suggestion)
|
||||
{
|
||||
var step = suggestion.Contains("goal_", StringComparison.Ordinal) ? BflStep.GOALS_AND_ROLE : BflStep.TEST_SITUATION;
|
||||
|
||||
var answer = BflMarkdownSections.Read(suggestion, step, new BflStepFormData());
|
||||
|
||||
Assert.That(answer.FieldSuggestions, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AChoiceWhichHoldsAnOptionGetsNoSuggestion()
|
||||
{
|
||||
var form = new BflStepFormData();
|
||||
form.Set(BflFieldCatalog.DIFFICULTY, BflFieldCatalog.DIFFICULTY_MODERATE);
|
||||
|
||||
var answer = BflMarkdownSections.Read($"## [FIELD:difficulty]\n{BflFieldCatalog.DIFFICULTY_DEMANDING}", BflStep.TEST_SITUATION, form);
|
||||
|
||||
Assert.That(answer.FieldSuggestions, Is.Empty);
|
||||
}
|
||||
|
||||
private static BflWorkStateAnswer Read(string answer) => BflMarkdownSections.Read(answer, BflStep.GOALS_AND_ROLE, new BflStepFormData());
|
||||
|
||||
private static string FrameworkWithSuggestions(string suggestions) => $"## [FORMAT]\n## [AUDIENCE]\n## [SPEAKING_TIME]\n## [PARTICIPANTS]\n## [CONFLICTS]\n## [OPEN_POINTS]\n{suggestions}";
|
||||
|
||||
@@ -24,7 +24,7 @@ public sealed class BflMissingDetailsTests
|
||||
|
||||
var missing = BflMissingDetails.Of(BflStep.FRAMEWORK, form);
|
||||
|
||||
Assert.That(missing.Select(field => field.Id), Is.EqualTo(new[] { BflFieldCatalog.FORMAT, BflFieldCatalog.AUDIENCE, BflFieldCatalog.CONTRIBUTION_TOPIC }));
|
||||
Assert.That(missing.Select(field => field.Id), Is.EqualTo(new[] { BflFieldCatalog.FORMAT, BflFieldCatalog.AUDIENCE, BflFieldCatalog.EVENT_VISIBILITY, BflFieldCatalog.CONTRIBUTION_TOPIC }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
|
||||
@@ -69,6 +69,7 @@ public sealed class BflPromptLibraryTests
|
||||
var onboarding = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.ONBOARDING_PATH).Select(name => $"{{{{{name}}}}}"));
|
||||
var workState = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.WORK_STATE_PATH).Select(name => $"{{{{{name}}}}}"));
|
||||
var result = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.ResultPath(BflResultKind.POSITION_PAPER)).Select(name => $"{{{{{name}}}}}"));
|
||||
var research = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.RESEARCH_PATH).Select(name => $"{{{{{name}}}}}"));
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
@@ -76,6 +77,7 @@ public sealed class BflPromptLibraryTests
|
||||
Assert.That(BflPromptComposer.ComposeOnboarding(new(onboarding, "Anrede"), "Deutsch"), Does.Not.Contain("{{"));
|
||||
Assert.That(BflPromptComposer.ComposeWorkStateRequest(workState, BflStep.FRAMEWORK, new BflStepFormData()), Does.Not.Contain("{{"));
|
||||
Assert.That(BflPromptComposer.ComposeResultRequest(result, new Dictionary<BflStep, BflWorkState>()), Does.Not.Contain("{{"));
|
||||
Assert.That(BflPromptComposer.ComposeResearch(new(research, "Auftrag"), [], "Deutsch"), Does.Not.Contain("{{"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -198,7 +198,7 @@ public sealed class BflPromptOverridesTests
|
||||
{
|
||||
var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
|
||||
var store = new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.directory });
|
||||
var service = new BflConversationService(store, settingsManager, null!, null!, NullLogger<BflConversationService>.Instance);
|
||||
var service = new BflConversationService(store, settingsManager, null!, null!, null!, NullLogger<BflConversationService>.Instance);
|
||||
await BflPromptOverrides.CopyBuiltInAsync(store.PromptOverridesDirectory);
|
||||
|
||||
settingsManager.ConfigurationData.BeyondFactsLab.AllowPromptOverrides = true;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks what the BeyondFacts Lab tells the person when its research cannot search the web.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BflResearchAvailabilityTests
|
||||
{
|
||||
[Test]
|
||||
public void EveryReasonHasAMessageOfItsOwn()
|
||||
{
|
||||
var reasons = Enum.GetValues<ToolOfferBlockReason>().Where(reason => reason is not ToolOfferBlockReason.NONE).ToList();
|
||||
var messages = reasons.Select(BflResearchAvailability.MessageFor).ToList();
|
||||
var unknown = BflResearchAvailability.MessageFor((ToolOfferBlockReason)int.MaxValue);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(messages, Has.All.Not.Empty);
|
||||
Assert.That(messages, Is.Unique);
|
||||
Assert.That(messages, Has.None.EqualTo(unknown), "Every reason this version knows has a text of its own.");
|
||||
Assert.That(unknown, Is.Not.Empty);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NothingInTheWayNeedsNoMessage()
|
||||
{
|
||||
var availability = new BflResearchAvailability(ToolOfferBlockReason.NONE, ToolOfferBlockReason.NONE);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(availability.CanResearch, Is.True);
|
||||
Assert.That(availability.Message, Is.Empty);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheWebSearchDecidesWhetherTheLabCanResearch()
|
||||
{
|
||||
var withoutReadingPages = new BflResearchAvailability(ToolOfferBlockReason.NONE, ToolOfferBlockReason.TOOL_SWITCHED_OFF);
|
||||
var withoutSearching = new BflResearchAvailability(ToolOfferBlockReason.NOT_CONFIGURED, ToolOfferBlockReason.NONE);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(withoutReadingPages.CanResearch, Is.True);
|
||||
Assert.That(withoutReadingPages.Message, Is.Empty);
|
||||
Assert.That(withoutSearching.CanResearch, Is.False);
|
||||
Assert.That(withoutSearching.Message, Is.EqualTo(BflResearchAvailability.MessageFor(ToolOfferBlockReason.NOT_CONFIGURED)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the research reaches the step "Strategy" of the BeyondFacts Lab, and when the lab starts it on its own.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BflResearchInStrategyTests
|
||||
{
|
||||
private static readonly BflPromptParts PARTS = new("{{RESEARCH}}", "Leitfaden", "Schritt", "Anrede");
|
||||
|
||||
private static readonly DateTimeOffset CREATED_AT_UTC = new(2026, 11, 10, 14, 0, 0, TimeSpan.Zero);
|
||||
|
||||
[Test]
|
||||
public void OnlyTheStepStrategyCarriesTheResearch()
|
||||
{
|
||||
var forms = Forms("Klimapolitik");
|
||||
var research = Research(forms, BflResearchTopic.PARTICIPANTS, "## Teilnehmende\n\n### Person A");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
foreach (var step in BflStep.All)
|
||||
{
|
||||
var prompt = BflPromptComposer.Compose(PARTS, step, forms, [], "Deutsch", null, null, research);
|
||||
if (step is BflStep.STRATEGY)
|
||||
{
|
||||
Assert.That(prompt, Does.Contain("### Person A"));
|
||||
Assert.That(prompt, Does.Contain($"<thema titel=\"{BflResearchTopic.PARTICIPANTS.Title}\" stand=\"2026-11-10\">"));
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.That(prompt, Does.Not.Contain("Person A"), step.ToString());
|
||||
Assert.That(prompt, Does.Contain("„Strategie“"), step.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheResearchComesInTheOrderOfItsTopics()
|
||||
{
|
||||
var forms = Forms("Klimapolitik");
|
||||
var research = new Dictionary<BflResearchTopic, BflResearch>
|
||||
{
|
||||
[BflResearchTopic.DEBATE] = BflResearch.Of("Debatte", [], [], BflPromptComposer.PublicDetailsOf(forms), CREATED_AT_UTC),
|
||||
[BflResearchTopic.EVENT] = BflResearch.Of("Veranstaltung", [], [], BflPromptComposer.PublicDetailsOf(forms), CREATED_AT_UTC),
|
||||
};
|
||||
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, forms, [], "Deutsch", null, null, research);
|
||||
|
||||
Assert.That(prompt.IndexOf("Veranstaltung", StringComparison.Ordinal), Is.LessThan(prompt.IndexOf("Debatte", StringComparison.Ordinal)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AResearchWhosePublicDetailsChangedIsMarked()
|
||||
{
|
||||
var research = Research(Forms("Klimapolitik"), BflResearchTopic.EVENT, "## Veranstaltung");
|
||||
|
||||
var current = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, Forms("Klimapolitik"), [], "Deutsch", null, null, research);
|
||||
var changed = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, Forms("Energiepolitik"), [], "Deutsch", null, null, research);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(current, Does.Not.Contain("haben sich seit dieser Recherche geändert"));
|
||||
Assert.That(changed, Does.Contain("haben sich seit dieser Recherche geändert"));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ForAnEventWhichIsNotPublicTheStepLearnsThatThereIsNoResearch()
|
||||
{
|
||||
var forms = Forms("Klimapolitik");
|
||||
var research = Research(forms, BflResearchTopic.PARTICIPANTS, "## Teilnehmende\n\n### Person A");
|
||||
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_VISIBILITY, BflFieldCatalog.EVENT_NOT_PUBLIC);
|
||||
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, forms, [], "Deutsch", null, null, research);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt, Does.Contain("nicht öffentlich"));
|
||||
Assert.That(prompt, Does.Not.Contain("Person A"), "An earlier research no longer counts once the person said that the event is not public.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ForAPublicEventTheStepGetsTheResearch()
|
||||
{
|
||||
var forms = Forms("Klimapolitik");
|
||||
var research = Research(forms, BflResearchTopic.PARTICIPANTS, "## Teilnehmende\n\n### Person A");
|
||||
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_VISIBILITY, BflFieldCatalog.EVENT_PUBLIC);
|
||||
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, forms, [], "Deutsch", null, null, research);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt, Does.Contain("### Person A"));
|
||||
Assert.That(prompt, Does.Not.Contain("haben sich seit dieser Recherche geändert"), "Whether the event is public is no detail which the research receives.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WithoutResearchTheStepLearnsThatThereIsNone()
|
||||
{
|
||||
var prompt = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, Forms("Klimapolitik"), [], "Deutsch");
|
||||
|
||||
Assert.That(prompt, Is.EqualTo("Es gibt noch keine Recherche."));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryTopicWithoutResearchIsDue()
|
||||
{
|
||||
var details = BflPromptComposer.PublicDetailsOf(Forms("Klimapolitik"));
|
||||
|
||||
Assert.That(BflResearchRunner.TopicsDue(new Dictionary<BflResearchTopic, BflResearch>(), details), Is.EqualTo(BflResearchTopic.All));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void OnlyTheTopicsWithoutAFittingResearchAreDue()
|
||||
{
|
||||
var forms = Forms("Klimapolitik");
|
||||
var details = BflPromptComposer.PublicDetailsOf(forms);
|
||||
var research = new Dictionary<BflResearchTopic, BflResearch>
|
||||
{
|
||||
[BflResearchTopic.EVENT] = BflResearch.Of("Veranstaltung", [], [], details, CREATED_AT_UTC),
|
||||
[BflResearchTopic.PARTICIPANTS] = BflResearch.Of("Teilnehmende", [], [], BflPromptComposer.PublicDetailsOf(Forms("Energiepolitik")), CREATED_AT_UTC),
|
||||
};
|
||||
|
||||
Assert.That(BflResearchRunner.TopicsDue(research, details), Is.EqualTo(new[] { BflResearchTopic.PARTICIPANTS, BflResearchTopic.DEBATE }));
|
||||
}
|
||||
|
||||
private static Dictionary<BflStep, BflStepFormData> Forms(string eventTopic)
|
||||
{
|
||||
var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData());
|
||||
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.FORMAT, "Podium");
|
||||
forms[BflStep.FRAMEWORK].Set(BflFieldCatalog.EVENT_TOPIC, eventTopic);
|
||||
return forms;
|
||||
}
|
||||
|
||||
private static Dictionary<BflResearchTopic, BflResearch> Research(IReadOnlyDictionary<BflStep, BflStepFormData> forms, BflResearchTopic topic, string text) => new()
|
||||
{
|
||||
[topic] = BflResearch.Of(text, [], [], BflPromptComposer.PublicDetailsOf(forms), CREATED_AT_UTC),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks what the research agents of the BeyondFacts Lab receive: the public details about the event, and nothing
|
||||
/// else of a preparation.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BflResearchPromptTests
|
||||
{
|
||||
[Test]
|
||||
public async Task OnlyThePublicDetailsReachTheResearch()
|
||||
{
|
||||
var forms = FormsWithAMarkerInEveryField();
|
||||
var details = BflPromptComposer.PublicDetailsOf(forms);
|
||||
|
||||
foreach (var topic in BflResearchTopic.All)
|
||||
{
|
||||
var prompt = BflPromptComposer.ComposeResearch(await BflPromptLibrary.LoadResearchPartsAsync(topic), details, "Deutsch");
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
foreach (var field in TextFields())
|
||||
{
|
||||
if (field.IsSharedWithResearch)
|
||||
Assert.That(prompt, Does.Contain(Marker(field)), $"{topic}: {field.Id}");
|
||||
else
|
||||
Assert.That(prompt, Does.Not.Contain(Marker(field)), $"{topic}: {field.Id}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThePublicDetailsAreTheFieldsMarkedForTheResearch()
|
||||
{
|
||||
var details = BflPromptComposer.PublicDetailsOf(FormsWithAMarkerInEveryField());
|
||||
var expected = BflFieldCatalog.FIELDS.Where(field => field.IsSharedWithResearch).Select(field => field.Id).ToList();
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(details.Select(detail => detail.FieldId), Is.EqualTo(expected));
|
||||
Assert.That(details.Select(detail => detail.Value), Is.EqualTo(expected.Select(id => $"MARKER-{id}")));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheTopicOfTheContributionNeverReachesTheResearch()
|
||||
{
|
||||
var field = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.CONTRIBUTION_TOPIC);
|
||||
|
||||
Assert.That(field.IsSharedWithResearch, Is.False, "The topic of the contribution may be unpublished.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ADetailOfAFieldWhichIsNotPublicIsLeftOut()
|
||||
{
|
||||
BflResearchDetail[] details =
|
||||
[
|
||||
new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Klimapolitik" },
|
||||
new() { FieldId = BflFieldCatalog.CONTRIBUTION_TOPIC, Value = "Unveröffentlichte Studie" },
|
||||
new() { FieldId = "unknown_field", Value = "Unbekannt" },
|
||||
];
|
||||
|
||||
var prompt = BflPromptComposer.ComposeResearch(new("{{PUBLIC_DETAILS}}", "Auftrag"), details, "Deutsch");
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(prompt, Does.Contain("Klimapolitik"));
|
||||
Assert.That(prompt, Does.Not.Contain("Unveröffentlichte Studie"));
|
||||
Assert.That(prompt, Does.Not.Contain("Unbekannt"));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheResearchHasNoPlaceForAnythingButThePublicDetails()
|
||||
{
|
||||
var placeholders = BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.RESEARCH_PATH);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(placeholders, Is.EquivalentTo(new[] { BflPromptTemplate.RESPONSE_LANGUAGE, BflPromptTemplate.TOPIC, BflPromptTemplate.TOOL_CALLS, BflPromptTemplate.PUBLIC_DETAILS }));
|
||||
foreach (var topic in BflResearchTopic.All)
|
||||
Assert.That(BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.ResearchTopicPath(topic)), Is.Empty, topic.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task EveryTopicHasATaskOfItsOwn()
|
||||
{
|
||||
var paths = BflResearchTopic.All.Select(BflPromptLibrary.ResearchTopicPath).ToList();
|
||||
var fileNames = BflResearchTopic.All.Select(topic => topic.FileName).ToList();
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(paths, Is.Unique);
|
||||
Assert.That(fileNames, Is.Unique);
|
||||
Assert.That(BflPromptLibrary.AllPaths, Is.SupersetOf(paths).And.Contains(BflPromptLibrary.RESEARCH_PATH));
|
||||
foreach (var path in paths)
|
||||
Assert.That(await BflPromptLibrary.ReadAsync(path), Is.Not.Empty, path);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheAgentKnowsItsBudgetOfToolCalls()
|
||||
{
|
||||
var prompt = BflPromptComposer.ComposeResearch(await BflPromptLibrary.LoadResearchPartsAsync(BflResearchTopic.PARTICIPANTS), [], "Deutsch");
|
||||
|
||||
Assert.That(prompt, Does.Contain($"höchstens {ToolSelectionRules.MAX_TOOL_CALLS} Werkzeugaufrufe"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheRequestNamesSectionsOfThePrompt()
|
||||
{
|
||||
var research = (await BflPromptLibrary.ReadAsync(BflPromptLibrary.RESEARCH_PATH)).ReplaceLineEndings("\n");
|
||||
var request = BflResearchRunner.REQUEST;
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
foreach (var section in new[] { "Dein Auftrag", "Vorgehen", "Form" })
|
||||
{
|
||||
Assert.That(request, Does.Contain($"„{section}“"));
|
||||
Assert.That(research, Does.Contain($"\n# {section}\n"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheSamePublicDetailsGiveTheSamePrompt()
|
||||
{
|
||||
var parts = await BflPromptLibrary.LoadResearchPartsAsync(BflResearchTopic.EVENT);
|
||||
var first = BflPromptComposer.ComposeResearch(parts, BflPromptComposer.PublicDetailsOf(FormsWithAMarkerInEveryField()), "Deutsch");
|
||||
var second = BflPromptComposer.ComposeResearch(parts, BflPromptComposer.PublicDetailsOf(FormsWithAMarkerInEveryField()), "Deutsch");
|
||||
|
||||
Assert.That(second, Is.EqualTo(first));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AResearchAgentSearchesTheWebAndReadsPagesInAChatOfItsOwn()
|
||||
{
|
||||
var provider = new AIStudio.Settings.Provider(1, "provider-1", "Test", LLMProviders.SELF_HOSTED, new Model("model", null));
|
||||
var first = BflThreadFactory.CreateHidden("research-event");
|
||||
var second = BflThreadFactory.CreateHidden("research-event");
|
||||
|
||||
BflThreadFactory.PrepareForRequest(first, provider, "Prompt", BflResearchRunner.TOOL_IDS);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(BflResearchRunner.TOOL_IDS, Is.EquivalentTo(new[] { ToolSelectionRules.WEB_SEARCH_TOOL_ID, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID }));
|
||||
Assert.That(first.RuntimeSelectedToolIds, Is.EquivalentTo(BflResearchRunner.TOOL_IDS));
|
||||
Assert.That(first.RuntimeToolsAreAssistantManaged, Is.True);
|
||||
Assert.That(first.RequiredOutboundDataRestriction.Restriction, Is.EqualTo(OutboundDataRestriction.UNRESTRICTED));
|
||||
Assert.That(first.DataSourceOptions.DisableDataSources, Is.True);
|
||||
Assert.That(first.Blocks, Is.Empty);
|
||||
Assert.That(first.ChatId, Is.Not.EqualTo(Guid.Empty).And.Not.EqualTo(second.ChatId));
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<BflFieldDefinition> TextFields() => BflFieldCatalog.FIELDS.Where(field => field.Kind is not (BflFieldKind.CHOICE or BflFieldKind.MULTI_CHOICE));
|
||||
|
||||
private static string Marker(BflFieldDefinition field) => $"MARKER-{field.Id}";
|
||||
|
||||
private static Dictionary<BflStep, BflStepFormData> FormsWithAMarkerInEveryField()
|
||||
{
|
||||
var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData());
|
||||
foreach (var field in TextFields())
|
||||
forms[field.Step].Set(field.Id, Marker(field));
|
||||
|
||||
foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.Kind is BflFieldKind.CHOICE or BflFieldKind.MULTI_CHOICE))
|
||||
forms[field.Step].SetChoices(field, [field.Options[0].Id]);
|
||||
|
||||
return forms;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the BeyondFacts Lab reads a research from the answer of an agent and the calls of its tools.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BflResearchTests
|
||||
{
|
||||
private static readonly DateTimeOffset CREATED_AT_UTC = new(2026, 11, 10, 14, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private static readonly BflResearchDetail[] DETAILS =
|
||||
[
|
||||
new() { FieldId = BflFieldCatalog.FORMAT, Value = "Podium" },
|
||||
new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Klimapolitik" },
|
||||
];
|
||||
|
||||
[Test]
|
||||
public void AResearchKeepsWhatLeftAIStudio()
|
||||
{
|
||||
ToolInvocationTrace[] invocations =
|
||||
[
|
||||
Call(2, ToolSelectionRules.WEB_SEARCH_TOOL_ID, BflResearch.QUERY_ARGUMENT, "Podium Klimapolitik 2026"),
|
||||
Call(1, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, BflResearch.URL_ARGUMENT, "https://example.org/event"),
|
||||
Call(3, ToolSelectionRules.WEB_SEARCH_TOOL_ID, BflResearch.QUERY_ARGUMENT, "Podium Klimapolitik 2026"),
|
||||
Call(4, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, BflResearch.URL_ARGUMENT, "https://example.org/unknown-title"),
|
||||
Call(5, ToolSelectionRules.WEB_SEARCH_TOOL_ID, BflResearch.QUERY_ARGUMENT, "Blocked query", wasExecuted: false),
|
||||
Call(6, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, BflResearch.URL_ARGUMENT, "not an address"),
|
||||
Call(7, "calculate", "expression", "1 + 1"),
|
||||
];
|
||||
|
||||
Source[] sources =
|
||||
[
|
||||
new("Event page", "https://example.org/event", SourceOrigin.TOOL),
|
||||
new("Written by the model", "https://example.org/unknown-title", SourceOrigin.LLM),
|
||||
];
|
||||
|
||||
var research = BflResearch.Of("## Veranstaltung\n\nEin Podium.", invocations, sources, DETAILS, CREATED_AT_UTC);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(research.Text, Is.EqualTo("## Veranstaltung\n\nEin Podium."));
|
||||
Assert.That(research.SearchQueries, Is.EqualTo(new[] { "Podium Klimapolitik 2026" }));
|
||||
Assert.That(research.Pages.Select(page => page.Url), Is.EqualTo(new[] { "https://example.org/event", "https://example.org/unknown-title" }));
|
||||
Assert.That(research.Pages.Select(page => page.Title), Is.EqualTo(new[] { "Event page", "https://example.org/unknown-title" }));
|
||||
Assert.That(research.Details.Select(detail => detail.Value), Is.EqualTo(new[] { "Podium", "Klimapolitik" }));
|
||||
Assert.That(research.Details, Has.None.SameAs(DETAILS[0]), "The research keeps a copy of the details.");
|
||||
Assert.That(research.CreatedAtUtc, Is.EqualTo(CREATED_AT_UTC));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ACodeBlockAroundTheAnswerIsTakenOff()
|
||||
{
|
||||
var research = BflResearch.Of("```markdown\n## Debatte\n\nText\n```", [], [], DETAILS, CREATED_AT_UTC);
|
||||
|
||||
Assert.That(research.Text, Is.EqualTo("## Debatte\n\nText"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AVeryLongAnswerIsShortenedAtTheEndOfALine()
|
||||
{
|
||||
var line = new string('x', 99);
|
||||
var answer = string.Join("\n", Enumerable.Repeat(line, 1_000));
|
||||
|
||||
var research = BflResearch.Of(answer, [], [], DETAILS, CREATED_AT_UTC);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(research.Text.Length, Is.LessThanOrEqualTo(BflResearch.MAX_TEXT_CHARACTERS));
|
||||
Assert.That(research.Text, Does.EndWith(line));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ALongTitleIsShortened()
|
||||
{
|
||||
var title = new string('T', BflResearch.MAX_TITLE_CHARACTERS + 50);
|
||||
var research = BflResearch.Of("Text", [Call(1, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, BflResearch.URL_ARGUMENT, "https://example.org/")], [new(title, "https://example.org/", SourceOrigin.TOOL)], DETAILS, CREATED_AT_UTC);
|
||||
|
||||
Assert.That(research.Pages.Single().Title, Has.Length.EqualTo(BflResearch.MAX_TITLE_CHARACTERS).And.EndWith("…"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnAnswerWithoutTextGivesAnEmptyResearch()
|
||||
{
|
||||
var research = BflResearch.Of(" \n ", [], [], DETAILS, CREATED_AT_UTC);
|
||||
|
||||
Assert.That(research.Text, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AResearchIsOutdatedOnceThePublicDetailsChanged()
|
||||
{
|
||||
var research = BflResearch.Of("Text", [], [], DETAILS, CREATED_AT_UTC);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(research.IsOutdated([new() { FieldId = BflFieldCatalog.FORMAT, Value = "Podium" }, new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Klimapolitik" }]), Is.False);
|
||||
Assert.That(research.IsOutdated([new() { FieldId = BflFieldCatalog.FORMAT, Value = "Podium" }, new() { FieldId = BflFieldCatalog.EVENT_TOPIC, Value = "Energiepolitik" }]), Is.True);
|
||||
Assert.That(research.IsOutdated([new() { FieldId = BflFieldCatalog.FORMAT, Value = "Podium" }]), Is.True);
|
||||
Assert.That(research.IsOutdated([.. DETAILS, new() { FieldId = BflFieldCatalog.EVENT_LINK, Value = "https://example.org/" }]), Is.True);
|
||||
}
|
||||
}
|
||||
|
||||
private static ToolInvocationTrace Call(int order, string toolId, string argument, string value, bool wasExecuted = true) => new()
|
||||
{
|
||||
Order = order,
|
||||
ToolId = toolId,
|
||||
WasExecuted = wasExecuted,
|
||||
Status = wasExecuted ? ToolInvocationTraceStatus.SUCCESS : ToolInvocationTraceStatus.BLOCKED,
|
||||
Arguments = new() { [argument] = value },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
using AIStudio.Assistants.BeyondFactsLab;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tests.BeyondFactsLab;
|
||||
|
||||
/// <summary>
|
||||
/// Checks what the chats of the BeyondFacts Lab learn about their tools with every request.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class BflToolStatusTests
|
||||
{
|
||||
private static readonly BflResearchAvailability EVERYTHING = new(ToolOfferBlockReason.NONE, ToolOfferBlockReason.NONE);
|
||||
|
||||
private static readonly BflResearchAvailability NOTHING = new(ToolOfferBlockReason.NOT_CONFIGURED, ToolOfferBlockReason.TOOL_SWITCHED_OFF_BY_USER);
|
||||
|
||||
[Test]
|
||||
public void EveryReasonHasASentenceOfItsOwn()
|
||||
{
|
||||
var sentences = Enum.GetValues<ToolOfferBlockReason>().Select(BflToolStatus.ReasonFor).ToList();
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(sentences, Has.All.Not.Empty);
|
||||
Assert.That(sentences, Is.Unique);
|
||||
Assert.That(sentences, Has.None.EqualTo(BflToolStatus.ReasonFor((ToolOfferBlockReason)int.MaxValue)), "Every reason this version knows has a sentence of its own.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AStepWithItsToolsReadsPagesAndKnowsTheResearch()
|
||||
{
|
||||
var text = BflToolStatus.ForStep(EVERYTHING);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(text, Does.Contain("Du kannst Webseiten lesen"));
|
||||
Assert.That(text, Does.Contain("Im Web suchen kannst du in diesem Gespräch nie"));
|
||||
Assert.That(text, Does.Contain("Mit der aktuellen Einrichtung ist die Recherche verfügbar."));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AStepWithoutItsToolsLearnsWhy()
|
||||
{
|
||||
var text = BflToolStatus.ForStep(NOTHING);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(text, Does.Contain("Webseiten lesen kannst du in diesem Gespräch nicht."));
|
||||
Assert.That(text, Does.Contain(BflToolStatus.ReasonFor(ToolOfferBlockReason.TOOL_SWITCHED_OFF_BY_USER)));
|
||||
Assert.That(text, Does.Contain("Mit der aktuellen Einrichtung ist die Recherche nicht verfügbar."));
|
||||
Assert.That(text, Does.Contain(BflToolStatus.ReasonFor(ToolOfferBlockReason.NOT_CONFIGURED)));
|
||||
Assert.That(text, Does.Not.Contain("Du kannst Webseiten lesen"));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheOnboardingTellsWhatTheLabCanDoWithTheCurrentSetup()
|
||||
{
|
||||
var everything = BflToolStatus.ForOnboarding(EVERYTHING);
|
||||
var nothing = BflToolStatus.ForOnboarding(NOTHING);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(everything, Does.Contain("kann das Lab im Web recherchieren"));
|
||||
Assert.That(everything, Does.Contain("können außerdem Webseiten lesen"));
|
||||
Assert.That(nothing, Does.Contain("kann das Lab nicht im Web recherchieren"));
|
||||
Assert.That(nothing, Does.Contain(BflToolStatus.ReasonFor(ToolOfferBlockReason.NOT_CONFIGURED)));
|
||||
Assert.That(nothing, Does.Contain(BflToolStatus.ReasonFor(ToolOfferBlockReason.TOOL_SWITCHED_OFF_BY_USER)));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheStepsAndTheOnboardingKnowThatTheLabNeverResearchesAnEventWhichIsNotPublic()
|
||||
{
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(BflToolStatus.ForStep(EVERYTHING), Does.Contain("Ist die Veranstaltung nicht öffentlich, recherchiert das Lab nie im Web"));
|
||||
Assert.That(BflToolStatus.ForStep(NOTHING), Does.Contain("Ist die Veranstaltung nicht öffentlich, recherchiert das Lab nie im Web"));
|
||||
Assert.That(BflToolStatus.ForOnboarding(EVERYTHING), Does.Contain("Ist die Veranstaltung nicht öffentlich, recherchiert das Lab nie im Web"));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryTextNamesExactlyTheDetailsTheResearchReceives()
|
||||
{
|
||||
var texts = new[] { BflToolStatus.ForStep(EVERYTHING), BflToolStatus.ForStep(NOTHING), BflToolStatus.ForOnboarding(EVERYTHING), BflToolStatus.ForOnboarding(NOTHING) };
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
foreach (var field in BflFieldCatalog.FIELDS)
|
||||
{
|
||||
foreach (var text in texts)
|
||||
{
|
||||
if (field.IsSharedWithResearch)
|
||||
Assert.That(text, Does.Contain($"„{field.Label}“"), field.Id);
|
||||
else
|
||||
Assert.That(text, Does.Not.Contain($"„{field.Label}“"), field.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheSameSetupGivesTheSamePrompt()
|
||||
{
|
||||
var parts = new BflPromptParts("{{TOOLS}}", "Leitfaden", "Schritt", "Anrede");
|
||||
var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData());
|
||||
|
||||
var first = BflPromptComposer.Compose(parts, BflStep.STRATEGY, forms, [], "Deutsch", null, new BflResearchAvailability(ToolOfferBlockReason.NONE, ToolOfferBlockReason.NONE));
|
||||
var second = BflPromptComposer.Compose(parts, BflStep.STRATEGY, forms, [], "Deutsch", null, new BflResearchAvailability(ToolOfferBlockReason.NONE, ToolOfferBlockReason.NONE));
|
||||
var other = BflPromptComposer.Compose(parts, BflStep.STRATEGY, forms, [], "Deutsch", null, NOTHING);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(second, Is.EqualTo(first));
|
||||
Assert.That(other, Is.Not.EqualTo(first));
|
||||
Assert.That(first, Does.Not.Contain("\r"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ public sealed class BflWorkStateRequestTests
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheRequestOffersSuggestionsOnlyForEmptyTextFields()
|
||||
public void TheRequestOffersSuggestionsOnlyForEmptyFields()
|
||||
{
|
||||
var form = new BflStepFormData();
|
||||
form.Set(BflFieldCatalog.FORMAT, "Podium");
|
||||
@@ -41,6 +41,21 @@ public sealed class BflWorkStateRequestTests
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheRequestNamesTheOptionsOfAnEmptyChoice()
|
||||
{
|
||||
var difficulty = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.DIFFICULTY);
|
||||
|
||||
var request = BflPromptComposer.ComposeWorkStateRequest("{{FIELDS}}", BflStep.TEST_SITUATION, new BflStepFormData());
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(request, Does.Contain($"[{BflMarkdownSections.FIELD_PREFIX}{BflFieldCatalog.DIFFICULTY}]"));
|
||||
foreach (var option in difficulty.Options)
|
||||
Assert.That(request, Does.Contain($"„{option.Label}“"), option.Id);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARequestWithoutEmptyFieldsAsksForNoSuggestions()
|
||||
{
|
||||
|
||||
@@ -61,12 +61,13 @@ public static partial class JsonTestRoots
|
||||
yield return Case(new(typeof(IReadOnlyList<string>), SqliteProvenanceStoreClientImplementation.JSON_OPTIONS));
|
||||
yield return Case(new(typeof(List<string>), SqliteProvenanceStoreClientImplementation.JSON_OPTIONS));
|
||||
|
||||
// The preparations of the BeyondFacts Lab, their forms, materials, work states, and results, and the preparation selected last:
|
||||
// The preparations of the BeyondFacts Lab, their forms, materials, work states, results, and research, and the preparation selected last:
|
||||
yield return Case(new(typeof(BflProject), BeyondFactsLabJson.PERSISTENCE));
|
||||
yield return Case(new(typeof(BflStepFormData), BeyondFactsLabJson.PERSISTENCE));
|
||||
yield return Case(new(typeof(BflMaterial), BeyondFactsLabJson.PERSISTENCE));
|
||||
yield return Case(new(typeof(BflWorkState), BeyondFactsLabJson.PERSISTENCE));
|
||||
yield return Case(new(typeof(BflResult), BeyondFactsLabJson.PERSISTENCE));
|
||||
yield return Case(new(typeof(BflResearch), BeyondFactsLabJson.PERSISTENCE));
|
||||
yield return Case(new(typeof(Guid?), BeyondFactsLabJson.PERSISTENCE), "selected preparation");
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user