Added profile, provider, and workspace preselection to chat templates (#1099)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Peer HogeterpandThorsten Sommer authored and GitHub committed 2026-10-22 14:15:10 +02:00
1 parent ab9df99cf0
commit 24c4c31035
34 files changed
+2084 -142

No files matched your search

@@ -7,8 +7,9 @@ using Lua.Standard;
namespace AIStudio.Tests.Settings;
/// <summary>
/// Checks the tools and data sources a chat template carries, across the two surfaces which have
/// to agree on them: the Lua a configuration plugin states, and the Lua the app exports.
/// Checks the profile, provider, workspace, tools, and data sources a chat template carries, across
/// the two surfaces which have to agree on them: the Lua a configuration plugin states, and the Lua
/// the app exports.
/// </summary>
/// <remarks>
/// The interesting part is not that a value survives, but that the difference between "this
@@ -88,6 +89,9 @@ public sealed class ChatTemplateConfigurationTests
{
Assert.That(luaCode, Does.Not.Contain("ToolIds"));
Assert.That(luaCode, Does.Not.Contain("DataSourceOptions"));
Assert.That(luaCode, Does.Not.Contain("ProviderId"));
Assert.That(luaCode, Does.Not.Contain("ProfileId"));
Assert.That(luaCode, Does.Not.Contain("WorkspaceName"));
});
var read = await ParseAsync(luaCode);
@@ -189,6 +193,140 @@ public sealed class ChatTemplateConfigurationTests
});
}
[Test]
public async Task APreselectedProviderAndProfileSurviveTheExport()
{
var written = NewTemplate() with
{
ProviderId = "44444444-4444-4444-4444-444444444444",
ProfileId = "55555555-5555-5555-5555-555555555555",
};
var read = await ExportAndReadBackAsync(written);
Assert.Multiple(() =>
{
Assert.That(read.ProviderId, Is.EqualTo(written.ProviderId));
Assert.That(read.ProfileId, Is.EqualTo(written.ProfileId));
});
}
[Test]
public async Task StartingWithoutAProfileIsAStatementOfItsOwn()
{
var read = await ExportAndReadBackAsync(NewTemplate() with { ProfileId = Profile.NO_PROFILE.Id });
var silent = await ExportAndReadBackAsync(NewTemplate());
Assert.Multiple(() =>
{
Assert.That(read.ProfileId, Is.EqualTo(Profile.NO_PROFILE.Id), "Reading this back as null would hand out the default profile the template ruled out.");
Assert.That(silent.ProfileId, Is.Null);
Assert.That(silent.ProviderId, Is.Null);
});
}
[Test]
public async Task APluginTemplateMayNameTheProviderAndProfileOfItsPlugin()
{
var read = await ParseAsync("""
CONFIG["CHAT_TEMPLATES"][#CONFIG["CHAT_TEMPLATES"]+1] = {
["Id"] = "33333333-3333-3333-3333-333333333333",
["Name"] = "Mail",
["SystemPrompt"] = "You help with mails.",
["AllowProfileUsage"] = true,
["ProviderId"] = "44444444-4444-4444-4444-444444444444",
["ProfileId"] = "55555555-5555-5555-5555-555555555555",
}
""");
Assert.Multiple(() =>
{
Assert.That(read.ProviderId, Is.EqualTo("44444444-4444-4444-4444-444444444444"));
Assert.That(read.ProfileId, Is.EqualTo("55555555-5555-5555-5555-555555555555"));
});
}
[TestCase("\"\"")]
[TestCase("\"no GUID\"")]
[TestCase("42")]
[TestCase("\"00000000-0000-0000-0000-000000000000\"")]
public async Task AProviderIdWhichNamesNoProviderIsDropped(string luaValue)
{
var read = await ParseAsync($$"""
CONFIG["CHAT_TEMPLATES"][#CONFIG["CHAT_TEMPLATES"]+1] = {
["Id"] = "33333333-3333-3333-3333-333333333333",
["Name"] = "Mail",
["SystemPrompt"] = "You help with mails.",
["ProviderId"] = {{luaValue}},
}
""");
Assert.That(read.ProviderId, Is.Null, "The template still works then, with the provider its chats would start with anyway.");
}
[Test]
public async Task AWorkspaceNameSurvivesTheExport()
{
var written = NewTemplate() with { WorkspaceName = "Mails \"inbox\"" };
var read = await ExportAndReadBackAsync(written);
var silent = await ExportAndReadBackAsync(NewTemplate());
Assert.Multiple(() =>
{
Assert.That(read.WorkspaceName, Is.EqualTo(written.WorkspaceName));
Assert.That(silent.WorkspaceName, Is.Null, "A template without a workspace must keep leaving the chat where it would go anyway.");
});
}
[TestCase("\"\"")]
[TestCase("\" \"")]
[TestCase("42")]
[TestCase("{}")]
public async Task AWorkspaceNameWhichNamesNothingIsDropped(string luaValue)
{
var read = await ParseAsync($$"""
CONFIG["CHAT_TEMPLATES"][#CONFIG["CHAT_TEMPLATES"]+1] = {
["Id"] = "33333333-3333-3333-3333-333333333333",
["Name"] = "Mail",
["SystemPrompt"] = "You help with mails.",
["WorkspaceName"] = {{luaValue}},
}
""");
Assert.That(read.WorkspaceName, Is.Null);
}
[Test]
public async Task APluginTemplateNamesItsWorkspaceWithoutTheSpacesAround()
{
var read = await ParseAsync("""
CONFIG["CHAT_TEMPLATES"][#CONFIG["CHAT_TEMPLATES"]+1] = {
["Id"] = "33333333-3333-3333-3333-333333333333",
["Name"] = "Mail",
["SystemPrompt"] = "You help with mails.",
["WorkspaceName"] = " Mails ",
}
""");
Assert.That(read.WorkspaceName, Is.EqualTo("Mails"), "Workspaces are matched by their trimmed name, so the spaces would only make the field look different from the name in the list.");
}
[Test]
public void ExportedProviderAndProfileIdsComeWithTheNoteThatTheyAreLocalOnes()
{
var withProvider = NewTemplate() with { ProviderId = "44444444-4444-4444-4444-444444444444" };
var withoutProfile = NewTemplate() with { ProfileId = Profile.NO_PROFILE.Id };
Assert.That(withProvider.TryExportAsConfigurationSection(out var withProviderLua, out var issue), Is.True, issue);
Assert.That(withoutProfile.TryExportAsConfigurationSection(out var withoutProfileLua, out issue), Is.True, issue);
Assert.Multiple(() =>
{
Assert.That(withProviderLua, Does.StartWith("--"), "A provider of this machine is mostly the user's own, which no configuration plugin knows.");
Assert.That(withoutProfileLua, Does.Not.StartWith("--"), "Starting without a profile names nothing which could be missing.");
});
}
[Test]
public void ExportedDataSourceIdsComeWithTheNoteThatTheyAreLocalOnes()
{
@@ -0,0 +1,154 @@
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tests.Tools.ToolCalling;
using AIStudio.Tools.Services;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Settings;
/// <summary>
/// Checks who decides the provider and the profile when an assistant chat launcher and the chat
/// template it opens its chat with both name one.
/// </summary>
/// <remarks>
/// The rule is the one for the tools and the data sources: the chat template wins. Whatever the
/// template names and cannot be used must not stop the launch, though, because the author of the
/// launcher could not fix it in the launcher. The launcher then decides as if the template named none.
/// The last tests cover the data sources, which only stop a launch while the chat can use them.
/// </remarks>
[TestFixture]
[NonParallelizable]
public sealed class ChatTemplateLauncherSelectionTests : ToolRegistryTestBase
{
private const string PLUGIN_NAME = "Mail assistant";
private const string TEMPLATE_PROFILE_ID = "55555555-5555-5555-5555-555555555555";
private static readonly Guid LAUNCHER_PROVIDER_ID = Guid.Parse("11111111-1111-1111-1111-111111111111");
private static readonly Guid LAUNCHER_PROFILE_ID = Guid.Parse("22222222-2222-2222-2222-222222222222");
[Test]
public void TheTemplateWinsOverTheLauncher()
{
var templateProvider = ToolCapableProvider();
this.SettingsManager.ConfigurationData.Providers.Add(templateProvider);
this.SettingsManager.ConfigurationData.Providers.Add(LauncherProvider());
var templateProfile = this.AddProfile(TEMPLATE_PROFILE_ID);
this.AddProfile(LAUNCHER_PROFILE_ID.ToString());
var template = NewTemplate() with { ProviderId = templateProvider.Id, ProfileId = TEMPLATE_PROFILE_ID };
var service = this.CreateService();
var providerResult = service.ResolveProvider(PLUGIN_NAME, template, LAUNCHER_PROVIDER_ID);
var profileResult = service.ResolveProfile(PLUGIN_NAME, template, LAUNCHER_PROFILE_ID);
Assert.Multiple(() =>
{
Assert.That(providerResult.Provider, Is.EqualTo(templateProvider));
Assert.That(providerResult.IsExplicit, Is.True);
Assert.That(profileResult.Profile, Is.EqualTo(templateProfile));
});
}
[Test]
public void WhatTheTemplateNamesAndIsGoneLeavesTheChoiceToTheLauncher()
{
var launcherProvider = LauncherProvider();
this.SettingsManager.ConfigurationData.Providers.Add(launcherProvider);
var launcherProfile = this.AddProfile(LAUNCHER_PROFILE_ID.ToString());
var template = NewTemplate() with { ProviderId = "66666666-6666-6666-6666-666666666666", ProfileId = TEMPLATE_PROFILE_ID };
var service = this.CreateService();
var providerResult = service.ResolveProvider(PLUGIN_NAME, template, LAUNCHER_PROVIDER_ID);
var profileResult = service.ResolveProfile(PLUGIN_NAME, template, LAUNCHER_PROFILE_ID);
Assert.Multiple(() =>
{
Assert.That(providerResult.Provider, Is.EqualTo(launcherProvider));
Assert.That(providerResult.ErrorMessage, Is.Empty);
Assert.That(profileResult.Profile, Is.EqualTo(launcherProfile));
Assert.That(profileResult.ErrorMessage, Is.Empty);
});
}
[Test]
public void ATemplateProviderBelowTheConfidenceOfTheChatDoesNotStopTheLaunch()
{
var templateProvider = LessTrustedProvider();
this.SettingsManager.ConfigurationData.Providers.Add(templateProvider);
this.SettingsManager.ConfigurationData.Confidence.EnforceGlobalMinimumConfidence = true;
this.SettingsManager.ConfigurationData.Confidence.GlobalMinimumConfidence = ConfidenceLevel.HIGH;
var template = NewTemplate() with { ProviderId = templateProvider.Id };
var providerResult = this.CreateService().ResolveProvider(PLUGIN_NAME, template, null);
Assert.Multiple(() =>
{
Assert.That(providerResult.Provider, Is.Not.EqualTo(templateProvider), "A template must not open the way to a provider the user could not pick by hand.");
Assert.That(providerResult.IsExplicit, Is.False, "The launcher names no provider, so the chat starts with its defaults instead of failing.");
Assert.That(providerResult.ErrorMessage, Is.Empty);
});
}
[Test]
public void ATemplateWhichForbidsProfilesLeavesTheProfileToTheLauncher()
{
this.AddProfile(TEMPLATE_PROFILE_ID);
var launcherProfile = this.AddProfile(LAUNCHER_PROFILE_ID.ToString());
var template = NewTemplate() with { AllowProfileUsage = false, ProfileId = TEMPLATE_PROFILE_ID };
var profileResult = this.CreateService().ResolveProfile(PLUGIN_NAME, template, LAUNCHER_PROFILE_ID);
Assert.That(profileResult.Profile, Is.EqualTo(launcherProfile), "The launch drops the profile afterward for such a template; picking the one of the template here would only hide that.");
}
[Test]
public async Task WithoutRagAMissingDataSourceDoesNotStopTheLaunch()
{
var options = new DataSourceOptions { DisableDataSources = false, PreselectedDataSourceIds = ["99999999-9999-9999-9999-999999999999"] };
var result = await this.CreateService().CheckChosenDataSourcesAsync(ToolCapableProvider(), options, originChatTemplate: null);
Assert.Multiple(() =>
{
Assert.That(result.Options, Is.SameAs(options), "The chat uses no data sources without RAG, so the options travel along unchecked and count once RAG is back.");
Assert.That(result.ErrorMessage, Is.Empty);
});
}
[Test]
public async Task WithRagAMissingDataSourceStillStopsTheLaunch()
{
this.SettingsManager.ConfigurationData.App.EnabledPreviewFeatures.Add(PreviewFeatures.PRE_RAG_2024);
var options = new DataSourceOptions { DisableDataSources = false, PreselectedDataSourceIds = ["99999999-9999-9999-9999-999999999999"] };
var result = await this.CreateService().CheckChosenDataSourcesAsync(ToolCapableProvider(), options, originChatTemplate: null);
Assert.Multiple(() =>
{
Assert.That(result.Options, Is.Null, "Opening a launcher is one click, so a missing source has to be said out loud.");
Assert.That(result.ErrorMessage, Is.Not.Empty);
});
}
// Data sources are not resolved by these tests beyond their lookup, so the service needs no data source service:
private DirectChatService CreateService() => new(this.SettingsManager, dataSourceService: null!, toolRegistry: this.CreateRegistry(), NullLogger<DirectChatService>.Instance);
private static AIStudio.Settings.Provider LauncherProvider() => LessTrustedProvider() with { Id = LAUNCHER_PROVIDER_ID.ToString() };
private Profile AddProfile(string profileId)
{
var profile = new Profile((uint)this.SettingsManager.ConfigurationData.Profiles.Count + 1, profileId, $"Profile {profileId[..4]}", "I read a lot of mails.", "Answer briefly.");
this.SettingsManager.ConfigurationData.Profiles.Add(profile);
return profile;
}
private static ChatTemplate NewTemplate() => new()
{
Num = 1,
Id = "33333333-3333-3333-3333-333333333333",
Name = "Mail",
SystemPrompt = "You help with mails.",
PredefinedUserPrompt = string.Empty,
ExampleConversation = [],
FileAttachments = [],
AllowProfileUsage = true,
};
}
@@ -24,10 +24,8 @@ public sealed class ChatTemplatePrecedenceTests
Assert.Multiple(() =>
{
Assert.That(toolChoice.ToolIds, Is.Null, "Nobody named any tool, so the chat has to keep using the tools of its own default.");
Assert.That(toolChoice.LauncherChoiceDropped, Is.False);
Assert.That(optionsChoice.Options, Is.Null, "Nobody named any data source, so the chat has to keep using its own default options.");
Assert.That(optionsChoice.LauncherChoiceDropped, Is.False);
Assert.That(toolChoice, Is.EqualTo(ChatTemplateToolChoice.NONE), "Nobody named any tool, so the chat has to keep using the tools of its own default.");
Assert.That(optionsChoice, Is.EqualTo(ChatTemplateDataSourceChoice.NONE), "Nobody named any data source, so the chat has to keep using its own default options.");
});
}
@@ -0,0 +1,268 @@
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tests.Tools.ToolCalling;
namespace AIStudio.Tests.Settings;
/// <summary>
/// Checks which profile and provider of a chat template a chat actually starts with.
/// </summary>
/// <remarks>
/// A preselection must never get past what the user would be allowed to pick by hand: a profile the
/// template forbids, or a provider below the confidence level of the chat. And it must never stop a
/// chat either: whatever the template names and cannot be used leaves the chat to start as if the
/// template said nothing.
/// </remarks>
[TestFixture]
[NonParallelizable]
public sealed class ChatTemplateStartSelectionTests : ToolRegistryTestBase
{
private const string PROFILE_ID = "55555555-5555-5555-5555-555555555555";
private const string DEFAULT_PROFILE_ID = "77777777-7777-7777-7777-777777777777";
private const string OTHER_TEMPLATE_ID = "88888888-8888-8888-8888-888888888888";
[Test]
public void ATemplateWhichSaysNothingLeavesBothInPlace()
{
this.AddProfile();
this.SettingsManager.ConfigurationData.Providers.Add(ToolCapableProvider());
Assert.Multiple(() =>
{
Assert.That(this.SettingsManager.GetChatTemplateProfile(NewTemplate()), Is.Null);
Assert.That(this.SettingsManager.GetChatTemplateProvider(NewTemplate()), Is.EqualTo(AIStudio.Settings.Provider.NONE));
});
}
[Test]
public void ATemplateHandsOutWhatItNames()
{
var profile = this.AddProfile();
var provider = ToolCapableProvider();
this.SettingsManager.ConfigurationData.Providers.Add(provider);
var template = NewTemplate() with { ProfileId = PROFILE_ID, ProviderId = provider.Id };
Assert.Multiple(() =>
{
Assert.That(this.SettingsManager.GetChatTemplateProfile(template), Is.EqualTo(profile));
Assert.That(this.SettingsManager.GetChatTemplateProvider(template), Is.EqualTo(provider));
});
}
[Test]
public void StartingWithoutAProfileIsAnAnswerOfItsOwn()
{
var template = NewTemplate() with { ProfileId = Profile.NO_PROFILE.Id };
Assert.That(this.SettingsManager.GetChatTemplateProfile(template), Is.EqualTo(Profile.NO_PROFILE), "Null would keep the default profile, which is exactly what this template rules out.");
}
[Test]
public void WhatIsGoneLeavesTheChatAsIfTheTemplateSaidNothing()
{
var template = NewTemplate() with { ProfileId = PROFILE_ID, ProviderId = "66666666-6666-6666-6666-666666666666" };
Assert.Multiple(() =>
{
Assert.That(this.SettingsManager.GetChatTemplateProfile(template), Is.Null);
Assert.That(this.SettingsManager.GetChatTemplateProvider(template), Is.EqualTo(AIStudio.Settings.Provider.NONE));
});
}
[Test]
public void AChatWhoseTemplateProfileIsGoneStartsWithTheDefaultProfile()
{
var template = NewTemplate() with { ProfileId = PROFILE_ID };
var withoutDefault = this.SettingsManager.GetChatTemplateStartProfile(template);
var defaultProfile = new Profile(2, "77777777-7777-7777-7777-777777777777", "Default", "I am the default.", string.Empty);
this.SettingsManager.ConfigurationData.Profiles.Add(defaultProfile);
this.SettingsManager.ConfigurationData.Chat.PreselectOptions = true;
this.SettingsManager.ConfigurationData.Chat.PreselectedProfile = defaultProfile.Id;
var withDefault = this.SettingsManager.GetChatTemplateStartProfile(template);
Assert.Multiple(() =>
{
Assert.That(withoutDefault, Is.EqualTo(Profile.NO_PROFILE), "Keeping the profile the chat had would hand out one the previous chat happened to use, a choice nobody made for this one.");
Assert.That(withDefault, Is.EqualTo(defaultProfile));
});
}
[Test]
public void ATemplateWhichSaysNothingKeepsTheProfileAChatStartsWith()
{
this.AddProfile();
Assert.Multiple(() =>
{
Assert.That(this.SettingsManager.GetChatTemplateStartProfile(NewTemplate()), Is.Null);
Assert.That(this.SettingsManager.GetChatTemplateStartProfile(NewTemplate() with { AllowProfileUsage = false, ProfileId = PROFILE_ID }), Is.Null);
Assert.That(this.SettingsManager.GetChatTemplateStartProfile(NewTemplate() with { ProfileId = PROFILE_ID })!.Id, Is.EqualTo(PROFILE_ID));
});
}
[Test]
public void ATemplateWhichForbidsProfilesHandsOutNone()
{
this.AddProfile();
var template = NewTemplate() with { AllowProfileUsage = false, ProfileId = PROFILE_ID };
Assert.That(this.SettingsManager.GetChatTemplateProfile(template), Is.Null, "The chat locks the profile selection for such a template, so a profile handed out here could neither be seen nor changed.");
}
[Test]
public void AProviderBelowTheConfidenceOfTheChatIsNotHandedOut()
{
var provider = LessTrustedProvider();
this.SettingsManager.ConfigurationData.Providers.Add(provider);
this.SettingsManager.ConfigurationData.Confidence.EnforceGlobalMinimumConfidence = true;
this.SettingsManager.ConfigurationData.Confidence.GlobalMinimumConfidence = ConfidenceLevel.HIGH;
var template = NewTemplate() with { ProviderId = provider.Id };
Assert.That(this.SettingsManager.GetChatTemplateProvider(template), Is.EqualTo(AIStudio.Settings.Provider.NONE), "A template must not open the way to a provider the user could not pick by hand.");
}
[Test]
public void AProviderIdFollowsTheSameRuleAsTheTemplateWhichNamesIt()
{
var trustedProvider = ToolCapableProvider();
var lessTrustedProvider = LessTrustedProvider();
this.SettingsManager.ConfigurationData.Providers.Add(trustedProvider);
this.SettingsManager.ConfigurationData.Providers.Add(lessTrustedProvider);
this.SettingsManager.ConfigurationData.Confidence.EnforceGlobalMinimumConfidence = true;
this.SettingsManager.ConfigurationData.Confidence.GlobalMinimumConfidence = ConfidenceLevel.HIGH;
Assert.Multiple(() =>
{
Assert.That(this.SettingsManager.GetChatTemplateProvider(trustedProvider.Id), Is.EqualTo(trustedProvider));
Assert.That(this.SettingsManager.GetChatTemplateProvider(lessTrustedProvider.Id), Is.EqualTo(AIStudio.Settings.Provider.NONE), "The chat template dialog and the import warn about exactly the providers a chat leaves out.");
Assert.That(this.SettingsManager.GetChatTemplateProvider("66666666-6666-6666-6666-666666666666"), Is.EqualTo(AIStudio.Settings.Provider.NONE));
Assert.That(this.SettingsManager.GetChatTemplateProvider((string?)null), Is.EqualTo(AIStudio.Settings.Provider.NONE));
});
}
[Test]
public void ATemplateNamesItsWorkspaceOnlyWhileWorkspacesAreUsed()
{
var template = NewTemplate() with { WorkspaceName = " Mails " };
this.SettingsManager.ConfigurationData.Workspace.StorageBehavior = WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY;
var whileUsed = this.SettingsManager.GetChatTemplateWorkspaceName(template);
this.SettingsManager.ConfigurationData.Workspace.StorageBehavior = WorkspaceStorageBehavior.DISABLE_WORKSPACES;
var whileSwitchedOff = this.SettingsManager.GetChatTemplateWorkspaceName(template);
Assert.Multiple(() =>
{
Assert.That(whileUsed, Is.EqualTo("Mails"));
Assert.That(whileSwitchedOff, Is.Null, "Without workspaces, a template must not bring one back for its chats.");
Assert.That(this.SettingsManager.GetChatTemplateWorkspaceName(NewTemplate()), Is.Null);
});
}
[Test]
public void WhatThePreviousTemplatePreselectedIsNotCarriedOver()
{
var templateProfile = this.AddProfile();
var defaultProfile = this.AddDefaultProfile();
var templateProvider = ToolCapableProvider();
var defaultProvider = LessTrustedProvider();
this.SettingsManager.ConfigurationData.Providers.Add(templateProvider);
this.SettingsManager.ConfigurationData.Providers.Add(defaultProvider);
this.SettingsManager.ConfigurationData.Chat.PreselectedProvider = defaultProvider.Id;
var previousTemplate = NewTemplate() with { ProfileId = PROFILE_ID, ProviderId = templateProvider.Id };
var nextTemplate = NewTemplate() with { Id = OTHER_TEMPLATE_ID };
Assert.Multiple(() =>
{
Assert.That(this.SettingsManager.ChooseChatTemplateProfile(nextTemplate, previousTemplate, templateProfile), Is.EqualTo(defaultProfile), "The profile was the choice of another template, not one made for this chat.");
Assert.That(this.SettingsManager.ChooseChatTemplateProvider(nextTemplate, previousTemplate, templateProvider), Is.EqualTo(defaultProvider), "The provider was the choice of another template, not one made for this chat.");
});
}
[Test]
public void AChoiceMadeByHandStaysWhenTheTemplateSaysNothing()
{
var handPickedProfile = this.AddProfile();
var defaultProfile = this.AddDefaultProfile();
var templateProvider = ToolCapableProvider();
var handPickedProvider = LessTrustedProvider();
this.SettingsManager.ConfigurationData.Providers.Add(templateProvider);
this.SettingsManager.ConfigurationData.Providers.Add(handPickedProvider);
var previousTemplate = NewTemplate() with { ProfileId = defaultProfile.Id, ProviderId = templateProvider.Id };
var nextTemplate = NewTemplate() with { Id = OTHER_TEMPLATE_ID };
Assert.Multiple(() =>
{
Assert.That(this.SettingsManager.ChooseChatTemplateProfile(nextTemplate, previousTemplate, handPickedProfile), Is.EqualTo(handPickedProfile));
Assert.That(this.SettingsManager.ChooseChatTemplateProvider(nextTemplate, previousTemplate, handPickedProvider), Is.EqualTo(handPickedProvider));
Assert.That(this.SettingsManager.ChooseChatTemplateProfile(NewTemplate(), NewTemplate(), handPickedProfile), Is.EqualTo(handPickedProfile), "Without any preselection, nothing changes.");
});
}
[Test]
public void TheNewTemplateDecidesWhenItNamesBoth()
{
var templateProfile = this.AddProfile();
var otherProfile = this.AddDefaultProfile();
var templateProvider = ToolCapableProvider();
var otherProvider = LessTrustedProvider();
this.SettingsManager.ConfigurationData.Providers.Add(templateProvider);
this.SettingsManager.ConfigurationData.Providers.Add(otherProvider);
var previousTemplate = NewTemplate() with { Id = OTHER_TEMPLATE_ID, ProfileId = otherProfile.Id, ProviderId = otherProvider.Id };
var nextTemplate = NewTemplate() with { ProfileId = PROFILE_ID, ProviderId = templateProvider.Id };
Assert.Multiple(() =>
{
Assert.That(this.SettingsManager.ChooseChatTemplateProfile(nextTemplate, previousTemplate, otherProfile), Is.EqualTo(templateProfile));
Assert.That(this.SettingsManager.ChooseChatTemplateProvider(nextTemplate, previousTemplate, otherProvider), Is.EqualTo(templateProvider));
});
}
[Test]
public void WithoutADefaultProviderTheChatKeepsItsProvider()
{
var templateProvider = ToolCapableProvider();
this.SettingsManager.ConfigurationData.Providers.Add(templateProvider);
this.SettingsManager.ConfigurationData.Providers.Add(LessTrustedProvider());
var previousTemplate = NewTemplate() with { ProviderId = templateProvider.Id };
var nextTemplate = NewTemplate() with { Id = OTHER_TEMPLATE_ID };
Assert.That(this.SettingsManager.ChooseChatTemplateProvider(nextTemplate, previousTemplate, templateProvider), Is.EqualTo(templateProvider), "A chat without any provider could not answer at all.");
}
private Profile AddDefaultProfile()
{
var defaultProfile = new Profile(2, DEFAULT_PROFILE_ID, "Default", "I am the default.", string.Empty);
this.SettingsManager.ConfigurationData.Profiles.Add(defaultProfile);
this.SettingsManager.ConfigurationData.Chat.PreselectOptions = true;
this.SettingsManager.ConfigurationData.Chat.PreselectedProfile = defaultProfile.Id;
return defaultProfile;
}
private Profile AddProfile()
{
var profile = new Profile(1, PROFILE_ID, "Mail", "I read a lot of mails.", "Answer briefly.");
this.SettingsManager.ConfigurationData.Profiles.Add(profile);
return profile;
}
private static ChatTemplate NewTemplate() => new()
{
Num = 1,
Id = "33333333-3333-3333-3333-333333333333",
Name = "Mail",
SystemPrompt = "You help with mails.",
PredefinedUserPrompt = string.Empty,
ExampleConversation = [],
FileAttachments = [],
AllowProfileUsage = true,
};
}
@@ -118,6 +118,9 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
Assert.That(imported.DataSourceOptions!.AutomaticDataSourceSelection, Is.EqualTo(template.DataSourceOptions!.AutomaticDataSourceSelection));
Assert.That(imported.DataSourceOptions.AutomaticValidation, Is.EqualTo(template.DataSourceOptions.AutomaticValidation));
Assert.That(imported.DataSourceOptions.PreselectedDataSourceIds, Is.EqualTo(template.DataSourceOptions.PreselectedDataSourceIds));
Assert.That(imported.ProviderId, Is.EqualTo(template.ProviderId));
Assert.That(imported.ProfileId, Is.EqualTo(template.ProfileId));
Assert.That(imported.WorkspaceName, Is.EqualTo(template.WorkspaceName));
});
}
@@ -417,14 +420,66 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
public async Task MissingReferencesOfAChatTemplateAreNamed()
{
var missingDataSourceId = Guid.NewGuid().ToString();
var snippet = ExportChatTemplate([], toolIds: [TOOL_ID, "missing_tool"], dataSourceIds: [missingDataSourceId]);
var missingProviderId = Guid.NewGuid().ToString();
var missingProfileId = Guid.NewGuid().ToString();
var snippet = ExportChatTemplate([], toolIds: [TOOL_ID, "missing_tool"], dataSourceIds: [missingDataSourceId], providerId: missingProviderId, profileId: missingProfileId);
var result = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(snippet, PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, this.CreateRegistry(new TestTool(Definition())));
Assert.Multiple(() =>
{
Assert.That(result.Hints, Has.Count.EqualTo(2), "One sentence for each kind of reference.");
Assert.That(result.Hints, Has.Count.EqualTo(4), "One sentence for each kind of reference.");
Assert.That(result.Hints, Has.Exactly(1).Contains(missingDataSourceId));
Assert.That(result.Hints, Has.Exactly(1).Contains("missing_tool"));
Assert.That(result.Hints, Has.Exactly(1).Contains(missingProviderId));
Assert.That(result.Hints, Has.Exactly(1).Contains(missingProfileId));
});
}
[Test]
public async Task AChatTemplateWhichStartsWithoutAProfileNamesNothingMissing()
{
var snippet = ExportChatTemplate([], profileId: Profile.NO_PROFILE.Id);
var result = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(snippet, PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, this.CreateRegistry(new TestTool(Definition())));
Assert.Multiple(() =>
{
Assert.That(result.Hints, Is.Empty, "Starting without a profile is a statement, not a reference to a profile which could be missing.");
Assert.That(((ChatTemplate)result.ConfigurationObject).ProfileId, Is.EqualTo(Profile.NO_PROFILE.Id));
});
}
[Test]
public async Task AProviderBelowTheConfidenceOfTheChatIsNamed()
{
// A chat template names its provider by a GUID, as every provider of the app has one:
var provider = LessTrustedProvider() with { Id = Guid.NewGuid().ToString() };
this.SettingsManager.ConfigurationData.Providers.Add(provider);
this.SettingsManager.ConfigurationData.Confidence.EnforceGlobalMinimumConfidence = true;
this.SettingsManager.ConfigurationData.Confidence.GlobalMinimumConfidence = ConfidenceLevel.HIGH;
var snippet = ExportChatTemplate([], providerId: provider.Id);
var result = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(snippet, PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, this.CreateRegistry(new TestTool(Definition())));
Assert.Multiple(() =>
{
Assert.That(result.Hints, Has.Count.EqualTo(1), "The provider exists, but chats started with the template would not use it.");
Assert.That(result.Hints, Has.Exactly(1).Contains(provider.InstanceName));
});
}
[Test]
public async Task AProfileOfATemplateWhichForbidsProfilesIsNamed()
{
var missingProfileId = Guid.NewGuid().ToString();
var template = ChatTemplateWith([], null, null, profileId: missingProfileId) with { AllowProfileUsage = false };
Assert.That(template.TryExportAsConfigurationSection(out var snippet, out var issue), Is.True, issue);
var result = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(snippet, PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, this.CreateRegistry(new TestTool(Definition())));
Assert.Multiple(() =>
{
Assert.That(result.Hints, Has.Count.EqualTo(1), "Whether the profile exists does not matter, since chats ignore it anyway.");
Assert.That(result.Hints, Has.None.Contains(missingProfileId));
});
}
@@ -440,7 +495,7 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
var registry = this.CreateRegistry(new TestTool(Definition()));
var policy = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(ExportPolicy(provider.Id, profile.Id, [TOOL_ID]), PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY), this.SettingsManager, registry);
var template = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(ExportChatTemplate([], [TOOL_ID], [dataSource.Id]), PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, registry);
var template = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(ExportChatTemplate([], [TOOL_ID], [dataSource.Id], provider.Id, profile.Id), PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, registry);
Assert.Multiple(() =>
{
@@ -589,7 +644,7 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
return (profile, profile.ExportAsConfigurationSection());
case PluginConfigurationObjectType.CHAT_TEMPLATE:
var template = ChatTemplateWith([], ["web_search"], [Guid.NewGuid().ToString()]);
var template = ChatTemplateWith([], ["web_search"], [Guid.NewGuid().ToString()], Guid.NewGuid().ToString(), Guid.NewGuid().ToString()) with { WorkspaceName = "Contracts" };
Assert.That(template.TryExportAsConfigurationSection(out var luaCode, out var issue), Is.True, issue);
return (template, luaCode);
@@ -677,14 +732,14 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
return (dataSource, dataSource.ExportAsConfigurationSection(encryptedSecret, usernamePasswordMode));
}
private static string ExportChatTemplate(IReadOnlyList<string> attachmentPaths, IReadOnlyList<string>? toolIds = null, IReadOnlyList<string>? dataSourceIds = null)
private static string ExportChatTemplate(IReadOnlyList<string> attachmentPaths, IReadOnlyList<string>? toolIds = null, IReadOnlyList<string>? dataSourceIds = null, string? providerId = null, string? profileId = null)
{
var template = ChatTemplateWith(attachmentPaths, toolIds, dataSourceIds);
var template = ChatTemplateWith(attachmentPaths, toolIds, dataSourceIds, providerId, profileId);
Assert.That(template.TryExportAsConfigurationSection(out var luaCode, out var issue), Is.True, issue);
return luaCode;
}
private static ChatTemplate ChatTemplateWith(IReadOnlyList<string> attachmentPaths, IReadOnlyList<string>? toolIds, IReadOnlyList<string>? dataSourceIds) => new()
private static ChatTemplate ChatTemplateWith(IReadOnlyList<string> attachmentPaths, IReadOnlyList<string>? toolIds, IReadOnlyList<string>? dataSourceIds, string? providerId = null, string? profileId = null) => new()
{
Num = 1,
Id = Guid.NewGuid().ToString(),
@@ -700,6 +755,8 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
],
ToolIds = toolIds?.ToHashSet(StringComparer.Ordinal),
DataSourceOptions = dataSourceIds is null ? null : new() { DisableDataSources = false, AutomaticValidation = true, PreselectedDataSourceIds = [..dataSourceIds] },
ProviderId = providerId,
ProfileId = profileId,
};
private static string ExportPolicy(string preselectedProvider = "", string preselectedProfile = "", IReadOnlyList<string>? allowedToolIds = null) => PolicyWith(preselectedProvider, preselectedProfile, allowedToolIds ?? []).ExportAsConfigurationSection();
@@ -270,8 +270,12 @@ public sealed class ConfigurationSnippetParserTests
],
ToolIds = ["web_search"],
// Preselected data sources make the export start with a comment:
// Preselected data sources make the export start with a comment, and so do a preselected
// provider and profile, with a comment of their own:
DataSourceOptions = new() { DisableDataSources = false, PreselectedDataSourceIds = [Guid.NewGuid().ToString()] },
ProviderId = Guid.NewGuid().ToString(),
ProfileId = Guid.NewGuid().ToString(),
WorkspaceName = QUOTED_TEXT,
};
Assert.That(template.TryExportAsConfigurationSection(out var luaCode, out var issue), Is.True, issue);
@@ -0,0 +1,100 @@
using AIStudio.Settings;
using AIStudio.Tools;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Checks how a chat finds the workspace its chat template names.
/// </summary>
/// <remarks>
/// A chat template names its workspace by name, because a workspace has an ID of its own on every
/// computer. The chat page and assistant chat launchers both go through the same method, so it has
/// to find the workspace the user sees in the list under that name, create it only when it is
/// missing, and never stop a chat over a name which names nothing.
/// </remarks>
[TestFixture]
[NonParallelizable]
public sealed class WorkspaceChatTemplateResolutionTests
{
private const string CHAT_TEMPLATE_NAME = "Mail";
private string? previousDataDirectory;
private string testDirectory = string.Empty;
[SetUp]
public async Task PrepareDataDirectory()
{
//
// The data directory and the workspace cache are static state of the whole application,
// which is why this fixture does not run alongside others, see WorkspaceTreeCacheTests:
//
this.previousDataDirectory = SettingsManager.DataDirectory;
this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-workspace-template-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.testDirectory);
SettingsManager.DataDirectory = this.testDirectory;
WorkspaceBehaviour.TestDataDirectory = this.testDirectory;
await WorkspaceBehaviour.ForceReloadWorkspaceTreeAsync();
}
[TearDown]
public void RemoveDataDirectory()
{
SettingsManager.DataDirectory = this.previousDataDirectory;
WorkspaceBehaviour.TestDataDirectory = null;
// Whoever reads the cache next must not get the workspaces of this test:
WorkspaceBehaviour.InvalidateWorkspaceTreeCache();
try
{
Directory.Delete(this.testDirectory, true);
}
catch (IOException)
{
// A temporary directory we could not remove says nothing about the code under test.
}
}
[Test]
public async Task AnExistingWorkspaceIsFoundByItsNameWithoutSpacesInAnyCase()
{
var created = await WorkspaceBehaviour.TryCreateWorkspaceAsync("Mails");
Assert.That(created.Success, Is.True);
var workspaceId = await WorkspaceBehaviour.ResolveOrCreateChatTemplateWorkspaceAsync(" mails ", CHAT_TEMPLATE_NAME);
var snapshot = await WorkspaceBehaviour.GetOrLoadWorkspaceTreeShellAsync();
Assert.Multiple(() =>
{
Assert.That(workspaceId, Is.EqualTo(created.Workspace.WorkspaceId), "The template names the workspace the user sees in the list under that name.");
Assert.That(snapshot.Workspaces, Has.Count.EqualTo(1), "A second workspace of the same name would split the chats of the template.");
});
}
[Test]
public async Task AMissingWorkspaceIsCreated()
{
var workspaceId = await WorkspaceBehaviour.ResolveOrCreateChatTemplateWorkspaceAsync("Contracts", CHAT_TEMPLATE_NAME);
var snapshot = await WorkspaceBehaviour.GetOrLoadWorkspaceTreeShellAsync();
Assert.Multiple(() =>
{
Assert.That(workspaceId, Is.Not.EqualTo(Guid.Empty));
Assert.That(snapshot.Workspaces.Single(workspace => workspace.WorkspaceId == workspaceId).Name, Is.EqualTo("Contracts"));
});
}
[Test]
public async Task ANameWhichNamesNothingGivesNoWorkspace()
{
var workspaceId = await WorkspaceBehaviour.ResolveOrCreateChatTemplateWorkspaceAsync(" ", CHAT_TEMPLATE_NAME);
var snapshot = await WorkspaceBehaviour.GetOrLoadWorkspaceTreeShellAsync();
Assert.Multiple(() =>
{
Assert.That(workspaceId, Is.EqualTo(Guid.Empty), "The chat then starts without a workspace, as it would without the template.");
Assert.That(snapshot.Workspaces, Is.Empty, "A workspace without a name would only show up as an unnamed one in the list.");
});
}
}