using AIStudio.Chat; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tests.Tools.ToolCalling; using AIStudio.Tools; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Services; using Microsoft.Extensions.Logging.Abstractions; using SharedTools; using AuthMethod = AIStudio.Tools.ERIClient.DataModel.AuthMethod; using Host = AIStudio.Provider.SelfHosted.Host; namespace AIStudio.Tests.Tools.PluginSystem; /// /// Checks that an imported configuration snippet becomes a configuration object of the user's own, /// which keeps what the export wrote and leaves out what does not exist on this computer. /// /// /// The exports encrypt their secrets with an encryption secret of the test. PluginFactory holds it /// for the whole process, as the app does, so this fixture must not run in parallel with others. /// Nothing here asserts on the wording of a hint or an issue, see ConfigurationSnippetParserTests. /// [TestFixture] [NonParallelizable] public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase { private const string API_KEY = "sk-imported-key"; private const string TOKEN = "eri-imported-token"; private static readonly PluginConfigurationObjectType[] IMPORTABLE_TYPES = [ PluginConfigurationObjectType.LLM_PROVIDER, PluginConfigurationObjectType.EMBEDDING_PROVIDER, PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, PluginConfigurationObjectType.PROFILE, PluginConfigurationObjectType.CHAT_TEMPLATE, PluginConfigurationObjectType.DATA_SOURCE, PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, ]; private string temporaryDirectory = string.Empty; private string dataDirectory = string.Empty; private string? previousDataDirectory; [SetUp] public void PrepareEncryptionAndFiles() { PluginFactory.InitializeEnterpriseEncryption(EnterpriseEncryption.GenerateSecret()); this.temporaryDirectory = Directory.CreateTempSubdirectory("snippet-import-").FullName; // The data directory is static state of the whole application, too. An exported tokenizer is found below it: this.previousDataDirectory = SettingsManager.DataDirectory; this.dataDirectory = Path.Combine(this.temporaryDirectory, "data"); SettingsManager.DataDirectory = this.dataDirectory; } [TearDown] public void RemoveEncryptionAndFiles() { RemoveEncryptionSecret(); SettingsManager.DataDirectory = this.previousDataDirectory; Directory.Delete(this.temporaryDirectory, recursive: true); } [TestCaseSource(nameof(IMPORTABLE_TYPES))] public void EveryExportBecomesAnObjectOfTheUsersOwn(PluginConfigurationObjectType type) { var (original, snippet) = Export(type); var imported = Import(snippet, type).ConfigurationObject; Assert.Multiple(() => { Assert.That(Guid.TryParse(imported.Id, out _), Is.True, "Every configuration object needs a GUID as its ID."); Assert.That(imported.Id, Is.Not.EqualTo(original.Id).And.Not.EqualTo(ExportedId(snippet, type)), "Importing the same snippet twice must not create two objects with one ID."); Assert.That(imported.Num, Is.Zero, "The number is given when the object is stored."); Assert.That(imported.IsEnterpriseConfiguration, Is.False); Assert.That(imported.EnterpriseConfigurationPluginId, Is.EqualTo(Guid.Empty)); Assert.That(imported.Name, Is.EqualTo(original.Name)); }); } [Test] public void EveryTypeOfConfigurationObjectCanBeImported() => Assert.That(IMPORTABLE_TYPES, Is.EquivalentTo(PluginConfigurationObjectDescriptors.ALL.Select(descriptor => descriptor.Type)), "A new type of configuration object needs a decision on how an import makes it local, and its export here."); [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)] [TestCase(PluginConfigurationObjectType.PROFILE)] [TestCase(PluginConfigurationObjectType.DATA_SOURCE)] public void AnImportKeepsEveryValueButTheIdentity(PluginConfigurationObjectType type) { var (original, snippet) = Export(type); var imported = Import(snippet, type).ConfigurationObject; Assert.That(WithIdentityOf(imported, original), Is.EqualTo(original)); } [Test] public void AChatTemplateKeepsItsPromptsConversationAndPreselections() { var (original, snippet) = Export(PluginConfigurationObjectType.CHAT_TEMPLATE); var template = (ChatTemplate)original; var imported = (ChatTemplate)Import(snippet, PluginConfigurationObjectType.CHAT_TEMPLATE).ConfigurationObject; Assert.Multiple(() => { Assert.That(imported.SystemPrompt, Is.EqualTo(template.SystemPrompt)); Assert.That(imported.PredefinedUserPrompt, Is.EqualTo(template.PredefinedUserPrompt)); Assert.That(imported.AllowProfileUsage, Is.EqualTo(template.AllowProfileUsage)); Assert.That(imported.ExampleConversation.Select(Message), Is.EqualTo(template.ExampleConversation.Select(Message))); Assert.That(imported.ToolIds, Is.EquivalentTo(template.ToolIds!)); Assert.That(imported.DataSourceOptions!.AutomaticDataSourceSelection, Is.EqualTo(template.DataSourceOptions!.AutomaticDataSourceSelection)); Assert.That(imported.DataSourceOptions.AutomaticValidation, Is.EqualTo(template.DataSourceOptions.AutomaticValidation)); Assert.That(imported.DataSourceOptions.PreselectedDataSourceIds, Is.EqualTo(template.DataSourceOptions.PreselectedDataSourceIds)); }); } [Test] public void APolicyKeepsItsRulesButNotItsProtection() { var (original, snippet) = Export(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY); var policy = (DataDocumentAnalysisPolicy)original; var imported = (DataDocumentAnalysisPolicy)Import(snippet, PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY).ConfigurationObject; Assert.Multiple(() => { Assert.That(imported.IsProtected, Is.False, "A configuration plugin protects its policies; the user's own policy is the user's to edit."); Assert.That(imported.PolicyDescription, Is.EqualTo(policy.PolicyDescription)); Assert.That(imported.AnalysisRules, Is.EqualTo(policy.AnalysisRules)); Assert.That(imported.OutputRules, Is.EqualTo(policy.OutputRules)); Assert.That(imported.MinimumProviderConfidence, Is.EqualTo(policy.MinimumProviderConfidence)); Assert.That(imported.AllowedToolIds, Is.EquivalentTo(policy.AllowedToolIds)); Assert.That(imported.PreselectedProvider, Is.EqualTo(policy.PreselectedProvider)); Assert.That(imported.PreselectedProfile, Is.EqualTo(policy.PreselectedProfile)); Assert.That(imported.HidePolicyDefinition, Is.EqualTo(policy.HidePolicyDefinition)); }); } [Test] public void ASnippetWhichIsNoValidConfigurationIsRefused() { const string PROFILE_WITHOUT_ACTIONS = """CONFIG["PROFILES"][#CONFIG["PROFILES"]+1] = { ["Id"] = "6b8e1a2c-3d4f-4a5b-9c6d-7e8f9a0b1c2d", ["Name"] = "Writer", ["NeedToKnow"] = "I write." }"""; Assert.Multiple(() => { Assert.That(ConfigurationSnippetImport.TryImport(PROFILE_WITHOUT_ACTIONS, PluginConfigurationObjectType.PROFILE, out var result, out var issue), Is.False); Assert.That(result, Is.Null); Assert.That(issue, Is.Not.Empty); }); } [Test] public void ASnippetWhichCannotBeReadIsRefused() { Assert.Multiple(() => { Assert.That(ConfigurationSnippetImport.TryImport("CONFIG[\"PROFILES\"] = os.getenv(\"HOME\")", PluginConfigurationObjectType.PROFILE, out _, out var issue), Is.False); Assert.That(issue, Is.Not.Empty); }); } [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)] public void TheAPIKeyOfAProviderIsDecrypted(PluginConfigurationObjectType type) { var result = Import(ExportProvider(type, Encrypt(API_KEY)).Snippet, type); Assert.Multiple(() => { Assert.That(result.Secret, Is.EqualTo(API_KEY)); Assert.That(result.Hints, Is.Empty); }); } [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)] public void AnAPIKeyOfAnotherOrganizationLeavesAHint(PluginConfigurationObjectType type) { var otherOrganization = new EnterpriseEncryption(NullLogger.Instance, EnterpriseEncryption.GenerateSecret()); Assert.That(otherOrganization.TryEncrypt(API_KEY, out var foreignApiKey), Is.True); AssertAPIKeyLeftOut(Import(ExportProvider(type, foreignApiKey).Snippet, type)); } [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)] public void AnAPIKeyWithoutAnEncryptionSecretLeavesAHint(PluginConfigurationObjectType type) { var snippet = ExportProvider(type, Encrypt(API_KEY)).Snippet; RemoveEncryptionSecret(); AssertAPIKeyLeftOut(Import(snippet, type)); } [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)] public void AnUnencryptedAPIKeyIsLeftOut(PluginConfigurationObjectType type) => AssertAPIKeyLeftOut(Import(ExportProvider(type, API_KEY).Snippet, type)); [Test] public void TheTokenOfADataSourceIsDecrypted() => Assert.That(Import(ExportDataSource(AuthMethod.TOKEN).Snippet, PluginConfigurationObjectType.DATA_SOURCE).Secret, Is.EqualTo(TOKEN)); [Test] public void ADataSourceWithATokenNeedsTheEncryptionSecret() { var snippet = ExportDataSource(AuthMethod.TOKEN).Snippet; RemoveEncryptionSecret(); AssertRefused(snippet, PluginConfigurationObjectType.DATA_SOURCE); } [Test] public void ADataSourceWithATokenOfAnotherOrganizationIsRefused() { var snippet = ExportDataSource(AuthMethod.TOKEN).Snippet; PluginFactory.InitializeEnterpriseEncryption(EnterpriseEncryption.GenerateSecret()); AssertRefused(snippet, PluginConfigurationObjectType.DATA_SOURCE); } [Test] public void ADataSourceWithoutSignInNeedsNoEncryptionSecret() { var snippet = ExportDataSource(AuthMethod.NONE).Snippet; RemoveEncryptionSecret(); Assert.That(Import(snippet, PluginConfigurationObjectType.DATA_SOURCE).Secret, Is.Empty); } [Test] public void AKerberosDataSourceIsRefused() => AssertRefused(ExportDataSource(AuthMethod.KERBEROS).Snippet, PluginConfigurationObjectType.DATA_SOURCE); [TestCase("jane.doe", "jane.doe")] [TestCase("", "")] public void ASharedUsernameIsKeptButNotItsPlaceholder(string username, string expectedUsername) { var (_, snippet) = ExportDataSource(AuthMethod.USERNAME_PASSWORD, username, DataSourceERIUsernamePasswordMode.SHARED_USERNAME_AND_PASSWORD); var result = Import(snippet, PluginConfigurationObjectType.DATA_SOURCE); var dataSource = (DataSourceERI_V1)result.ConfigurationObject; Assert.Multiple(() => { Assert.That(dataSource.Username, Is.EqualTo(expectedUsername)); Assert.That(dataSource.UsernamePasswordMode, Is.EqualTo(DataSourceERIUsernamePasswordMode.USER_MANAGED), "Only an organization manages the sign-in for its users."); Assert.That(result.Secret, Is.EqualTo(TOKEN)); }); } [Test] public void AttachmentsWhichAreNotOnThisComputerAreToBeRelinked() { var existing = this.CreateFile("report.pdf"); var missing = Path.Combine(this.temporaryDirectory, "missing.pdf"); const string RELATIVE = "attachments/notes.txt"; const string WINDOWS_SHARE = @"\\fileserver\share\minutes.docx"; const string UNIX_SHARE = "//fileserver/share/minutes.docx"; var result = Import(ExportChatTemplate([existing, missing, RELATIVE, WINDOWS_SHARE, UNIX_SHARE]), PluginConfigurationObjectType.CHAT_TEMPLATE); var template = (ChatTemplate)result.ConfigurationObject; Assert.Multiple(() => { Assert.That(template.FileAttachments.Select(attachment => attachment.FilePath), Is.EqualTo(new[] { existing })); Assert.That(result.AttachmentsToRelink, Is.EqualTo(new[] { missing, RELATIVE, WINDOWS_SHARE, UNIX_SHARE })); }); } [Test] public void AttachmentsOnThisComputerAreNamedInAHint() { var existing = this.CreateFile("id_rsa"); var result = Import(ExportChatTemplate([existing]), PluginConfigurationObjectType.CHAT_TEMPLATE); Assert.That(result.Hints, Has.Exactly(1).Contains(existing), "A snippet can name any file of this computer, so the user has to see which ones every chat would carry."); } [Test] public void AChatTemplateWithoutAttachmentsLeavesNoHint() { var result = Import(ExportChatTemplate([]), PluginConfigurationObjectType.CHAT_TEMPLATE); Assert.Multiple(() => { Assert.That(result.Hints, Is.Empty); Assert.That(result.AttachmentsToRelink, Is.Empty); }); } [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] public void ATokenizerOnThisComputerIsKept(PluginConfigurationObjectType type) { // The export keeps only the file name of the tokenizer and names the folder of the provider's copy itself: var (original, snippet) = ExportProvider(type, null, Path.Combine(this.temporaryDirectory, "tokenizer.json")); var copy = this.CreateTokenizerCopy(type, original.Id); var result = Import(snippet, type); Assert.Multiple(() => { Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.EqualTo(copy), "Imported on the computer it came from, the export leads back to the copy of the provider."); Assert.That(result.Hints, Is.Empty); }); } [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] public void ATokenizerWhichIsNotOnThisComputerIsLeftOut(PluginConfigurationObjectType type) { var result = Import(ExportProvider(type, null, Path.Combine(this.temporaryDirectory, "missing-tokenizer.json")).Snippet, type); Assert.Multiple(() => { Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.Empty, "Saving the provider would fail on a tokenizer which cannot be copied."); Assert.That(result.Hints, Has.Exactly(1).Contains("missing-tokenizer.json")); }); } [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] public void TheAbsoluteTokenizerPathOfAnOlderExportIsKept(PluginConfigurationObjectType type) { var tokenizer = this.CreateFile("tokenizer.json"); var result = Import(ExportProviderWithTokenizerPath(type, tokenizer), type); Assert.Multiple(() => { Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.EqualTo(tokenizer), "Versions before v26.10.1 exported the absolute path of the tokenizer."); Assert.That(result.Hints, Is.Empty); }); } [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] public void AnExportCannotNameAnotherFileOfTheDataDirectoryAsItsTokenizer(PluginConfigurationObjectType type) { Directory.CreateDirectory(this.dataDirectory); File.WriteAllText(Path.Combine(this.dataDirectory, "settings.json"), "{}"); var result = Import(ExportProviderWithTokenizerPath(type, "tokenizers/../settings.json"), type); Assert.Multiple(() => { Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.Empty, "Only the folder of the tokenizers holds what an export may name."); Assert.That(result.Hints, Has.Count.EqualTo(1)); }); } [Test] public async Task MissingReferencesOfAPolicyAreNamed() { var missingProfileId = Guid.NewGuid().ToString(); var snippet = ExportPolicy(preselectedProvider: "missing-provider", preselectedProfile: missingProfileId, allowedToolIds: [TOOL_ID, "missing_tool"]); var result = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(snippet, PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY), this.SettingsManager, this.CreateRegistry(new TestTool(Definition()))); Assert.Multiple(() => { Assert.That(result.Hints, Has.Count.EqualTo(3), "One sentence for each kind of reference."); Assert.That(result.Hints, Has.Exactly(1).Contains("missing-provider")); Assert.That(result.Hints, Has.Exactly(1).Contains(missingProfileId)); Assert.That(result.Hints, Has.Exactly(1).Contains("missing_tool")); Assert.That(result.Hints, Has.None.Contains(TOOL_ID), "The tool exists."); }); } [Test] public async Task MissingReferencesOfAChatTemplateAreNamed() { var missingDataSourceId = Guid.NewGuid().ToString(); var snippet = ExportChatTemplate([], toolIds: [TOOL_ID, "missing_tool"], dataSourceIds: [missingDataSourceId]); var result = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(snippet, PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, this.CreateRegistry(new TestTool(Definition()))); Assert.Multiple(() => { Assert.That(result.Hints, Has.Count.EqualTo(2), "One sentence for each kind of reference."); Assert.That(result.Hints, Has.Exactly(1).Contains(missingDataSourceId)); Assert.That(result.Hints, Has.Exactly(1).Contains("missing_tool")); }); } [Test] public async Task ReferencesWhichExistLeaveNoHint() { var provider = ExportProvider(PluginConfigurationObjectType.LLM_PROVIDER, null).Original; var profile = Export(PluginConfigurationObjectType.PROFILE).Original; var dataSource = ExportDataSource(AuthMethod.NONE).Original; this.SettingsManager.ConfigurationData.Providers.Add((AIStudio.Settings.Provider)provider); this.SettingsManager.ConfigurationData.Profiles.Add((Profile)profile); this.SettingsManager.ConfigurationData.DataSources.Add((DataSourceERI_V1)dataSource); var registry = this.CreateRegistry(new TestTool(Definition())); var policy = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(ExportPolicy(provider.Id, profile.Id, [TOOL_ID]), PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY), this.SettingsManager, registry); var template = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(ExportChatTemplate([], [TOOL_ID], [dataSource.Id]), PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, registry); Assert.Multiple(() => { Assert.That(policy.Hints, Is.Empty); Assert.That(template.Hints, Is.Empty); }); } [Test] public async Task AToolOfACollectionIsSelectedAsItsCollection() { const string COLLECTION_ID = "test_collection"; var registry = this.CreateRegistry([new TestCollection(COLLECTION_ID, ConfidenceLevel.NONE, TOOL_ID)], new TestTool(Definition())); var policy = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(ExportPolicy(allowedToolIds: [TOOL_ID]), PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY), this.SettingsManager, registry); var template = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(ExportChatTemplate([], [TOOL_ID]), PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, registry); Assert.Multiple(() => { Assert.That(((DataDocumentAnalysisPolicy)policy.ConfigurationObject).AllowedToolIds, Is.EquivalentTo(new[] { COLLECTION_ID })); Assert.That(((ChatTemplate)template.ConfigurationObject).ToolIds, Is.EquivalentTo(new[] { COLLECTION_ID })); Assert.That(policy.Hints, Is.Empty); Assert.That(template.Hints, Is.Empty); }); } [Test] public async Task AChatTemplateWhichSaysNothingAboutToolsKeepsSayingNothing() { var result = await ConfigurationSnippetImport.ResolveReferencesAsync(Import(ExportChatTemplate([], toolIds: null), PluginConfigurationObjectType.CHAT_TEMPLATE), this.SettingsManager, this.CreateRegistry(new TestTool(Definition()))); Assert.That(((ChatTemplate)result.ConfigurationObject).ToolIds, Is.Null, "No tools at all would be a statement the template never made."); } private static ConfigurationSnippetImportResult Import(string snippet, PluginConfigurationObjectType type) { Assert.That(ConfigurationSnippetImport.TryImport(snippet, type, out var result, out var issue), Is.True, issue); return result!; } private static void AssertRefused(string snippet, PluginConfigurationObjectType type) { Assert.Multiple(() => { Assert.That(ConfigurationSnippetImport.TryImport(snippet, type, out var result, out var issue), Is.False); Assert.That(result, Is.Null); Assert.That(issue, Is.Not.Empty); }); } private static void AssertAPIKeyLeftOut(ConfigurationSnippetImportResult result) { Assert.Multiple(() => { Assert.That(result.Secret, Is.Empty); Assert.That(result.Hints, Has.Count.EqualTo(1), "The provider is imported, but the user has to enter a key."); }); } /// /// Leaves this computer without the encryption secret of an organization, as most computers are. /// private static void RemoveEncryptionSecret() => PluginFactory.InitializeEnterpriseEncryption(encryptionSecret: null); private static string Encrypt(string secret) { Assert.That(PluginFactory.EnterpriseEncryption!.TryEncrypt(secret, out var encryptedSecret), Is.True); return encryptedSecret; } private static string ExportedId(string snippet, PluginConfigurationObjectType type) { Assert.That(ConfigurationSnippetParser.TryParse(snippet, type, out var table, out var issue), Is.True, issue); return table["Id"].Read(); } private static object WithIdentityOf(IConfigurationObject imported, IConfigurationObject original) => imported switch { ConfigurationBaseObject configObject => configObject with { Id = original.Id, Num = original.Num }, DataSourceERI_V1 dataSource => dataSource with { Id = original.Id, Num = original.Num }, _ => throw new ArgumentOutOfRangeException(nameof(imported), imported, "This test cannot compare this type."), }; private static string TokenizerPathOf(IConfigurationObject configObject) => configObject switch { AIStudio.Settings.Provider provider => provider.TokenizerPath, EmbeddingProvider provider => provider.TokenizerPath, _ => throw new ArgumentOutOfRangeException(nameof(configObject), configObject, "This type has no tokenizer."), }; private static (ChatRole Role, string Text) Message(ContentBlock block) => (block.Role, (block.Content as ContentText)?.Text ?? string.Empty); private string CreateFile(string fileName) { var path = Path.Combine(this.temporaryDirectory, fileName); File.WriteAllText(path, "{}"); return path; } /// /// Creates the tokenizer copy of a provider where the runtime stores it, below the data directory. /// private string CreateTokenizerCopy(PluginConfigurationObjectType type, string providerId) { var modelId = type is PluginConfigurationObjectType.LLM_PROVIDER ? TokenizerModelId.ForProviderId(providerId) : TokenizerModelId.ForEmbeddingProviderId(providerId); var path = Path.Combine(this.dataDirectory, "tokenizers", modelId, "tokenizer.json"); Directory.CreateDirectory(Path.GetDirectoryName(path)!); File.WriteAllText(path, "{}"); return path; } /// /// What an export of a provider carries when its tokenizer path was written otherwise, e.g., by /// an older version or by hand. /// private static string ExportProviderWithTokenizerPath(PluginConfigurationObjectType type, string tokenizerPath) { const string EMPTY_TOKENIZER_PATH = """["TokenizerPath"] = "","""; var snippet = ExportProvider(type, null).Snippet; Assert.That(snippet, Does.Contain(EMPTY_TOKENIZER_PATH)); return snippet.Replace(EMPTY_TOKENIZER_PATH, $"""["TokenizerPath"] = "{LuaTools.EscapeLuaString(tokenizerPath)}","""); } /// /// What the app exports for one item of each type, along with the item. /// /// /// The items are made here rather than in a test case source: NUnit reads such a source before /// TestHost runs, and the provider types need its logger factory. Texts span lines with line /// feeds only, as Lua turns every other line break of a long string into one. /// private static (IConfigurationObject Original, string Snippet) Export(PluginConfigurationObjectType type) { switch (type) { case PluginConfigurationObjectType.LLM_PROVIDER or PluginConfigurationObjectType.EMBEDDING_PROVIDER or PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER: return ExportProvider(type, null); case PluginConfigurationObjectType.PROFILE: var profile = new Profile(1, Guid.NewGuid().ToString(), "Technical writer", "I write manuals.\nMy readers are engineers.", "Keep the answers short."); return (profile, profile.ExportAsConfigurationSection()); case PluginConfigurationObjectType.CHAT_TEMPLATE: var template = ChatTemplateWith([], ["web_search"], [Guid.NewGuid().ToString()]); Assert.That(template.TryExportAsConfigurationSection(out var luaCode, out var issue), Is.True, issue); return (template, luaCode); case PluginConfigurationObjectType.DATA_SOURCE: return ExportDataSource(AuthMethod.TOKEN); case PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY: var policy = PolicyWith(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), ["web_search", "read_web_page"]); return (policy, policy.ExportAsConfigurationSection()); default: throw new ArgumentOutOfRangeException(nameof(type), type, "This test knows no export for this type."); } } private static (IConfigurationObject Original, string Snippet) ExportProvider(PluginConfigurationObjectType type, string? encryptedApiKey, string tokenizerPath = "") { switch (type) { case PluginConfigurationObjectType.LLM_PROVIDER: var provider = new AIStudio.Settings.Provider( 1, Guid.NewGuid().ToString(), "Ollama on the workstation", LLMProviders.SELF_HOSTED, new Model("llama-3.3-70b", "Llama 3.3"), IsSelfHosted: true, Hostname: "http://localhost:11434", Host: Host.OLLAMA, AdditionalJsonApiParameters: "\"temperature\": 0.5", TokenizerPath: tokenizerPath, CapabilityOverrides: new ProviderCapabilityOverrides { FunctionCalling = true, ContextWindowTokens = 32_768 }); return (provider, provider.ExportAsConfigurationSection(encryptedApiKey)); case PluginConfigurationObjectType.EMBEDDING_PROVIDER: var embeddingProvider = new EmbeddingProvider( 1, Guid.NewGuid().ToString(), "Embeddings on the workstation", LLMProviders.SELF_HOSTED, new Model("nomic-embed-text", "Nomic"), IsSelfHosted: true, Hostname: "http://localhost:11434", Host: Host.OLLAMA, TokenizerPath: tokenizerPath, EmbeddingBatchSize: 16, TokenLimit: 512); return (embeddingProvider, embeddingProvider.ExportAsConfigurationSection(encryptedApiKey)); case PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER: var transcriptionProvider = new TranscriptionProvider( 1, Guid.NewGuid().ToString(), "Whisper on the workstation", LLMProviders.SELF_HOSTED, new Model("whisper-large-v3", "Whisper"), IsSelfHosted: true, Hostname: "http://localhost:8080", Host: Host.WHISPER_CPP); return (transcriptionProvider, transcriptionProvider.ExportAsConfigurationSection(encryptedApiKey)); default: throw new ArgumentOutOfRangeException(nameof(type), type, "This is no type of provider."); } } private static (IConfigurationObject Original, string Snippet) ExportDataSource(AuthMethod authMethod, string username = "", DataSourceERIUsernamePasswordMode usernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED) { var dataSource = new DataSourceERI_V1 { Num = 1, Id = Guid.NewGuid().ToString(), Name = "Contracts", Type = DataSourceType.ERI_V1, Hostname = "https://eri.example.org", Port = 8443, AuthMethod = authMethod, Username = username, SecurityPolicy = DataSourceSecurity.SELF_HOSTED, SelectedRetrievalId = "semantic-search", MaxMatches = 25, }; var encryptedSecret = authMethod is AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD ? Encrypt(TOKEN) : null; return (dataSource, dataSource.ExportAsConfigurationSection(encryptedSecret, usernamePasswordMode)); } private static string ExportChatTemplate(IReadOnlyList attachmentPaths, IReadOnlyList? toolIds = null, IReadOnlyList? dataSourceIds = null) { var template = ChatTemplateWith(attachmentPaths, toolIds, dataSourceIds); Assert.That(template.TryExportAsConfigurationSection(out var luaCode, out var issue), Is.True, issue); return luaCode; } private static ChatTemplate ChatTemplateWith(IReadOnlyList attachmentPaths, IReadOnlyList? toolIds, IReadOnlyList? dataSourceIds) => new() { Num = 1, Id = Guid.NewGuid().ToString(), Name = "Contract review", SystemPrompt = "You review contracts.\nYou quote every clause you mention.", PredefinedUserPrompt = "Please review: ", AllowProfileUsage = true, FileAttachments = attachmentPaths.Select(path => new FileAttachment(FileAttachmentType.DOCUMENT, Path.GetFileName(path), path, 0)).ToList(), ExampleConversation = [ new ContentBlock { ContentType = ContentType.TEXT, Role = ChatRole.USER, Content = new ContentText { Text = "Is clause 4 fair?" } }, new ContentBlock { ContentType = ContentType.TEXT, Role = ChatRole.AI, Content = new ContentText { Text = "Clause 4 says:\n> The tenant pays." } }, ], ToolIds = toolIds?.ToHashSet(StringComparer.Ordinal), DataSourceOptions = dataSourceIds is null ? null : new() { DisableDataSources = false, AutomaticValidation = true, PreselectedDataSourceIds = [..dataSourceIds] }, }; private static string ExportPolicy(string preselectedProvider = "", string preselectedProfile = "", IReadOnlyList? allowedToolIds = null) => PolicyWith(preselectedProvider, preselectedProfile, allowedToolIds ?? []).ExportAsConfigurationSection(); private static DataDocumentAnalysisPolicy PolicyWith(string preselectedProvider, string preselectedProfile, IReadOnlyList allowedToolIds) => new() { Num = 1, Id = Guid.NewGuid().ToString(), PolicyName = "Contract check", PolicyDescription = "Checks contracts for the clauses our legal department cares about.", AnalysisRules = "Find every clause on liability.\nFind every clause on termination.", OutputRules = "- Answer in a table.\n- Quote the clause.", MinimumProviderConfidence = ConfidenceLevel.MEDIUM, AllowedToolIds = allowedToolIds.ToHashSet(StringComparer.Ordinal), PreselectedProvider = preselectedProvider, PreselectedProfile = preselectedProfile, HidePolicyDefinition = true, IsProtected = true, }; }