From f4780939fcc6aa9fc7bb197dd62ab82ff147b904 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sat, 15 Feb 2025 15:41:12 +0100 Subject: [PATCH] Integrated data sources into any chat (#282) --- README.md | 2 +- app/MindWork AI Studio/Agents/AgentBase.cs | 7 +- .../Agents/AgentTextContentCleaner.cs | 3 +- .../Assistants/AssistantBase.razor | 2 +- .../Assistants/AssistantBase.razor.cs | 7 +- app/MindWork AI Studio/Chat/ChatThread.cs | 6 + .../Chat/ContentBlockComponent.razor.cs | 3 +- app/MindWork AI Studio/Chat/ContentImage.cs | 3 +- app/MindWork AI Studio/Chat/ContentText.cs | 41 +- app/MindWork AI Studio/Chat/IContent.cs | 3 +- .../Components/ChatComponent.razor | 7 +- .../Components/ChatComponent.razor.cs | 144 ++++- .../Components/ConfidenceInfo.razor | 4 +- .../Components/ConfidenceInfo.razor.cs | 2 +- .../Components/DataSourceSelection.razor | 93 ++++ .../Components/DataSourceSelection.razor.cs | 260 +++++++++ .../Components/DataSourceSelectionMode.cs | 23 + ...denceInfoMode.cs => PopoverTriggerMode.cs} | 2 +- .../Components/ProfileSelection.razor.cs | 5 +- .../Components/SelectDirectory.razor.cs | 1 + .../Components/SelectFile.razor.cs | 1 + .../Components/Settings/SettingsPanelBase.cs | 1 + .../Settings/SettingsPanelChat.razor | 7 + .../Components/TextInfoLine.razor.cs | 1 + .../Components/TextInfoLines.razor | 1 + .../Components/TextInfoLines.razor.cs | 13 + .../Dialogs/DataSourceERI-V1InfoDialog.razor | 5 +- .../DataSourceERI-V1InfoDialog.razor.cs | 1 + .../Dialogs/DataSourceERI_V1Dialog.razor | 8 + .../Dialogs/DataSourceERI_V1Dialog.razor.cs | 28 + .../DataSourceLocalDirectoryDialog.razor | 9 + .../DataSourceLocalDirectoryDialog.razor.cs | 3 + .../DataSourceLocalDirectoryInfoDialog.razor | 4 + .../Dialogs/DataSourceLocalFileDialog.razor | 8 + .../DataSourceLocalFileDialog.razor.cs | 3 + .../DataSourceLocalFileInfoDialog.razor | 3 + .../Dialogs/EmbeddingProviderDialog.razor.cs | 1 + .../Dialogs/ProviderDialog.razor.cs | 2 +- .../Layout/MainLayout.razor.cs | 1 - app/MindWork AI Studio/Pages/Writer.razor.cs | 6 +- app/MindWork AI Studio/Program.cs | 17 + .../Provider/BaseProvider.cs | 3 +- .../Settings/ConfigurationSelectData.cs | 6 + .../Settings/DataModel/DataChat.cs | 10 + .../Settings/DataModel/DataSourceERI_V1.cs | 3 + .../DataModel/DataSourceLocalDirectory.cs | 3 + .../Settings/DataModel/DataSourceLocalFile.cs | 3 + .../Settings/DataModel/DataSourceOptions.cs | 65 +++ .../Settings/DataModel/DataSourceSecurity.cs | 19 + .../DataModel/DataSourceSecurityExtensions.cs | 32 ++ ...nsions.cs => PreviewFeaturesExtensions.cs} | 2 +- .../DataModel/SendToChatDataSourceBehavior.cs | 7 + .../SendToChatDataSourceBehaviorExtensions.cs | 12 + .../Settings/IDataSource.cs | 5 + .../Tools/AllowedSelectedDataSources.cs | 13 + .../Tools/ERIClient/ERIClientV1.cs | 522 +++++++++++------- .../Tools/ERIClient/IERIClient.cs | 1 + .../Tools/Services/DataSourceService.cs | 209 +++++++ .../{ => Services}/RustService.APIKeys.cs | 2 +- .../Tools/{ => Services}/RustService.App.cs | 2 +- .../{ => Services}/RustService.Clipboard.cs | 2 +- .../{ => Services}/RustService.FileSystem.cs | 2 +- .../{ => Services}/RustService.Secrets.cs | 2 +- .../{ => Services}/RustService.Updates.cs | 2 +- .../Tools/{ => Services}/RustService.cs | 2 +- .../Tools/Services/UpdateService.cs | 35 +- app/MindWork AI Studio/Tools/TextColor.cs | 11 + .../Tools/TextColorExtensions.cs | 18 + .../Tools/Validation/DataSourceValidation.cs | 18 + .../wwwroot/changelog/v0.9.29.md | 4 + 70 files changed, 1506 insertions(+), 250 deletions(-) create mode 100644 app/MindWork AI Studio/Components/DataSourceSelection.razor create mode 100644 app/MindWork AI Studio/Components/DataSourceSelection.razor.cs create mode 100644 app/MindWork AI Studio/Components/DataSourceSelectionMode.cs rename app/MindWork AI Studio/Components/{ConfidenceInfoMode.cs => PopoverTriggerMode.cs} (63%) create mode 100644 app/MindWork AI Studio/Settings/DataModel/DataSourceOptions.cs create mode 100644 app/MindWork AI Studio/Settings/DataModel/DataSourceSecurity.cs create mode 100644 app/MindWork AI Studio/Settings/DataModel/DataSourceSecurityExtensions.cs rename app/MindWork AI Studio/Settings/DataModel/{PreviewFeatureExtensions.cs => PreviewFeaturesExtensions.cs} (93%) create mode 100644 app/MindWork AI Studio/Settings/DataModel/SendToChatDataSourceBehavior.cs create mode 100644 app/MindWork AI Studio/Settings/DataModel/SendToChatDataSourceBehaviorExtensions.cs create mode 100644 app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs create mode 100644 app/MindWork AI Studio/Tools/Services/DataSourceService.cs rename app/MindWork AI Studio/Tools/{ => Services}/RustService.APIKeys.cs (99%) rename app/MindWork AI Studio/Tools/{ => Services}/RustService.App.cs (99%) rename app/MindWork AI Studio/Tools/{ => Services}/RustService.Clipboard.cs (98%) rename app/MindWork AI Studio/Tools/{ => Services}/RustService.FileSystem.cs (97%) rename app/MindWork AI Studio/Tools/{ => Services}/RustService.Secrets.cs (99%) rename app/MindWork AI Studio/Tools/{ => Services}/RustService.Updates.cs (97%) rename app/MindWork AI Studio/Tools/{ => Services}/RustService.cs (98%) create mode 100644 app/MindWork AI Studio/Tools/TextColor.cs create mode 100644 app/MindWork AI Studio/Tools/TextColorExtensions.cs diff --git a/README.md b/README.md index ebd2b5e..06469a0 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Things we are currently working on: - [ ] App: Implement the continuous process of vectorizing data - [x] ~~App: Define a common retrieval context interface for the integration of RAG processes in chats (PR [#281](https://github.com/MindWorkAI/AI-Studio/pull/281))~~ - [ ] App: Define a common augmentation interface for the integration of RAG processes in chats - - [ ] App: Integrate data sources in chats + - [x] ~~App: Integrate data sources in chats (PR [#282](https://github.com/MindWorkAI/AI-Studio/pull/282))~~ - Since September 2024: Experiments have been started on how we can work on long texts with AI Studio. Let's say you want to write a fantasy novel or create a complex project proposal and use LLM for support. The initial experiments were promising, but not yet satisfactory. We are testing further approaches until a satisfactory solution is found. The current state of our experiment is available as an experimental preview feature through your app configuration. Related PR: ~~[#167](https://github.com/MindWorkAI/AI-Studio/pull/167), [#226](https://github.com/MindWorkAI/AI-Studio/pull/226)~~. diff --git a/app/MindWork AI Studio/Agents/AgentBase.cs b/app/MindWork AI Studio/Agents/AgentBase.cs index 639fa99..35c4f39 100644 --- a/app/MindWork AI Studio/Agents/AgentBase.cs +++ b/app/MindWork AI Studio/Agents/AgentBase.cs @@ -1,13 +1,16 @@ using AIStudio.Chat; using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.Services; // ReSharper disable MemberCanBePrivate.Global namespace AIStudio.Agents; -public abstract class AgentBase(ILogger logger, SettingsManager settingsManager, ThreadSafeRandom rng) : IAgent +public abstract class AgentBase(ILogger logger, SettingsManager settingsManager, DataSourceService dataSourceService, ThreadSafeRandom rng) : IAgent { + protected DataSourceService DataSourceService { get; init; } = dataSourceService; + protected SettingsManager SettingsManager { get; init; } = settingsManager; protected ThreadSafeRandom RNG { get; init; } = rng; @@ -107,6 +110,6 @@ public abstract class AgentBase(ILogger logger, SettingsManager setti // Use the selected provider to get the AI response. // By awaiting this line, we wait for the entire // content to be streamed. - await aiText.CreateFromProviderAsync(providerSettings.CreateProvider(this.Logger), this.SettingsManager, providerSettings.Model, this.lastUserPrompt, thread); + await aiText.CreateFromProviderAsync(providerSettings.CreateProvider(this.Logger), this.SettingsManager, this.DataSourceService, providerSettings.Model, this.lastUserPrompt, thread); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Agents/AgentTextContentCleaner.cs b/app/MindWork AI Studio/Agents/AgentTextContentCleaner.cs index e8d19aa..d8e8381 100644 --- a/app/MindWork AI Studio/Agents/AgentTextContentCleaner.cs +++ b/app/MindWork AI Studio/Agents/AgentTextContentCleaner.cs @@ -1,9 +1,10 @@ using AIStudio.Chat; using AIStudio.Settings; +using AIStudio.Tools.Services; namespace AIStudio.Agents; -public sealed class AgentTextContentCleaner(ILogger logger, SettingsManager settingsManager, ThreadSafeRandom rng) : AgentBase(logger, settingsManager, rng) +public sealed class AgentTextContentCleaner(ILogger logger, SettingsManager settingsManager, DataSourceService dataSourceService, ThreadSafeRandom rng) : AgentBase(logger, settingsManager, dataSourceService, rng) { private static readonly ContentBlock EMPTY_BLOCK = new() { diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor b/app/MindWork AI Studio/Assistants/AssistantBase.razor index 2390a23..90a86b0 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor @@ -121,7 +121,7 @@ @if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence) { - + } @if (this.AllowProfiles && this.ShowProfileSelection) diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index 63867ff..e7dcba2 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -1,12 +1,12 @@ using AIStudio.Chat; using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; using MudBlazor.Utilities; -using RustService = AIStudio.Tools.RustService; using Timer = System.Timers.Timer; namespace AIStudio.Assistants; @@ -28,6 +28,9 @@ public abstract partial class AssistantBase : ComponentBase, IMessageBusReceiver [Inject] protected RustService RustService { get; init; } = null!; + [Inject] + protected DataSourceService DataSourceService { get; init; } = null!; + [Inject] protected NavigationManager NavigationManager { get; init; } = null!; @@ -290,7 +293,7 @@ public abstract partial class AssistantBase : ComponentBase, IMessageBusReceiver // Use the selected provider to get the AI response. // By awaiting this line, we wait for the entire // content to be streamed. - await aiText.CreateFromProviderAsync(this.providerSettings.CreateProvider(this.Logger), this.SettingsManager, this.providerSettings.Model, this.lastUserPrompt, this.chatThread); + await aiText.CreateFromProviderAsync(this.providerSettings.CreateProvider(this.Logger), this.SettingsManager, this.DataSourceService, this.providerSettings.Model, this.lastUserPrompt, this.chatThread); this.isProcessing = false; this.StateHasChanged(); diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 4f4f573..3e41161 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -1,4 +1,5 @@ using AIStudio.Settings; +using AIStudio.Settings.DataModel; namespace AIStudio.Chat; @@ -27,6 +28,11 @@ public sealed record ChatThread /// public string SelectedProfile { get; set; } = string.Empty; + /// + /// The data source options for this chat thread. + /// + public DataSourceOptions DataSourceOptions { get; set; } = new(); + /// /// The name of the chat thread. Usually generated by an AI model or manually edited by the user. /// diff --git a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs index 5797590..49e102a 100644 --- a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs +++ b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs @@ -1,9 +1,8 @@ using AIStudio.Settings; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; -using RustService = AIStudio.Tools.RustService; - namespace AIStudio.Chat; /// diff --git a/app/MindWork AI Studio/Chat/ContentImage.cs b/app/MindWork AI Studio/Chat/ContentImage.cs index 3a5fbd1..0d83145 100644 --- a/app/MindWork AI Studio/Chat/ContentImage.cs +++ b/app/MindWork AI Studio/Chat/ContentImage.cs @@ -2,6 +2,7 @@ using System.Text.Json.Serialization; using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.Services; namespace AIStudio.Chat; @@ -29,7 +30,7 @@ public sealed class ContentImage : IContent public Func StreamingEvent { get; set; } = () => Task.CompletedTask; /// - public Task CreateFromProviderAsync(IProvider provider, SettingsManager settings, Model chatModel, IContent? lastPrompt, ChatThread? chatChatThread, CancellationToken token = default) + public Task CreateFromProviderAsync(IProvider provider, SettingsManager settings, DataSourceService dataSourceService, Model chatModel, IContent? lastPrompt, ChatThread? chatChatThread, CancellationToken token = default) { throw new NotImplementedException(); } diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index 2061391..19928b4 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -2,6 +2,7 @@ using System.Text.Json.Serialization; using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.Services; namespace AIStudio.Chat; @@ -35,7 +36,7 @@ public sealed class ContentText : IContent public Func StreamingEvent { get; set; } = () => Task.CompletedTask; /// - public async Task CreateFromProviderAsync(IProvider provider, SettingsManager settings, Model chatModel, IContent? lastPrompt, ChatThread? chatThread, CancellationToken token = default) + public async Task CreateFromProviderAsync(IProvider provider, SettingsManager settings, DataSourceService dataSourceService, Model chatModel, IContent? lastPrompt, ChatThread? chatThread, CancellationToken token = default) { if(chatThread is null) return; @@ -43,15 +44,35 @@ public sealed class ContentText : IContent // // Check if the user wants to bind any data sources to the chat: // - - // - // Trigger the retrieval part of the (R)AG process: - // - - // - // Perform the augmentation of the R(A)G process: - // - + if (chatThread.DataSourceOptions.IsEnabled()) + { + // + // When the user wants to bind data sources to the chat, we + // have to check if the data sources are available for the + // selected provider. Also, we have to check if any ERI + // data sources changed its security requirements. + // + List preselectedDataSources = chatThread.DataSourceOptions.PreselectedDataSourceIds.Select(id => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == id)).Where(ds => ds is not null).ToList()!; + var dataSources = await dataSourceService.GetDataSources(provider, preselectedDataSources); + var selectedDataSources = dataSources.SelectedDataSources; + + // + // Should the AI select the data sources? + // + if (chatThread.DataSourceOptions.AutomaticDataSourceSelection) + { + // TODO: Start agent based on allowed data sources. + } + + // + // Trigger the retrieval part of the (R)AG process: + // + + // + // Perform the augmentation of the R(A)G process: + // + } + // Store the last time we got a response. We use this later // to determine whether we should notify the UI about the // new content or not. Depends on the energy saving mode diff --git a/app/MindWork AI Studio/Chat/IContent.cs b/app/MindWork AI Studio/Chat/IContent.cs index 987bada..86189a2 100644 --- a/app/MindWork AI Studio/Chat/IContent.cs +++ b/app/MindWork AI Studio/Chat/IContent.cs @@ -2,6 +2,7 @@ using System.Text.Json.Serialization; using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.Services; namespace AIStudio.Chat; @@ -42,5 +43,5 @@ public interface IContent /// /// Uses the provider to create the content. /// - public Task CreateFromProviderAsync(IProvider provider, SettingsManager settings, Model chatModel, IContent? lastPrompt, ChatThread? chatChatThread, CancellationToken token = default); + public Task CreateFromProviderAsync(IProvider provider, SettingsManager settings, DataSourceService dataSourceService, Model chatModel, IContent? lastPrompt, ChatThread? chatChatThread, CancellationToken token = default); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor b/app/MindWork AI Studio/Components/ChatComponent.razor index ffd5c8f..2ae6eb2 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor +++ b/app/MindWork AI Studio/Components/ChatComponent.razor @@ -97,7 +97,7 @@ @if (this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence) { - + } @if (this.isStreaming && this.cancellationTokenSource is not null) @@ -108,6 +108,11 @@ } + + @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager)) + { + + } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index 466d210..10b8bd7 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -3,6 +3,7 @@ using AIStudio.Dialogs; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Web; @@ -40,9 +41,14 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable [Inject] private IDialogService DialogService { get; init; } = null!; + [Inject] + private DataSourceService DataSourceService { get; init; } = null!; + private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top; private static readonly Dictionary USER_INPUT_ATTRIBUTES = new(); + private DataSourceSelection? dataSourceSelectionComponent; + private DataSourceOptions earlyDataSourceOptions = new(); private Profile currentProfile = Profile.NO_PROFILE; private bool hasUnsavedChanges; private bool mustScrollToBottomAfterRender; @@ -66,21 +72,40 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable protected override async Task OnInitializedAsync() { + // Apply the filters for the message bus: this.ApplyFilters([], [ Event.HAS_CHAT_UNSAVED_CHANGES, Event.RESET_CHAT_STATE, Event.CHAT_STREAMING_DONE, Event.WORKSPACE_LOADED_CHAT_CHANGED ]); // Configure the spellchecking for the user input: this.SettingsManager.InjectSpellchecking(USER_INPUT_ATTRIBUTES); - + + // Get the preselected profile: this.currentProfile = this.SettingsManager.GetPreselectedProfile(Tools.Components.CHAT); + + // + // Check for deferred messages of the kind 'SEND_TO_CHAT', + // aka the user sends an assistant result to the chat: + // var deferredContent = MessageBus.INSTANCE.CheckDeferredMessages(Event.SEND_TO_CHAT).FirstOrDefault(); if (deferredContent is not null) { + // + // Yes, the user sent an assistant result to the chat. + // + + // Use chat thread sent by the user: this.ChatThread = deferredContent; this.Logger.LogInformation($"The chat '{this.ChatThread.Name}' with {this.ChatThread.Blocks.Count} messages was deferred and will be rendered now."); await this.ChatThreadChanged.InvokeAsync(this.ChatThread); + // We know already that the chat thread is not null, + // but we have to check it again for the nullability + // for the compiler: if (this.ChatThread is not null) { + // + // Check if the chat thread has a name. If not, we + // generate the name now: + // if (string.IsNullOrWhiteSpace(this.ChatThread.Name)) { var firstUserBlock = this.ChatThread.Blocks.FirstOrDefault(x => x.Role == ChatRole.USER); @@ -94,12 +119,24 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable } } + // + // Check if the user wants to apply the standard chat data source options: + // + if (this.SettingsManager.ConfigurationData.Chat.SendToChatDataSourceBehavior is SendToChatDataSourceBehavior.APPLY_STANDARD_CHAT_DATA_SOURCE_OPTIONS) + this.ChatThread.DataSourceOptions = this.SettingsManager.ConfigurationData.Chat.PreselectedDataSourceOptions.CreateCopy(); + + // + // Check if the user wants to store the chat automatically: + // if (this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY) { this.autoSaveEnabled = true; this.mustStoreChat = true; - // Ensure the workspace exists: + // + // When a standard workspace is used, we have to ensure + // that the workspace is available: + // if(this.ChatThread.WorkspaceId == KnownWorkspaces.ERI_SERVER_WORKSPACE_ID) await WorkspaceBehaviour.EnsureERIServerWorkspace(); @@ -108,14 +145,38 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable } } } + else + { + // + // No, the user did not send an assistant result to the chat. + // + this.ApplyStandardDataSourceOptions(); + } + // + // Check if the user wants to show the latest message after loading: + // if (this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading) { + // + // We cannot scroll to the bottom right now because the + // chat component is not rendered yet. We have to wait for + // the rendering process to finish. Thus, we set a flag + // to scroll to the bottom after the rendering process.: + // this.mustScrollToBottomAfterRender = true; this.scrollRenderCountdown = 4; this.StateHasChanged(); } + // + // Check if another component deferred the loading of a chat. + // + // This is used, e.g., for the bias-of-the-day component: + // when the bias for this day was already produced, the bias + // component sends a message to the chat component to load + // the chat with the bias: + // var deferredLoading = MessageBus.INSTANCE.CheckDeferredMessages(Event.LOAD_CHAT).FirstOrDefault(); if (deferredLoading != default) { @@ -124,6 +185,11 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable this.Logger.LogInformation($"The loading of the chat '{this.loadChat.ChatId}' was deferred and will be loaded now."); } + // + // When for whatever reason we have a chat thread, we have to + // ensure that the corresponding workspace id is set and the + // workspace name is loaded: + // if (this.ChatThread is not null) { this.currentWorkspaceId = this.ChatThread.WorkspaceId; @@ -131,6 +197,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable this.WorkspaceName(this.currentWorkspaceName); } + // Select the correct provider: await this.SelectProviderWhenLoadingChat(); await base.OnInitializedAsync(); } @@ -204,6 +271,13 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable private string UserInputClass => this.SettingsManager.ConfigurationData.LLMProviders.ShowProviderConfidence ? "confidence-border" : string.Empty; + private void ApplyStandardDataSourceOptions() + { + var chatDefaultOptions = this.SettingsManager.ConfigurationData.Chat.PreselectedDataSourceOptions.CreateCopy(); + this.earlyDataSourceOptions = chatDefaultOptions; + this.dataSourceSelectionComponent?.ChangeOptionWithoutSaving(chatDefaultOptions); + } + private string ExtractThreadName(string firstUserInput) { // We select the first 10 words of the user input: @@ -231,8 +305,35 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable await this.ChatThreadChanged.InvokeAsync(this.ChatThread); } + private DataSourceOptions GetCurrentDataSourceOptions() + { + if (this.ChatThread is not null) + return this.ChatThread.DataSourceOptions; + + return this.earlyDataSourceOptions; + } + + private async Task SetCurrentDataSourceOptions(DataSourceOptions updatedOptions) + { + if (this.ChatThread is not null) + { + this.hasUnsavedChanges = true; + this.ChatThread.DataSourceOptions = updatedOptions; + if(this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY) + { + await this.SaveThread(); + this.hasUnsavedChanges = false; + } + } + else + this.earlyDataSourceOptions = updatedOptions; + } + private async Task InputKeyEvent(KeyboardEventArgs keyEvent) { + if(this.dataSourceSelectionComponent?.IsVisible ?? false) + this.dataSourceSelectionComponent.Hide(); + this.hasUnsavedChanges = true; var key = keyEvent.Code.ToLowerInvariant(); @@ -276,6 +377,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable SystemPrompt = SystemPrompts.DEFAULT, WorkspaceId = this.currentWorkspaceId, ChatId = Guid.NewGuid(), + DataSourceOptions = this.earlyDataSourceOptions, Name = this.ExtractThreadName(this.userInput), Seed = this.RNG.Next(), Blocks = [], @@ -365,7 +467,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable // Use the selected provider to get the AI response. // By awaiting this line, we wait for the entire // content to be streamed. - await aiText.CreateFromProviderAsync(this.Provider.CreateProvider(this.Logger), this.SettingsManager, this.Provider.Model, lastUserPrompt, this.ChatThread, this.cancellationTokenSource.Token); + await aiText.CreateFromProviderAsync(this.Provider.CreateProvider(this.Logger), this.SettingsManager, this.DataSourceService, this.Provider.Model, lastUserPrompt, this.ChatThread, this.cancellationTokenSource.Token); } this.cancellationTokenSource = null; @@ -412,6 +514,10 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable private async Task StartNewChat(bool useSameWorkspace = false, bool deletePreviousChat = false) { + // + // Want the user to manage the chat storage manually? In that case, we have to ask the user + // about possible data loss: + // if (this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_MANUALLY && this.hasUnsavedChanges) { var dialogParameters = new DialogParameters @@ -425,6 +531,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable return; } + // + // Delete the previous chat when desired and necessary: + // if (this.ChatThread is not null && deletePreviousChat) { string chatPath; @@ -439,10 +548,16 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable await this.Workspaces.DeleteChat(chatPath, askForConfirmation: false, unloadChat: true); } + // + // Reset our state: + // this.isStreaming = false; this.hasUnsavedChanges = false; this.userInput = string.Empty; + // + // Reset the LLM provider considering the user's settings: + // switch (this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior) { case AddChatProviderBehavior.ADDED_CHATS_USE_DEFAULT_PROVIDER: @@ -461,8 +576,16 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable break; } + // + // Reset the chat thread or create a new one: + // if (!useSameWorkspace) { + // + // When the user wants to start a new chat outside the current workspace, + // we have to reset the workspace id and the workspace name. Also, we have + // to reset the chat thread: + // this.ChatThread = null; this.currentWorkspaceId = Guid.Empty; this.currentWorkspaceName = string.Empty; @@ -470,6 +593,11 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable } else { + // + // When the user wants to start a new chat in the same workspace, we have to + // reset the chat thread only. The workspace id and the workspace name remain + // the same: + // this.ChatThread = new() { SelectedProvider = this.Provider.Id, @@ -482,8 +610,11 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable Blocks = [], }; } - - this.userInput = string.Empty; + + // Now, we have to reset the data source options as well: + this.ApplyStandardDataSourceOptions(); + + // Notify the parent component about the change: await this.ChatThreadChanged.InvokeAsync(this.ChatThread); } @@ -543,12 +674,14 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable this.currentWorkspaceId = this.ChatThread.WorkspaceId; this.currentWorkspaceName = await WorkspaceBehaviour.LoadWorkspaceName(this.ChatThread.WorkspaceId); this.WorkspaceName(this.currentWorkspaceName); + this.dataSourceSelectionComponent?.ChangeOptionWithoutSaving(this.ChatThread.DataSourceOptions); } else { this.currentWorkspaceId = Guid.Empty; this.currentWorkspaceName = string.Empty; this.WorkspaceName(this.currentWorkspaceName); + this.ApplyStandardDataSourceOptions(); } await this.SelectProviderWhenLoadingChat(); @@ -572,6 +705,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable this.WorkspaceName(this.currentWorkspaceName); this.ChatThread = null; + this.ApplyStandardDataSourceOptions(); await this.ChatThreadChanged.InvokeAsync(this.ChatThread); } diff --git a/app/MindWork AI Studio/Components/ConfidenceInfo.razor b/app/MindWork AI Studio/Components/ConfidenceInfo.razor index f120b91..48a8bec 100644 --- a/app/MindWork AI Studio/Components/ConfidenceInfo.razor +++ b/app/MindWork AI Studio/Components/ConfidenceInfo.razor @@ -1,7 +1,7 @@ @using AIStudio.Provider
- @if (this.Mode is ConfidenceInfoMode.ICON) + @if (this.Mode is PopoverTriggerMode.ICON) { } @@ -20,7 +20,7 @@ Confidence Card - + Description diff --git a/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs b/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs index 97db85f..b586a2f 100644 --- a/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs +++ b/app/MindWork AI Studio/Components/ConfidenceInfo.razor.cs @@ -8,7 +8,7 @@ namespace AIStudio.Components; public partial class ConfidenceInfo : ComponentBase, IMessageBusReceiver, IDisposable { [Parameter] - public ConfidenceInfoMode Mode { get; set; } = ConfidenceInfoMode.BUTTON; + public PopoverTriggerMode Mode { get; set; } = PopoverTriggerMode.BUTTON; [Parameter] public LLMProviders LLMProvider { get; set; } diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor b/app/MindWork AI Studio/Components/DataSourceSelection.razor new file mode 100644 index 0000000..f9dff17 --- /dev/null +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor @@ -0,0 +1,93 @@ +@using AIStudio.Settings + +@if (this.SelectionMode is DataSourceSelectionMode.SELECTION_MODE) +{ +
+ + @if (this.PopoverTriggerMode is PopoverTriggerMode.ICON) + { + + } + else + { + + Select data sources + + } + + + + + + + + Data Source Selection + + + + @if (this.waitingForDataSources) + { + + + + } + else if (this.showDataSourceSelection) + { + + @if (this.areDataSourcesEnabled) + { + + + + + @foreach (var source in this.availableDataSources) + { + + @source.Name + + } + + + } + } + + + + Close + + + + +
+} +else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE) +{ + + + Data Source Selection + + @if (!string.IsNullOrWhiteSpace(this.ConfigurationHeaderMessage)) + { + + @this.ConfigurationHeaderMessage + + } + + + @if (this.areDataSourcesEnabled) + { + + + + + @foreach (var source in this.availableDataSources) + { + + @source.Name + + } + + + } + +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs b/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs new file mode 100644 index 0000000..7b75bbd --- /dev/null +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs @@ -0,0 +1,260 @@ +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +public partial class DataSourceSelection : ComponentBase, IMessageBusReceiver, IDisposable +{ + [Parameter] + public DataSourceSelectionMode SelectionMode { get; set; } = DataSourceSelectionMode.SELECTION_MODE; + + [Parameter] + public PopoverTriggerMode PopoverTriggerMode { get; set; } = PopoverTriggerMode.BUTTON; + + [Parameter] + public string PopoverButtonClasses { get; set; } = string.Empty; + + [Parameter] + public required AIStudio.Settings.Provider LLMProvider { get; set; } + + [Parameter] + public required DataSourceOptions DataSourceOptions { get; set; } + + [Parameter] + public EventCallback DataSourceOptionsChanged { get; set; } + + [Parameter] + public string ConfigurationHeaderMessage { get; set; } = string.Empty; + + [Parameter] + public bool AutoSaveAppSettings { get; set; } + + [Inject] + private SettingsManager SettingsManager { get; init; } = null!; + + [Inject] + private MessageBus MessageBus { get; init; } = null!; + + [Inject] + private DataSourceService DataSourceService { get; init; } = null!; + + [Inject] + private ILogger Logger { get; init; } = null!; + + private bool internalChange; + private bool showDataSourceSelection; + private bool waitingForDataSources = true; + private IReadOnlyList availableDataSources = []; + private IReadOnlyCollection selectedDataSources = []; + private bool aiBasedSourceSelection; + private bool aiBasedValidation; + private bool areDataSourcesEnabled; + + #region Overrides of ComponentBase + + protected override async Task OnInitializedAsync() + { + this.MessageBus.RegisterComponent(this); + this.MessageBus.ApplyFilters(this, [], [ Event.COLOR_THEME_CHANGED ]); + + // + // Load the settings: + // + this.aiBasedSourceSelection = this.DataSourceOptions.AutomaticDataSourceSelection; + this.aiBasedValidation = this.DataSourceOptions.AutomaticValidation; + this.areDataSourcesEnabled = !this.DataSourceOptions.DisableDataSources; + this.waitingForDataSources = this.areDataSourcesEnabled; + + // + // Preselect the data sources. Right now, we cannot filter + // the data sources. Later, when the component is shown, we + // will filter the data sources. + // + // Right before the preselection would be used to kick off the + // RAG process, we will filter the data sources as well. + // + var preselectedSources = new List(this.DataSourceOptions.PreselectedDataSourceIds.Count); + foreach (var preselectedDataSourceId in this.DataSourceOptions.PreselectedDataSourceIds) + { + var dataSource = this.SettingsManager.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == preselectedDataSourceId); + if (dataSource is not null) + preselectedSources.Add(dataSource); + } + + this.selectedDataSources = preselectedSources; + await base.OnInitializedAsync(); + } + + protected override async Task OnParametersSetAsync() + { + if (!this.internalChange) + { + this.aiBasedSourceSelection = this.DataSourceOptions.AutomaticDataSourceSelection; + this.aiBasedValidation = this.DataSourceOptions.AutomaticValidation; + this.areDataSourcesEnabled = !this.DataSourceOptions.DisableDataSources; + } + + switch (this.SelectionMode) + { + // + // In selection mode, we have to load & filter the data sources + // when the component is shown: + // + case DataSourceSelectionMode.SELECTION_MODE: + + // + // For external changes, we have to reload & filter + // the data sources: + // + if (this.showDataSourceSelection && !this.internalChange) + await this.LoadAndApplyFilters(); + else + this.internalChange = false; + + break; + + // + // In configuration mode, we have to load all data sources: + // + case DataSourceSelectionMode.CONFIGURATION_MODE: + this.availableDataSources = this.SettingsManager.ConfigurationData.DataSources; + break; + } + + await base.OnParametersSetAsync(); + } + + #endregion + + public void ChangeOptionWithoutSaving(DataSourceOptions options) + { + this.DataSourceOptions = options; + this.aiBasedSourceSelection = this.DataSourceOptions.AutomaticDataSourceSelection; + this.aiBasedValidation = this.DataSourceOptions.AutomaticValidation; + this.areDataSourcesEnabled = !this.DataSourceOptions.DisableDataSources; + this.selectedDataSources = this.SettingsManager.ConfigurationData.DataSources.Where(ds => this.DataSourceOptions.PreselectedDataSourceIds.Contains(ds.Id)).ToList(); + this.waitingForDataSources = false; + + // + // Remark: We do not apply the filters here. This is done later + // when either the parameters are changed or just before the + // RAG process is started (outside of this component). + // + // In fact, when we apply the filters here, multiple calls + // to the filter method would be made. We would get conflicts. + // + } + + public bool IsVisible => this.showDataSourceSelection; + + public void Hide() + { + this.showDataSourceSelection = false; + this.StateHasChanged(); + } + + private async Task LoadAndApplyFilters() + { + if(this.DataSourceOptions.DisableDataSources) + return; + + this.waitingForDataSources = true; + this.StateHasChanged(); + + // Load the data sources: + var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.selectedDataSources); + this.availableDataSources = sources.AllowedDataSources; + this.selectedDataSources = sources.SelectedDataSources; + this.waitingForDataSources = false; + this.StateHasChanged(); + } + + private async Task EnabledChanged(bool state) + { + this.areDataSourcesEnabled = state; + this.DataSourceOptions.DisableDataSources = !this.areDataSourcesEnabled; + + await this.LoadAndApplyFilters(); + await this.OptionsChanged(); + this.StateHasChanged(); + } + + private async Task AutoModeChanged(bool state) + { + this.aiBasedSourceSelection = state; + this.DataSourceOptions.AutomaticDataSourceSelection = this.aiBasedSourceSelection; + + await this.OptionsChanged(); + } + + private async Task ValidationModeChanged(bool state) + { + this.aiBasedValidation = state; + this.DataSourceOptions.AutomaticValidation = this.aiBasedValidation; + + await this.OptionsChanged(); + } + + private async Task SelectionChanged(IReadOnlyCollection? chosenDataSources) + { + this.selectedDataSources = chosenDataSources ?? []; + this.DataSourceOptions.PreselectedDataSourceIds = this.selectedDataSources.Select(ds => ds.Id).ToList(); + + await this.OptionsChanged(); + } + + private async Task OptionsChanged() + { + this.internalChange = true; + + await this.DataSourceOptionsChanged.InvokeAsync(this.DataSourceOptions); + + if(this.AutoSaveAppSettings) + await this.SettingsManager.StoreSettings(); + } + + private async Task ToggleDataSourceSelection() + { + this.showDataSourceSelection = !this.showDataSourceSelection; + if (this.showDataSourceSelection) + await this.LoadAndApplyFilters(); + } + + private void HideDataSourceSelection() => this.showDataSourceSelection = false; + + #region Implementation of IMessageBusReceiver + + public string ComponentName => nameof(ConfidenceInfo); + + public Task ProcessMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) + { + switch (triggeredEvent) + { + case Event.COLOR_THEME_CHANGED: + this.showDataSourceSelection = false; + this.StateHasChanged(); + break; + } + + return Task.CompletedTask; + } + + public Task ProcessMessageWithResult(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data) + { + return Task.FromResult(default); + } + + #endregion + + #region Implementation of IDisposable + + public void Dispose() + { + this.MessageBus.Unregister(this); + } + + #endregion +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataSourceSelectionMode.cs b/app/MindWork AI Studio/Components/DataSourceSelectionMode.cs new file mode 100644 index 0000000..e10a013 --- /dev/null +++ b/app/MindWork AI Studio/Components/DataSourceSelectionMode.cs @@ -0,0 +1,23 @@ +namespace AIStudio.Components; + +public enum DataSourceSelectionMode +{ + /// + /// The user is selecting data sources for, e.g., the chat. + /// + /// + /// In this case, we have to filter the data sources based on the + /// selected provider and check security requirements. + /// + SELECTION_MODE, + + /// + /// The user is configuring the default data sources, e.g., for the chat. + /// + /// + /// In this case, all data sources are available for selection. + /// They get filtered later based on the selected provider and + /// security requirements. + /// + CONFIGURATION_MODE, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ConfidenceInfoMode.cs b/app/MindWork AI Studio/Components/PopoverTriggerMode.cs similarity index 63% rename from app/MindWork AI Studio/Components/ConfidenceInfoMode.cs rename to app/MindWork AI Studio/Components/PopoverTriggerMode.cs index d7e63da..c122e40 100644 --- a/app/MindWork AI Studio/Components/ConfidenceInfoMode.cs +++ b/app/MindWork AI Studio/Components/PopoverTriggerMode.cs @@ -1,6 +1,6 @@ namespace AIStudio.Components; -public enum ConfidenceInfoMode +public enum PopoverTriggerMode { BUTTON, ICON, diff --git a/app/MindWork AI Studio/Components/ProfileSelection.razor.cs b/app/MindWork AI Studio/Components/ProfileSelection.razor.cs index 55f2fa9..d2b41a5 100644 --- a/app/MindWork AI Studio/Components/ProfileSelection.razor.cs +++ b/app/MindWork AI Studio/Components/ProfileSelection.razor.cs @@ -14,11 +14,14 @@ public partial class ProfileSelection : ComponentBase [Parameter] public string MarginLeft { get; set; } = "ml-3"; + + [Parameter] + public string MarginRight { get; set; } = string.Empty; [Inject] private SettingsManager SettingsManager { get; init; } = null!; - private string MarginClass => $"{this.MarginLeft}"; + private string MarginClass => $"{this.MarginLeft} {this.MarginRight}"; private async Task SelectionChanged(Profile profile) { diff --git a/app/MindWork AI Studio/Components/SelectDirectory.razor.cs b/app/MindWork AI Studio/Components/SelectDirectory.razor.cs index 79bc18e..a4ebbf8 100644 --- a/app/MindWork AI Studio/Components/SelectDirectory.razor.cs +++ b/app/MindWork AI Studio/Components/SelectDirectory.razor.cs @@ -1,4 +1,5 @@ using AIStudio.Settings; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; diff --git a/app/MindWork AI Studio/Components/SelectFile.razor.cs b/app/MindWork AI Studio/Components/SelectFile.razor.cs index 5d1b7f0..d4a03ad 100644 --- a/app/MindWork AI Studio/Components/SelectFile.razor.cs +++ b/app/MindWork AI Studio/Components/SelectFile.razor.cs @@ -1,4 +1,5 @@ using AIStudio.Settings; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelBase.cs b/app/MindWork AI Studio/Components/Settings/SettingsPanelBase.cs index f7f3a1d..c338416 100644 --- a/app/MindWork AI Studio/Components/Settings/SettingsPanelBase.cs +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelBase.cs @@ -1,4 +1,5 @@ using AIStudio.Settings; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelChat.razor b/app/MindWork AI Studio/Components/Settings/SettingsPanelChat.razor index 4f677ae..8b27440 100644 --- a/app/MindWork AI Studio/Components/Settings/SettingsPanelChat.razor +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelChat.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings +@using AIStudio.Settings.DataModel @inherits SettingsPanelBase @@ -12,4 +13,10 @@ + + @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager)) + { + + + } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/TextInfoLine.razor.cs b/app/MindWork AI Studio/Components/TextInfoLine.razor.cs index 4c4e640..206fae4 100644 --- a/app/MindWork AI Studio/Components/TextInfoLine.razor.cs +++ b/app/MindWork AI Studio/Components/TextInfoLine.razor.cs @@ -1,4 +1,5 @@ using AIStudio.Settings; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; diff --git a/app/MindWork AI Studio/Components/TextInfoLines.razor b/app/MindWork AI Studio/Components/TextInfoLines.razor index 02dadb4..6818631 100644 --- a/app/MindWork AI Studio/Components/TextInfoLines.razor +++ b/app/MindWork AI Studio/Components/TextInfoLines.razor @@ -9,6 +9,7 @@ Lines="3" MaxLines="@this.MaxLines" AutoGrow="@true" + Style="@this.GetColor()" UserAttributes="@USER_INPUT_ATTRIBUTES" /> @if (this.ShowingCopyButton) diff --git a/app/MindWork AI Studio/Components/TextInfoLines.razor.cs b/app/MindWork AI Studio/Components/TextInfoLines.razor.cs index 64d5924..2427133 100644 --- a/app/MindWork AI Studio/Components/TextInfoLines.razor.cs +++ b/app/MindWork AI Studio/Components/TextInfoLines.razor.cs @@ -1,4 +1,5 @@ using AIStudio.Settings; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; @@ -20,6 +21,9 @@ public partial class TextInfoLines : ComponentBase [Parameter] public bool ShowingCopyButton { get; set; } = true; + + [Parameter] + public TextColor Color { get; set; } = TextColor.DEFAULT; [Inject] private RustService RustService { get; init; } = null!; @@ -47,4 +51,13 @@ public partial class TextInfoLines : ComponentBase private string ClipboardTooltip => $"Copy {this.ClipboardTooltipSubject} to the clipboard"; private async Task CopyToClipboard(string content) => await this.RustService.CopyText2Clipboard(this.Snackbar, content); + + private string GetColor() + { + var htmlColorCode = this.Color.GetHTMLColor(this.SettingsManager); + if(string.IsNullOrWhiteSpace(htmlColorCode)) + return string.Empty; + + return $"color: {htmlColorCode} !important;"; + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor index a1ebe82..984d2e4 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor @@ -1,4 +1,6 @@ +@using AIStudio.Settings.DataModel @using AIStudio.Tools.ERIClient.DataModel + @@ -25,7 +27,8 @@ } - + + Retrieval information diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor.cs index 304c4f6..192347d 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor.cs @@ -6,6 +6,7 @@ using AIStudio.Assistants.ERI; using AIStudio.Settings.DataModel; using AIStudio.Tools.ERIClient; using AIStudio.Tools.ERIClient.DataModel; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor index ac40ffa..fcc9fff 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor @@ -1,3 +1,4 @@ +@using AIStudio.Settings.DataModel @using AIStudio.Tools.ERIClient.DataModel @@ -108,6 +109,13 @@ InputType="InputType.Password" UserAttributes="@SPELLCHECK_ATTRIBUTES"/> } + + + @foreach (var policy in Enum.GetValues()) + { + @policy.ToSelectionText() + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs index aaa4dac..e042f92 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs @@ -3,6 +3,7 @@ using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.ERIClient; using AIStudio.Tools.ERIClient.DataModel; +using AIStudio.Tools.Services; using AIStudio.Tools.Validation; using Microsoft.AspNetCore.Components; @@ -45,6 +46,8 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId private string dataSecretStorageIssue = string.Empty; private string dataEditingPreviousInstanceName = string.Empty; private List availableAuthMethods = []; + private DataSourceSecurity dataSecurityPolicy; + private SecurityRequirements dataSourceSecurityRequirements; private bool connectionTested; private bool connectionSuccessfulTested; @@ -71,6 +74,7 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId GetTestedConnection = () => this.connectionTested, GetTestedConnectionResult = () => this.connectionSuccessfulTested, GetAvailableAuthMethods = () => this.availableAuthMethods, + GetSecurityRequirements = () => this.dataSourceSecurityRequirements, }; } @@ -95,6 +99,7 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId this.dataPort = this.DataSource.Port; this.dataAuthMethod = this.DataSource.AuthMethod; this.dataUsername = this.DataSource.Username; + this.dataSecurityPolicy = this.DataSource.SecurityPolicy; if (this.dataAuthMethod is AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD) { @@ -147,6 +152,7 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId AuthMethod = this.dataAuthMethod, Username = this.dataUsername, Type = DataSourceType.ERI_V1, + SecurityPolicy = this.dataSecurityPolicy, }; } @@ -196,6 +202,28 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId this.availableAuthMethods = authSchemes.Data!.Select(n => n.AuthMethod).ToList(); + var loginResult = await client.AuthenticateAsync(this.DataSource, this.RustService, cts.Token); + if (!loginResult.Successful) + { + await this.form.Validate(); + + Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1); + this.dataIssues[^1] = loginResult.Message; + return; + } + + var securityRequirementsRequest = await client.GetSecurityRequirementsAsync(cts.Token); + if (!securityRequirementsRequest.Successful) + { + await this.form.Validate(); + + Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1); + this.dataIssues[^1] = securityRequirementsRequest.Message; + return; + } + + this.dataSourceSecurityRequirements = securityRequirementsRequest.Data; + this.connectionTested = true; this.connectionSuccessfulTested = true; this.Logger.LogInformation("Connection to the ERI v1 server was successful tested."); diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index 22d1c98..e786328 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -1,3 +1,5 @@ +@using AIStudio.Settings.DataModel + @@ -60,6 +62,13 @@ } } + + + @foreach (var policy in Enum.GetValues()) + { + @policy.ToSelectionText() + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index 9f1b4f1..30cccbe 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -42,6 +42,7 @@ public partial class DataSourceLocalDirectoryDialog : ComponentBase private bool dataUserAcknowledgedCloudEmbedding; private string dataEmbeddingId = string.Empty; private string dataPath = string.Empty; + private DataSourceSecurity dataSecurityPolicy; // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; @@ -75,6 +76,7 @@ public partial class DataSourceLocalDirectoryDialog : ComponentBase this.dataName = this.DataSource.Name; this.dataEmbeddingId = this.DataSource.EmbeddingId; this.dataPath = this.DataSource.Path; + this.dataSecurityPolicy = this.DataSource.SecurityPolicy; } await base.OnInitializedAsync(); @@ -102,6 +104,7 @@ public partial class DataSourceLocalDirectoryDialog : ComponentBase Type = DataSourceType.LOCAL_DIRECTORY, EmbeddingId = this.dataEmbeddingId, Path = this.dataPath, + SecurityPolicy = this.dataSecurityPolicy, }; private async Task Store() diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor index cbd951d..a4b647b 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor @@ -1,3 +1,5 @@ +@using AIStudio.Settings.DataModel + @@ -31,6 +33,8 @@ } + + @if (this.directorySizeNumFiles > 100) diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index c60d32d..ebd0a5b 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -1,3 +1,4 @@ +@using AIStudio.Settings.DataModel @@ -59,6 +60,13 @@ } } + + + @foreach (var policy in Enum.GetValues()) + { + @policy.ToSelectionText() + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index 6a2cd14..726f211 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -42,6 +42,7 @@ public partial class DataSourceLocalFileDialog : ComponentBase private bool dataUserAcknowledgedCloudEmbedding; private string dataEmbeddingId = string.Empty; private string dataFilePath = string.Empty; + private DataSourceSecurity dataSecurityPolicy; // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; @@ -75,6 +76,7 @@ public partial class DataSourceLocalFileDialog : ComponentBase this.dataName = this.DataSource.Name; this.dataEmbeddingId = this.DataSource.EmbeddingId; this.dataFilePath = this.DataSource.FilePath; + this.dataSecurityPolicy = this.DataSource.SecurityPolicy; } await base.OnInitializedAsync(); @@ -102,6 +104,7 @@ public partial class DataSourceLocalFileDialog : ComponentBase Type = DataSourceType.LOCAL_FILE, EmbeddingId = this.dataEmbeddingId, FilePath = this.dataFilePath, + SecurityPolicy = this.dataSecurityPolicy, }; private async Task Store() diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor index 6fb4b12..0605ff9 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor @@ -1,3 +1,5 @@ +@using AIStudio.Settings.DataModel + @@ -31,6 +33,7 @@ } + diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs index 399c9b8..759d9a1 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs @@ -1,5 +1,6 @@ using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.Services; using AIStudio.Tools.Validation; using Microsoft.AspNetCore.Components; diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs index 87e5569..a721b95 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs @@ -1,11 +1,11 @@ using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.Services; using AIStudio.Tools.Validation; using Microsoft.AspNetCore.Components; using Host = AIStudio.Provider.SelfHosted.Host; -using RustService = AIStudio.Tools.RustService; namespace AIStudio.Dialogs; diff --git a/app/MindWork AI Studio/Layout/MainLayout.razor.cs b/app/MindWork AI Studio/Layout/MainLayout.razor.cs index 47a09be..b302b23 100644 --- a/app/MindWork AI Studio/Layout/MainLayout.razor.cs +++ b/app/MindWork AI Studio/Layout/MainLayout.razor.cs @@ -8,7 +8,6 @@ using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Routing; using DialogOptions = AIStudio.Dialogs.DialogOptions; -using RustService = AIStudio.Tools.RustService; namespace AIStudio.Layout; diff --git a/app/MindWork AI Studio/Pages/Writer.razor.cs b/app/MindWork AI Studio/Pages/Writer.razor.cs index 60280d1..7f95d7e 100644 --- a/app/MindWork AI Studio/Pages/Writer.razor.cs +++ b/app/MindWork AI Studio/Pages/Writer.razor.cs @@ -1,6 +1,7 @@ using AIStudio.Chat; using AIStudio.Components; using AIStudio.Provider; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Web; @@ -14,6 +15,9 @@ public partial class Writer : MSGComponentBase, IAsyncDisposable [Inject] private ILogger Logger { get; init; } = null!; + [Inject] + private DataSourceService DataSourceService { get; init; } = null!; + private static readonly Dictionary USER_INPUT_ATTRIBUTES = new(); private readonly Timer typeTimer = new(TimeSpan.FromMilliseconds(1_500)); @@ -139,7 +143,7 @@ public partial class Writer : MSGComponentBase, IAsyncDisposable this.isStreaming = true; this.StateHasChanged(); - await aiText.CreateFromProviderAsync(this.providerSettings.CreateProvider(this.Logger), this.SettingsManager, this.providerSettings.Model, lastUserPrompt, this.chatThread); + await aiText.CreateFromProviderAsync(this.providerSettings.CreateProvider(this.Logger), this.SettingsManager, this.DataSourceService, this.providerSettings.Model, lastUserPrompt, this.chatThread); this.suggestion = aiText.Text; this.isStreaming = false; diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index 0844cae..b18bdd1 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -115,6 +115,7 @@ internal sealed class Program builder.Services.AddMudMarkdownClipboardService(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddTransient(); builder.Services.AddTransient(); builder.Services.AddHostedService(); @@ -143,12 +144,18 @@ internal sealed class Program // Execute the builder to get the app: var app = builder.Build(); + // Get a program logger: + var programLogger = app.Services.GetRequiredService>(); + programLogger.LogInformation("Starting the AI Studio server."); + // Initialize the encryption service: + programLogger.LogInformation("Initializing the encryption service."); var encryptionLogger = app.Services.GetRequiredService>(); var encryption = new Encryption(encryptionLogger, secretPassword, secretKeySalt); var encryptionInitializer = encryption.Initialize(); // Set the logger for the Rust service: + programLogger.LogInformation("Initializing the Rust service."); var rustLogger = app.Services.GetRequiredService>(); rust.SetLogger(rustLogger); rust.SetEncryptor(encryption); @@ -156,6 +163,7 @@ internal sealed class Program RUST_SERVICE = rust; ENCRYPTION = encryption; + programLogger.LogInformation("Initialize internal file system."); app.Use(Redirect.HandlerContentAsync); #if DEBUG @@ -175,9 +183,18 @@ internal sealed class Program .AddInteractiveServerRenderMode(); var serverTask = app.RunAsync(); + programLogger.LogInformation("Server was started successfully."); await encryptionInitializer; await rust.AppIsReady(); + programLogger.LogInformation("The AI Studio server is ready."); + + TaskScheduler.UnobservedTaskException += (sender, taskArgs) => + { + programLogger.LogError(taskArgs.Exception, $"Unobserved task exception by sender '{sender ?? "n/a"}'."); + taskArgs.SetObserved(); + }; + await serverTask; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 2899e4f..3ad4e8f 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -4,8 +4,7 @@ using System.Text.Json; using AIStudio.Chat; using AIStudio.Settings; - -using RustService = AIStudio.Tools.RustService; +using AIStudio.Tools.Services; namespace AIStudio.Provider; diff --git a/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs b/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs index 3f6ec93..5a2b8f1 100644 --- a/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs +++ b/app/MindWork AI Studio/Settings/ConfigurationSelectData.cs @@ -94,6 +94,12 @@ public static class ConfigurationSelectDataFactory yield return new(source.GetPreviewDescription(), source); } + public static IEnumerable> GetSendToChatDataSourceBehaviorData() + { + foreach (var behavior in Enum.GetValues()) + yield return new(behavior.Description(), behavior); + } + public static IEnumerable> GetNavBehaviorData() { yield return new("Navigation expands on mouse hover", NavBehavior.EXPAND_ON_HOVER); diff --git a/app/MindWork AI Studio/Settings/DataModel/DataChat.cs b/app/MindWork AI Studio/Settings/DataModel/DataChat.cs index 8283150..baf995f 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataChat.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataChat.cs @@ -17,6 +17,11 @@ public sealed class DataChat ///
public AddChatProviderBehavior AddChatProviderBehavior { get; set; } = AddChatProviderBehavior.ADDED_CHATS_USE_LATEST_PROVIDER; + /// + /// Defines the data source behavior when sending assistant results to a chat. + /// + public SendToChatDataSourceBehavior SendToChatDataSourceBehavior { get; set; } = SendToChatDataSourceBehavior.NO_DATA_SOURCES; + /// /// Preselect any chat options? /// @@ -31,6 +36,11 @@ public sealed class DataChat /// Preselect a profile? /// public string PreselectedProfile { get; set; } = string.Empty; + + /// + /// Should we preselect data sources options for a created chat? + /// + public DataSourceOptions PreselectedDataSourceOptions { get; set; } = new(); /// /// Should we show the latest message after loading? When false, we show the first (aka oldest) message. diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs index addec25..a9931e3 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs @@ -36,4 +36,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource /// public string Username { get; init; } = string.Empty; + + /// + public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs index 963b9ba..61c30d9 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs @@ -24,6 +24,9 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource /// public string EmbeddingId { get; init; } = Guid.Empty.ToString(); + /// + public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + /// /// The path to the directory. /// diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs index a608819..571fb0a 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs @@ -24,6 +24,9 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource /// public string EmbeddingId { get; init; } = Guid.Empty.ToString(); + /// + public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + /// /// The path to the file. /// diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceOptions.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceOptions.cs new file mode 100644 index 0000000..1def988 --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceOptions.cs @@ -0,0 +1,65 @@ +namespace AIStudio.Settings.DataModel; + +public sealed class DataSourceOptions +{ + /// + /// Whether data sources are disabled in this context. + /// + public bool DisableDataSources { get; set; } = true; + + /// + /// Whether the data sources should be selected automatically. + /// + /// + /// When true, the appropriate data sources for the current prompt are + /// selected automatically. When false, the user has to select the + /// data sources manually. + /// + /// This setting does not affect the selection of the actual data + /// for augmentation. + /// + public bool AutomaticDataSourceSelection { get; set; } + + /// + /// Whether the retrieved data should be validated for the current prompt. + /// + /// + /// When true, the retrieved data is validated against the current prompt. + /// An AI will decide whether the data point is useful for answering the + /// prompt or not. + /// + public bool AutomaticValidation { get; set; } + + /// + /// The preselected data source IDs. When these data sources are available + /// for the selected provider, they are pre-selected. + /// + public List PreselectedDataSourceIds { get; set; } = []; + + /// + /// Returns true when data sources are enabled. + /// + /// True when data sources are enabled. + public bool IsEnabled() + { + if(this.DisableDataSources) + return false; + + if(this.AutomaticDataSourceSelection) + return true; + + return this.PreselectedDataSourceIds.Count > 0; + } + + /// + /// Creates a copy of the current data source options. + /// + /// A copy of the current data source options. + public DataSourceOptions CreateCopy() => new() + { + DisableDataSources = this.DisableDataSources, + AutomaticDataSourceSelection = this.AutomaticDataSourceSelection, + AutomaticValidation = this.AutomaticValidation, + PreselectedDataSourceIds = [..this.PreselectedDataSourceIds], + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurity.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurity.cs new file mode 100644 index 0000000..58caf45 --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurity.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Settings.DataModel; + +public enum DataSourceSecurity +{ + /// + /// The security of the data source is not specified yet. + /// + NOT_SPECIFIED, + + /// + /// This data can be used with any LLM provider. + /// + ALLOW_ANY, + + /// + /// This data can only be used for self-hosted LLM providers. + /// + SELF_HOSTED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurityExtensions.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurityExtensions.cs new file mode 100644 index 0000000..6e52d0f --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurityExtensions.cs @@ -0,0 +1,32 @@ +namespace AIStudio.Settings.DataModel; + +public static class DataSourceSecurityExtensions +{ + public static string ToSelectionText(this DataSourceSecurity security) => security switch + { + DataSourceSecurity.NOT_SPECIFIED => "Please select a security policy", + + DataSourceSecurity.ALLOW_ANY => "This data source can be used with any LLM provider. Your data may be sent to a cloud-based provider.", + DataSourceSecurity.SELF_HOSTED => "This data source can only be used with a self-hosted LLM provider. Your data will not be sent to any cloud-based provider.", + + _ => "Unknown security policy" + }; + + public static string ToInfoText(this DataSourceSecurity security) => security switch + { + DataSourceSecurity.NOT_SPECIFIED => "The security of the data source is not specified yet. You cannot use this data source until you specify a security policy.", + + DataSourceSecurity.ALLOW_ANY => "This data source can be used with any LLM provider. Your data may be sent to a cloud-based provider.", + DataSourceSecurity.SELF_HOSTED => "This data source can only be used with a self-hosted LLM provider. Your data will not be sent to any cloud-based provider.", + + _ => "Unknown security policy" + }; + + public static TextColor GetColor(this DataSourceSecurity security) => security switch + { + DataSourceSecurity.ALLOW_ANY => TextColor.WARN, + DataSourceSecurity.SELF_HOSTED => TextColor.SUCCESS, + + _ => TextColor.ERROR + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/PreviewFeatureExtensions.cs b/app/MindWork AI Studio/Settings/DataModel/PreviewFeaturesExtensions.cs similarity index 93% rename from app/MindWork AI Studio/Settings/DataModel/PreviewFeatureExtensions.cs rename to app/MindWork AI Studio/Settings/DataModel/PreviewFeaturesExtensions.cs index a6e6cf0..133d653 100644 --- a/app/MindWork AI Studio/Settings/DataModel/PreviewFeatureExtensions.cs +++ b/app/MindWork AI Studio/Settings/DataModel/PreviewFeaturesExtensions.cs @@ -1,6 +1,6 @@ namespace AIStudio.Settings.DataModel; -public static class PreviewFeatureExtensions +public static class PreviewFeaturesExtensions { public static string GetPreviewDescription(this PreviewFeatures feature) => feature switch { diff --git a/app/MindWork AI Studio/Settings/DataModel/SendToChatDataSourceBehavior.cs b/app/MindWork AI Studio/Settings/DataModel/SendToChatDataSourceBehavior.cs new file mode 100644 index 0000000..fcbcaf4 --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/SendToChatDataSourceBehavior.cs @@ -0,0 +1,7 @@ +namespace AIStudio.Settings.DataModel; + +public enum SendToChatDataSourceBehavior +{ + NO_DATA_SOURCES, + APPLY_STANDARD_CHAT_DATA_SOURCE_OPTIONS, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/SendToChatDataSourceBehaviorExtensions.cs b/app/MindWork AI Studio/Settings/DataModel/SendToChatDataSourceBehaviorExtensions.cs new file mode 100644 index 0000000..3894ba2 --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/SendToChatDataSourceBehaviorExtensions.cs @@ -0,0 +1,12 @@ +namespace AIStudio.Settings.DataModel; + +public static class SendToChatDataSourceBehaviorExtensions +{ + public static string Description(this SendToChatDataSourceBehavior behavior) => behavior switch + { + SendToChatDataSourceBehavior.NO_DATA_SOURCES => "Use no data sources, when sending an assistant result to a chat", + SendToChatDataSourceBehavior.APPLY_STANDARD_CHAT_DATA_SOURCE_OPTIONS => "Apply standard chat data source options, when sending an assistant result to a chat", + + _ => "Unknown behavior", + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/IDataSource.cs b/app/MindWork AI Studio/Settings/IDataSource.cs index 28bc3b9..72f4ad3 100644 --- a/app/MindWork AI Studio/Settings/IDataSource.cs +++ b/app/MindWork AI Studio/Settings/IDataSource.cs @@ -32,4 +32,9 @@ public interface IDataSource /// Which type of data source is this? /// public DataSourceType Type { get; init; } + + /// + /// Which data security policy is applied to this data source? + /// + public DataSourceSecurity SecurityPolicy { get; init; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs b/app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs new file mode 100644 index 0000000..1aed9d1 --- /dev/null +++ b/app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs @@ -0,0 +1,13 @@ +using AIStudio.Settings; + +namespace AIStudio.Tools; + +/// +/// Contains both the allowed and selected data sources. +/// +/// +/// The selected data sources are a subset of the allowed data sources. +/// +/// The allowed data sources. +/// The selected data sources, which are a subset of the allowed data sources. +public readonly record struct AllowedSelectedDataSources(IReadOnlyList AllowedDataSources, IReadOnlyList SelectedDataSources); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ERIClient/ERIClientV1.cs b/app/MindWork AI Studio/Tools/ERIClient/ERIClientV1.cs index 62e3f55..c57d927 100644 --- a/app/MindWork AI Studio/Tools/ERIClient/ERIClientV1.cs +++ b/app/MindWork AI Studio/Tools/ERIClient/ERIClientV1.cs @@ -3,6 +3,7 @@ using System.Text.Json; using AIStudio.Settings; using AIStudio.Tools.ERIClient.DataModel; +using AIStudio.Tools.Services; namespace AIStudio.Tools.ERIClient; @@ -12,332 +13,465 @@ public class ERIClientV1(string baseAddress) : ERIClientBase(baseAddress), IERIC public async Task>> GetAuthMethodsAsync(CancellationToken cancellationToken = default) { - using var response = await this.httpClient.GetAsync("/auth/methods", cancellationToken); - if(!response.IsSuccessStatusCode) + try + { + using var response = await this.httpClient.GetAsync("/auth/methods", cancellationToken); + if (!response.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to retrieve the authentication methods: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + }; + } + + var authMethods = await response.Content.ReadFromJsonAsync>(JSON_OPTIONS, cancellationToken); + if (authMethods is null) + { + return new() + { + Successful = false, + Message = "Failed to retrieve the authentication methods: the ERI server did not return a valid response." + }; + } + + return new() + { + Successful = true, + Data = authMethods + }; + } + catch (TaskCanceledException) { return new() { Successful = false, - Message = $"Failed to retrieve the authentication methods: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + Message = "Failed to retrieve the authentication methods: the request was canceled either by the user or due to a timeout." }; } - - var authMethods = await response.Content.ReadFromJsonAsync>(JSON_OPTIONS, cancellationToken); - if(authMethods is null) + catch (Exception e) { return new() { Successful = false, - Message = "Failed to retrieve the authentication methods: the ERI server did not return a valid response." + Message = $"Failed to retrieve the authentication methods due to an exception: {e.Message}" }; } - - return new() - { - Successful = true, - Data = authMethods - }; } public async Task> AuthenticateAsync(IERIDataSource dataSource, RustService rustService, CancellationToken cancellationToken = default) { - var authMethod = dataSource.AuthMethod; - var username = dataSource.Username; - switch (dataSource.AuthMethod) + try { - case AuthMethod.NONE: - using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}")) - { - using var noneAuthResponse = await this.httpClient.SendAsync(request, cancellationToken); - if(!noneAuthResponse.IsSuccessStatusCode) + var authMethod = dataSource.AuthMethod; + var username = dataSource.Username; + switch (dataSource.AuthMethod) + { + case AuthMethod.NONE: + using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}")) { + using var noneAuthResponse = await this.httpClient.SendAsync(request, cancellationToken); + if(!noneAuthResponse.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to authenticate with the ERI server. Code: {noneAuthResponse.StatusCode}, Reason: {noneAuthResponse.ReasonPhrase}" + }; + } + + var noneAuthResult = await noneAuthResponse.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); + if(noneAuthResult == default) + { + return new() + { + Successful = false, + Message = "Failed to authenticate with the ERI server: the response was invalid." + }; + } + + this.securityToken = noneAuthResult.Token ?? string.Empty; return new() { - Successful = false, - Message = $"Failed to authenticate with the ERI server. Code: {noneAuthResponse.StatusCode}, Reason: {noneAuthResponse.ReasonPhrase}" + Successful = true, + Data = noneAuthResult }; } - - var noneAuthResult = await noneAuthResponse.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); - if(noneAuthResult == default) - { - return new() - { - Successful = false, - Message = "Failed to authenticate with the ERI server: the response was invalid." - }; - } - - this.securityToken = noneAuthResult.Token ?? string.Empty; - return new() - { - Successful = true, - Data = noneAuthResult - }; - } - case AuthMethod.USERNAME_PASSWORD: - var passwordResponse = await rustService.GetSecret(dataSource); - if (!passwordResponse.Success) - { - return new() - { - Successful = false, - Message = "Failed to retrieve the password." - }; - } - - var password = await passwordResponse.Secret.Decrypt(Program.ENCRYPTION); - using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}")) - { - // We must send both values inside the header. The username field is named 'user'. - // The password field is named 'password'. - request.Headers.Add("user", username); - request.Headers.Add("password", password); - - using var usernamePasswordAuthResponse = await this.httpClient.SendAsync(request, cancellationToken); - if(!usernamePasswordAuthResponse.IsSuccessStatusCode) + case AuthMethod.USERNAME_PASSWORD: + var passwordResponse = await rustService.GetSecret(dataSource); + if (!passwordResponse.Success) { return new() { Successful = false, - Message = $"Failed to authenticate with the ERI server. Code: {usernamePasswordAuthResponse.StatusCode}, Reason: {usernamePasswordAuthResponse.ReasonPhrase}" + Message = "Failed to retrieve the password." }; } - - var usernamePasswordAuthResult = await usernamePasswordAuthResponse.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); - if(usernamePasswordAuthResult == default) - { - return new() - { - Successful = false, - Message = "Failed to authenticate with the server: the response was invalid." - }; - } - - this.securityToken = usernamePasswordAuthResult.Token ?? string.Empty; - return new() - { - Successful = true, - Data = usernamePasswordAuthResult - }; - } - case AuthMethod.TOKEN: - var tokenResponse = await rustService.GetSecret(dataSource); - if (!tokenResponse.Success) - { - return new() + var password = await passwordResponse.Secret.Decrypt(Program.ENCRYPTION); + using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}")) { - Successful = false, - Message = "Failed to retrieve the access token." - }; - } + // We must send both values inside the header. The username field is named 'user'. + // The password field is named 'password'. + request.Headers.Add("user", username); + request.Headers.Add("password", password); - var token = await tokenResponse.Secret.Decrypt(Program.ENCRYPTION); - using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}")) - { - request.Headers.Add("Authorization", $"Bearer {token}"); + using var usernamePasswordAuthResponse = await this.httpClient.SendAsync(request, cancellationToken); + if(!usernamePasswordAuthResponse.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to authenticate with the ERI server. Code: {usernamePasswordAuthResponse.StatusCode}, Reason: {usernamePasswordAuthResponse.ReasonPhrase}" + }; + } - using var tokenAuthResponse = await this.httpClient.SendAsync(request, cancellationToken); - if(!tokenAuthResponse.IsSuccessStatusCode) + var usernamePasswordAuthResult = await usernamePasswordAuthResponse.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); + if(usernamePasswordAuthResult == default) + { + return new() + { + Successful = false, + Message = "Failed to authenticate with the server: the response was invalid." + }; + } + + this.securityToken = usernamePasswordAuthResult.Token ?? string.Empty; + return new() + { + Successful = true, + Data = usernamePasswordAuthResult + }; + } + + case AuthMethod.TOKEN: + var tokenResponse = await rustService.GetSecret(dataSource); + if (!tokenResponse.Success) { return new() { Successful = false, - Message = $"Failed to authenticate with the ERI server. Code: {tokenAuthResponse.StatusCode}, Reason: {tokenAuthResponse.ReasonPhrase}" + Message = "Failed to retrieve the access token." }; } - - var tokenAuthResult = await tokenAuthResponse.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); - if(tokenAuthResult == default) + + var token = await tokenResponse.Secret.Decrypt(Program.ENCRYPTION); + using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}")) { + request.Headers.Add("Authorization", $"Bearer {token}"); + + using var tokenAuthResponse = await this.httpClient.SendAsync(request, cancellationToken); + if(!tokenAuthResponse.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to authenticate with the ERI server. Code: {tokenAuthResponse.StatusCode}, Reason: {tokenAuthResponse.ReasonPhrase}" + }; + } + + var tokenAuthResult = await tokenAuthResponse.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); + if(tokenAuthResult == default) + { + return new() + { + Successful = false, + Message = "Failed to authenticate with the ERI server: the response was invalid." + }; + } + + this.securityToken = tokenAuthResult.Token ?? string.Empty; return new() { - Successful = false, - Message = "Failed to authenticate with the ERI server: the response was invalid." + Successful = true, + Data = tokenAuthResult }; } - - this.securityToken = tokenAuthResult.Token ?? string.Empty; - return new() - { - Successful = true, - Data = tokenAuthResult - }; - } - default: - this.securityToken = string.Empty; - return new() - { - Successful = false, - Message = "The authentication method is not supported yet." - }; + default: + this.securityToken = string.Empty; + return new() + { + Successful = false, + Message = "The authentication method is not supported yet." + }; + } + } + catch(TaskCanceledException) + { + return new() + { + Successful = false, + Message = "Failed to authenticate with the ERI server: the request was canceled either by the user or due to a timeout." + }; + } + catch (Exception e) + { + return new() + { + Successful = false, + Message = $"Failed to authenticate with the ERI server due to an exception: {e.Message}" + }; } } public async Task> GetDataSourceInfoAsync(CancellationToken cancellationToken = default) { - using var request = new HttpRequestMessage(HttpMethod.Get, "/dataSource"); - request.Headers.Add("token", this.securityToken); + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/dataSource"); + request.Headers.Add("token", this.securityToken); - using var response = await this.httpClient.SendAsync(request, cancellationToken); - if(!response.IsSuccessStatusCode) + using var response = await this.httpClient.SendAsync(request, cancellationToken); + if(!response.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to retrieve the data source information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + }; + } + + var dataSourceInfo = await response.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); + if(dataSourceInfo == default) + { + return new() + { + Successful = false, + Message = "Failed to retrieve the data source information: the ERI server did not return a valid response." + }; + } + + return new() + { + Successful = true, + Data = dataSourceInfo + }; + } + catch(TaskCanceledException) { return new() { Successful = false, - Message = $"Failed to retrieve the data source information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + Message = "Failed to retrieve the data source information: the request was canceled either by the user or due to a timeout." }; } - - var dataSourceInfo = await response.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); - if(dataSourceInfo == default) + catch (Exception e) { return new() { Successful = false, - Message = "Failed to retrieve the data source information: the ERI server did not return a valid response." + Message = $"Failed to retrieve the data source information due to an exception: {e.Message}" }; } - - return new() - { - Successful = true, - Data = dataSourceInfo - }; } public async Task>> GetEmbeddingInfoAsync(CancellationToken cancellationToken = default) { - using var request = new HttpRequestMessage(HttpMethod.Get, "/embedding/info"); - request.Headers.Add("token", this.securityToken); + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/embedding/info"); + request.Headers.Add("token", this.securityToken); - using var response = await this.httpClient.SendAsync(request, cancellationToken); - if(!response.IsSuccessStatusCode) + using var response = await this.httpClient.SendAsync(request, cancellationToken); + if(!response.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to retrieve the embedding information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + }; + } + + var embeddingInfo = await response.Content.ReadFromJsonAsync>(JSON_OPTIONS, cancellationToken); + if(embeddingInfo is null) + { + return new() + { + Successful = false, + Message = "Failed to retrieve the embedding information: the ERI server did not return a valid response." + }; + } + + return new() + { + Successful = true, + Data = embeddingInfo + }; + } + catch(TaskCanceledException) { return new() { Successful = false, - Message = $"Failed to retrieve the embedding information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + Message = "Failed to retrieve the embedding information: the request was canceled either by the user or due to a timeout." }; } - - var embeddingInfo = await response.Content.ReadFromJsonAsync>(JSON_OPTIONS, cancellationToken); - if(embeddingInfo is null) + catch (Exception e) { return new() { Successful = false, - Message = "Failed to retrieve the embedding information: the ERI server did not return a valid response." + Message = $"Failed to retrieve the embedding information due to an exception: {e.Message}" }; } - - return new() - { - Successful = true, - Data = embeddingInfo - }; } public async Task>> GetRetrievalInfoAsync(CancellationToken cancellationToken = default) { - using var request = new HttpRequestMessage(HttpMethod.Get, "/retrieval/info"); - request.Headers.Add("token", this.securityToken); + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/retrieval/info"); + request.Headers.Add("token", this.securityToken); - using var response = await this.httpClient.SendAsync(request, cancellationToken); - if(!response.IsSuccessStatusCode) + using var response = await this.httpClient.SendAsync(request, cancellationToken); + if(!response.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to retrieve the retrieval information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + }; + } + + var retrievalInfo = await response.Content.ReadFromJsonAsync>(JSON_OPTIONS, cancellationToken); + if(retrievalInfo is null) + { + return new() + { + Successful = false, + Message = "Failed to retrieve the retrieval information: the ERI server did not return a valid response." + }; + } + + return new() + { + Successful = true, + Data = retrievalInfo + }; + } + catch(TaskCanceledException) { return new() { Successful = false, - Message = $"Failed to retrieve the retrieval information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + Message = "Failed to retrieve the retrieval information: the request was canceled either by the user or due to a timeout." }; } - - var retrievalInfo = await response.Content.ReadFromJsonAsync>(JSON_OPTIONS, cancellationToken); - if(retrievalInfo is null) + catch (Exception e) { return new() { Successful = false, - Message = "Failed to retrieve the retrieval information: the ERI server did not return a valid response." + Message = $"Failed to retrieve the retrieval information due to an exception: {e.Message}" }; } - - return new() - { - Successful = true, - Data = retrievalInfo - }; } public async Task>> ExecuteRetrievalAsync(RetrievalRequest request, CancellationToken cancellationToken = default) { - using var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/retrieval"); - requestMessage.Headers.Add("token", this.securityToken); + try + { + using var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/retrieval"); + requestMessage.Headers.Add("token", this.securityToken); - using var content = new StringContent(JsonSerializer.Serialize(request, JSON_OPTIONS), Encoding.UTF8, "application/json"); - requestMessage.Content = content; + using var content = new StringContent(JsonSerializer.Serialize(request, JSON_OPTIONS), Encoding.UTF8, "application/json"); + requestMessage.Content = content; - using var response = await this.httpClient.SendAsync(requestMessage, cancellationToken); - if(!response.IsSuccessStatusCode) + using var response = await this.httpClient.SendAsync(requestMessage, cancellationToken); + if(!response.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to execute the retrieval request: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + }; + } + + var contexts = await response.Content.ReadFromJsonAsync>(JSON_OPTIONS, cancellationToken); + if(contexts is null) + { + return new() + { + Successful = false, + Message = "Failed to execute the retrieval request: the ERI server did not return a valid response." + }; + } + + return new() + { + Successful = true, + Data = contexts + }; + } + catch(TaskCanceledException) { return new() { Successful = false, - Message = $"Failed to execute the retrieval request: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + Message = "Failed to execute the retrieval request: the request was canceled either by the user or due to a timeout." }; } - - var contexts = await response.Content.ReadFromJsonAsync>(JSON_OPTIONS, cancellationToken); - if(contexts is null) + catch (Exception e) { return new() { Successful = false, - Message = "Failed to execute the retrieval request: the ERI server did not return a valid response." + Message = $"Failed to execute the retrieval request due to an exception: {e.Message}" }; } - - return new() - { - Successful = true, - Data = contexts - }; } public async Task> GetSecurityRequirementsAsync(CancellationToken cancellationToken = default) { - using var request = new HttpRequestMessage(HttpMethod.Get, "/security/requirements"); - request.Headers.Add("token", this.securityToken); + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/security/requirements"); + request.Headers.Add("token", this.securityToken); - using var response = await this.httpClient.SendAsync(request, cancellationToken); - if(!response.IsSuccessStatusCode) + using var response = await this.httpClient.SendAsync(request, cancellationToken); + if(!response.IsSuccessStatusCode) + { + return new() + { + Successful = false, + Message = $"Failed to retrieve the security requirements: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + }; + } + + var securityRequirements = await response.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); + if(securityRequirements == default) + { + return new() + { + Successful = false, + Message = "Failed to retrieve the security requirements: the ERI server did not return a valid response." + }; + } + + return new() + { + Successful = true, + Data = securityRequirements + }; + } + catch(TaskCanceledException) { return new() { Successful = false, - Message = $"Failed to retrieve the security requirements: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}" + Message = "Failed to retrieve the security requirements: the request was canceled either by the user or due to a timeout." }; } - - var securityRequirements = await response.Content.ReadFromJsonAsync(JSON_OPTIONS, cancellationToken); - if(securityRequirements == default) + catch (Exception e) { return new() { Successful = false, - Message = "Failed to retrieve the security requirements: the ERI server did not return a valid response." + Message = $"Failed to retrieve the security requirements due to an exception: {e.Message}" }; } - - return new() - { - Successful = true, - Data = securityRequirements - }; } #endregion diff --git a/app/MindWork AI Studio/Tools/ERIClient/IERIClient.cs b/app/MindWork AI Studio/Tools/ERIClient/IERIClient.cs index 9cf27fb..0408e8d 100644 --- a/app/MindWork AI Studio/Tools/ERIClient/IERIClient.cs +++ b/app/MindWork AI Studio/Tools/ERIClient/IERIClient.cs @@ -1,5 +1,6 @@ using AIStudio.Settings; using AIStudio.Tools.ERIClient.DataModel; +using AIStudio.Tools.Services; namespace AIStudio.Tools.ERIClient; diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceService.cs new file mode 100644 index 0000000..de59308 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceService.cs @@ -0,0 +1,209 @@ +using AIStudio.Assistants.ERI; +using AIStudio.Provider; +using AIStudio.Provider.SelfHosted; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.ERIClient; +using AIStudio.Tools.ERIClient.DataModel; + +namespace AIStudio.Tools.Services; + +public sealed class DataSourceService +{ + private readonly RustService rustService; + private readonly SettingsManager settingsManager; + private readonly ILogger logger; + + public DataSourceService(SettingsManager settingsManager, ILogger logger, RustService rustService) + { + this.logger = logger; + this.rustService = rustService; + this.settingsManager = settingsManager; + + this.logger.LogInformation("The data source service has been initialized."); + } + + /// + /// Returns a list of data sources that are allowed for the selected LLM provider. + /// It also returns the data sources selected before when they are still allowed. + /// + /// The selected LLM provider. + /// The data sources selected before. + /// The allowed data sources and the data sources selected before -- when they are still allowed. + public async Task GetDataSources(AIStudio.Settings.Provider selectedLLMProvider, IReadOnlyCollection? previousSelectedDataSources = null) + { + // + // Case: Somehow the selected LLM provider was not set. The default provider + // does not mean anything. We cannot filter the data sources by any means. + // We return an empty list. Better safe than sorry. + // + if (selectedLLMProvider == default) + { + this.logger.LogWarning("The selected LLM provider is not set. We cannot filter the data sources by any means."); + return new([], []); + } + + return await this.GetDataSources(selectedLLMProvider.IsSelfHosted, previousSelectedDataSources); + } + + /// + /// Returns a list of data sources that are allowed for the selected LLM provider. + /// It also returns the data sources selected before when they are still allowed. + /// + /// The selected LLM provider. + /// The data sources selected before. + /// The allowed data sources and the data sources selected before -- when they are still allowed. + public async Task GetDataSources(IProvider selectedLLMProvider, IReadOnlyCollection? previousSelectedDataSources = null) + { + // + // Case: Somehow the selected LLM provider was not set. The default provider + // does not mean anything. We cannot filter the data sources by any means. + // We return an empty list. Better safe than sorry. + // + if (selectedLLMProvider is NoProvider) + { + this.logger.LogWarning("The selected LLM provider is the default provider. We cannot filter the data sources by any means."); + return new([], []); + } + + return await this.GetDataSources(selectedLLMProvider is ProviderSelfHosted, previousSelectedDataSources); + } + + private async Task GetDataSources(bool usingSelfHostedProvider, IReadOnlyCollection? previousSelectedDataSources = null) + { + var allDataSources = this.settingsManager.ConfigurationData.DataSources; + var filteredDataSources = new List(allDataSources.Count); + var filteredSelectedDataSources = new List(previousSelectedDataSources?.Count ?? 0); + var tasks = new List>(allDataSources.Count); + + // Start all checks in parallel: + foreach (var source in allDataSources) + tasks.Add(this.CheckOneDataSource(source, usingSelfHostedProvider)); + + // Wait for all checks and collect the results: + foreach (var task in tasks) + { + var source = await task; + if (source is not null) + { + filteredDataSources.Add(source); + if (previousSelectedDataSources is not null && previousSelectedDataSources.Contains(source)) + filteredSelectedDataSources.Add(source); + } + } + + return new(filteredDataSources, filteredSelectedDataSources); + } + + private async Task CheckOneDataSource(IDataSource source, bool usingSelfHostedProvider) + { + // + // Unfortunately, we have to live-check any ERI source for its security requirements. + // Because the ERI server operator might change the security requirements at any time. + // + SecurityRequirements? eriSourceRequirements = null; + if (source is DataSourceERI_V1 eriSource) + { + using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(6)); + using var client = ERIClientFactory.Get(ERIVersion.V1, eriSource); + if(client is null) + { + this.logger.LogError($"Could not create ERI client for source '{source.Name}' (id={source.Id}). We skip this source."); + return null; + } + + this.logger.LogInformation($"Authenticating with ERI source '{source.Name}' (id={source.Id})..."); + var loginResult = await client.AuthenticateAsync(eriSource, this.rustService, cancellationTokenSource.Token); + if (!loginResult.Successful) + { + this.logger.LogWarning($"Authentication with ERI source '{source.Name}' (id={source.Id}) failed. We skip this source. Reason: {loginResult.Message}"); + return null; + } + + this.logger.LogInformation($"Checking security requirements for ERI source '{source.Name}' (id={source.Id})..."); + var securityRequest = await client.GetSecurityRequirementsAsync(cancellationTokenSource.Token); + if (!securityRequest.Successful) + { + this.logger.LogWarning($"Could not retrieve security requirements for ERI source '{source.Name}' (id={source.Id}). We skip this source. Reason: {loginResult.Message}"); + return null; + } + + eriSourceRequirements = securityRequest.Data; + this.logger.LogInformation($"Security requirements for ERI source '{source.Name}' (id={source.Id}) retrieved successfully."); + } + + switch (source.SecurityPolicy) + { + case DataSourceSecurity.ALLOW_ANY: + + // + // Case: The data source allows any provider type. We want to use a self-hosted provider. + // There is no issue with this source. Accept it. + // + if(usingSelfHostedProvider) + return source; + + // + // Case: This is a local data source. When the source allows any provider type, we can use it. + // Accept it. + // + if(eriSourceRequirements is null) + return source; + + // + // Case: The ERI source requires a self-hosted provider. This misconfiguration happens + // when the ERI server operator changes the security requirements. The ERI server + // operator owns the data -- we have to respect their rules. We skip this source. + // + if (eriSourceRequirements is { AllowedProviderType: ProviderType.SELF_HOSTED }) + { + this.logger.LogWarning($"The ERI source '{source.Name}' (id={source.Id}) requires a self-hosted provider. We skip this source."); + return null; + } + + // + // Case: The ERI source allows any provider type. The data source configuration is correct. + // Accept it. + // + if(eriSourceRequirements is { AllowedProviderType: ProviderType.ANY }) + return source; + + // + // Case: Missing rules. We skip this source. Better safe than sorry. + // + this.logger.LogDebug($"The ERI source '{source.Name}' (id={source.Id}) was filtered out due to missing rules."); + return null; + + // + // Case: The data source requires a self-hosted provider. We want to use a self-hosted provider. + // There is no issue with this source. Accept it. + // + case DataSourceSecurity.SELF_HOSTED when usingSelfHostedProvider: + return source; + + // + // Case: The data source requires a self-hosted provider. We want to use a cloud provider. + // We skip this source. + // + case DataSourceSecurity.SELF_HOSTED when !usingSelfHostedProvider: + this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) requires a self-hosted provider. We skip this source."); + return null; + + // + // Case: The data source did not specify a security policy. We skip this source. + // Better safe than sorry. + // + case DataSourceSecurity.NOT_SPECIFIED: + this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) has no security policy. We skip this source."); + return null; + + // + // Case: Some developer forgot to implement a security policy. We skip this source. + // Better safe than sorry. + // + default: + this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) was filtered out due unknown security policy."); + return null; + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/RustService.APIKeys.cs b/app/MindWork AI Studio/Tools/Services/RustService.APIKeys.cs similarity index 99% rename from app/MindWork AI Studio/Tools/RustService.APIKeys.cs rename to app/MindWork AI Studio/Tools/Services/RustService.APIKeys.cs index 1a3b4af..7b89fe9 100644 --- a/app/MindWork AI Studio/Tools/RustService.APIKeys.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.APIKeys.cs @@ -1,6 +1,6 @@ using AIStudio.Tools.Rust; -namespace AIStudio.Tools; +namespace AIStudio.Tools.Services; public sealed partial class RustService { diff --git a/app/MindWork AI Studio/Tools/RustService.App.cs b/app/MindWork AI Studio/Tools/Services/RustService.App.cs similarity index 99% rename from app/MindWork AI Studio/Tools/RustService.App.cs rename to app/MindWork AI Studio/Tools/Services/RustService.App.cs index ea27f6d..8671e89 100644 --- a/app/MindWork AI Studio/Tools/RustService.App.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.App.cs @@ -1,6 +1,6 @@ using System.Security.Cryptography; -namespace AIStudio.Tools; +namespace AIStudio.Tools.Services; public sealed partial class RustService { diff --git a/app/MindWork AI Studio/Tools/RustService.Clipboard.cs b/app/MindWork AI Studio/Tools/Services/RustService.Clipboard.cs similarity index 98% rename from app/MindWork AI Studio/Tools/RustService.Clipboard.cs rename to app/MindWork AI Studio/Tools/Services/RustService.Clipboard.cs index d3e6520..9a1b601 100644 --- a/app/MindWork AI Studio/Tools/RustService.Clipboard.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Clipboard.cs @@ -1,6 +1,6 @@ using AIStudio.Tools.Rust; -namespace AIStudio.Tools; +namespace AIStudio.Tools.Services; public sealed partial class RustService { diff --git a/app/MindWork AI Studio/Tools/RustService.FileSystem.cs b/app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs similarity index 97% rename from app/MindWork AI Studio/Tools/RustService.FileSystem.cs rename to app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs index bd4c26a..411400f 100644 --- a/app/MindWork AI Studio/Tools/RustService.FileSystem.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs @@ -1,6 +1,6 @@ using AIStudio.Tools.Rust; -namespace AIStudio.Tools; +namespace AIStudio.Tools.Services; public sealed partial class RustService { diff --git a/app/MindWork AI Studio/Tools/RustService.Secrets.cs b/app/MindWork AI Studio/Tools/Services/RustService.Secrets.cs similarity index 99% rename from app/MindWork AI Studio/Tools/RustService.Secrets.cs rename to app/MindWork AI Studio/Tools/Services/RustService.Secrets.cs index a70a127..b41806e 100644 --- a/app/MindWork AI Studio/Tools/RustService.Secrets.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Secrets.cs @@ -1,6 +1,6 @@ using AIStudio.Tools.Rust; -namespace AIStudio.Tools; +namespace AIStudio.Tools.Services; public sealed partial class RustService { diff --git a/app/MindWork AI Studio/Tools/RustService.Updates.cs b/app/MindWork AI Studio/Tools/Services/RustService.Updates.cs similarity index 97% rename from app/MindWork AI Studio/Tools/RustService.Updates.cs rename to app/MindWork AI Studio/Tools/Services/RustService.Updates.cs index edb1466..1686b77 100644 --- a/app/MindWork AI Studio/Tools/RustService.Updates.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Updates.cs @@ -1,6 +1,6 @@ using AIStudio.Tools.Rust; -namespace AIStudio.Tools; +namespace AIStudio.Tools.Services; public sealed partial class RustService { diff --git a/app/MindWork AI Studio/Tools/RustService.cs b/app/MindWork AI Studio/Tools/Services/RustService.cs similarity index 98% rename from app/MindWork AI Studio/Tools/RustService.cs rename to app/MindWork AI Studio/Tools/Services/RustService.cs index 08d68d3..38fab8c 100644 --- a/app/MindWork AI Studio/Tools/RustService.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.cs @@ -3,7 +3,7 @@ using System.Text.Json; // ReSharper disable NotAccessedPositionalProperty.Local -namespace AIStudio.Tools; +namespace AIStudio.Tools.Services; /// /// Calling Rust functions. diff --git a/app/MindWork AI Studio/Tools/Services/UpdateService.cs b/app/MindWork AI Studio/Tools/Services/UpdateService.cs index bfabc7a..ecabbd7 100644 --- a/app/MindWork AI Studio/Tools/Services/UpdateService.cs +++ b/app/MindWork AI Studio/Tools/Services/UpdateService.cs @@ -30,9 +30,15 @@ public sealed class UpdateService : BackgroundService, IMessageBusReceiver protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + // + // Wait until the app is fully initialized. + // while (!stoppingToken.IsCancellationRequested && !IS_INITIALIZED) await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken); + // + // Set the update interval based on the user's settings. + // this.updateInterval = this.settingsManager.ConfigurationData.App.UpdateBehavior switch { UpdateBehavior.NO_CHECK => Timeout.InfiniteTimeSpan, @@ -45,16 +51,35 @@ public sealed class UpdateService : BackgroundService, IMessageBusReceiver _ => TimeSpan.FromHours(1) }; + // + // When the user doesn't want to check for updates, we can + // return early. + // if(this.settingsManager.ConfigurationData.App.UpdateBehavior is UpdateBehavior.NO_CHECK) return; + // + // Check for updates at the beginning. The user aspects this when the app + // is started. + // await this.CheckForUpdate(); + + // + // Start the update loop. This will check for updates based on the + // user's settings. + // while (!stoppingToken.IsCancellationRequested) { await Task.Delay(this.updateInterval, stoppingToken); await this.CheckForUpdate(); } } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + this.messageBus.Unregister(this); + await base.StopAsync(cancellationToken); + } #endregion @@ -79,16 +104,6 @@ public sealed class UpdateService : BackgroundService, IMessageBusReceiver #endregion - #region Overrides of BackgroundService - - public override async Task StopAsync(CancellationToken cancellationToken) - { - this.messageBus.Unregister(this); - await base.StopAsync(cancellationToken); - } - - #endregion - private async Task CheckForUpdate(bool notifyUserWhenNoUpdate = false) { if(!IS_INITIALIZED) diff --git a/app/MindWork AI Studio/Tools/TextColor.cs b/app/MindWork AI Studio/Tools/TextColor.cs new file mode 100644 index 0000000..d7531b5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/TextColor.cs @@ -0,0 +1,11 @@ +namespace AIStudio.Tools; + +public enum TextColor +{ + DEFAULT, + + WARN, + ERROR, + SUCCESS, + INFO, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/TextColorExtensions.cs b/app/MindWork AI Studio/Tools/TextColorExtensions.cs new file mode 100644 index 0000000..eb70213 --- /dev/null +++ b/app/MindWork AI Studio/Tools/TextColorExtensions.cs @@ -0,0 +1,18 @@ +using AIStudio.Settings; + +namespace AIStudio.Tools; + +public static class TextColorExtensions +{ + public static string GetHTMLColor(this TextColor color, SettingsManager settingsManager) => color switch + { + TextColor.DEFAULT => string.Empty, + + TextColor.ERROR => settingsManager.IsDarkMode ? "#ff6c6c" : "#ff0000", + TextColor.WARN => settingsManager.IsDarkMode ? "#c7a009" : "#c7c000", + TextColor.SUCCESS => settingsManager.IsDarkMode ? "#08b342" : "#009933", + TextColor.INFO => settingsManager.IsDarkMode ? "#5279b8" : "#2d67c4", + + _ => string.Empty, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs index cdd2f13..b3f8542 100644 --- a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs +++ b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs @@ -1,3 +1,4 @@ +using AIStudio.Settings.DataModel; using AIStudio.Tools.ERIClient.DataModel; namespace AIStudio.Tools.Validation; @@ -11,6 +12,8 @@ public sealed class DataSourceValidation public Func> GetUsedDataSourceNames { get; init; } = () => []; public Func GetAuthMethod { get; init; } = () => AuthMethod.NONE; + + public Func GetSecurityRequirements { get; init; } = () => null; public Func GetSelectedCloudEmbedding { get; init; } = () => false; @@ -42,6 +45,21 @@ public sealed class DataSourceValidation return null; } + public string? ValidateSecurityPolicy(DataSourceSecurity securityPolicy) + { + if(securityPolicy is DataSourceSecurity.NOT_SPECIFIED) + return "Please select your security policy."; + + var dataSourceSecurity = this.GetSecurityRequirements(); + if (dataSourceSecurity is null) + return null; + + if(dataSourceSecurity.Value.AllowedProviderType is ProviderType.SELF_HOSTED && securityPolicy is not DataSourceSecurity.SELF_HOSTED) + return "This data source can only be used with a self-hosted LLM provider. Please change the security policy."; + + return null; + } + public string? ValidateUsername(string username) { if(this.GetAuthMethod() is not AuthMethod.USERNAME_PASSWORD) diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md b/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md index ecb016a..d13c768 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md @@ -1 +1,5 @@ # v0.9.29, build 204 (2025-02-xx xx:xx UTC) +- Added the possibility to select data sources for chats. This preview feature is hidden behind the RAG feature flag, check your app options in case you want to enable it. +- Added an option to all data sources to select a local security policy. This preview feature is hidden behind the RAG feature flag. +- Added an option to preselect data sources and options for new chats. This preview feature is hidden behind the RAG feature flag. +- Improved confidence card for small spaces. \ No newline at end of file