using AIStudio.Agents; using AIStudio.Chat; using AIStudio.Tools.Security; using AIStudio.Tools.Web; using Microsoft.AspNetCore.Components; namespace AIStudio.Components; public partial class ReadWebContent : MSGComponentBase { /// /// How long loading one page may take. /// /// /// The user is watching a progress indicator while this runs, so it is shorter than what the /// tools allow themselves for a page fetched in the background. /// private const int TIMEOUT_SECONDS = 60; [Inject] private WebPageRetrievalService WebPageRetrievalService { get; init; } = null!; [Inject] private ILogger Logger { get; init; } = null!; [Inject] private AgentTextContentCleaner AgentTextContentCleaner { get; init; } = null!; [Inject] private PromptInjectionGuardService PromptInjectionGuardService { get; init; } = null!; [Parameter] public string Content { get; set; } = string.Empty; [Parameter] public EventCallback ContentChanged { get; set; } /// /// The URL the content is loaded from. /// /// /// The URL belongs to the parent, so that it is cleared when the parent resets its form and /// is kept when the parent stores its state. /// [Parameter] public string URL { get; set; } = string.Empty; [Parameter] public EventCallback URLChanged { get; set; } [Parameter] public AIStudio.Settings.Provider ProviderSettings { get; set; } = AIStudio.Settings.Provider.NONE; [Parameter] public bool AgentIsRunning { get; set; } [Parameter] public EventCallback AgentIsRunningChanged { get; set; } [Parameter] public bool Preselect { get; set; } [Parameter] public EventCallback PreselectChanged { get; set; } [Parameter] public bool PreselectContentCleanerAgent { get; set; } [Parameter] public EventCallback PreselectContentCleanerAgentChanged { get; set; } private readonly Process process = Process.INSTANCE; private ProcessStepValue processStep; /// /// The model the content cleaner runs with. /// /// /// This is a resolved value, not a chosen one: the reader has no model selection of its own, /// it takes what the assistant around it uses, unless a dedicated one for the cleaner or an /// app-wide default takes precedence. Because the assistant's model can change at any moment, /// this is resolved again on every render instead of being remembered from the first one. /// private AIStudio.Settings.Provider providerSettings = AIStudio.Settings.Provider.NONE; #region Overrides of ComponentBase protected override async Task OnInitializedAsync() { this.ApplyFilters([], [ Event.CONFIGURATION_CHANGED ]); this.ResolveProvider(); await base.OnInitializedAsync(); } protected override async Task OnParametersSetAsync() { this.ResolveProvider(); await base.OnParametersSetAsync(); } #endregion #region Overrides of MSGComponentBase protected override Task ProcessIncomingMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default { if (triggeredEvent is Event.CONFIGURATION_CHANGED) { // // A dedicated model for the cleaner, or the app-wide default, may be set while this // assistant is open. Nothing about that reaches us as a parameter, so without this the // user would have to leave the assistant and come back for it to take effect. // this.ResolveProvider(); this.StateHasChanged(); } return Task.CompletedTask; } #endregion /// /// Determines the model the content cleaner runs with. /// /// /// Called from both lifecycle methods, and with the same arguments: the assistant's model is a /// parameter, and a parameter arrives whenever the parent renders. Resolving only once would /// leave the cleaner with whatever was set the first time this component was built. /// private void ResolveProvider() => this.providerSettings = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_TEXT_CONTENT_CLEANER, this.ProviderSettings.Id, true); private async Task LoadFromWeb() { if(!this.IsReady) return; var markdown = string.Empty; try { this.processStep = this.process[ReadWebContentSteps.LOADING]; this.StateHasChanged(); // // The same retrieval the read web page tool uses, so a page is fetched and read one // way throughout AI Studio. The difference is the target policy: here the user typed // the URL, so their own network is not off limits. // var retrievedPage = await this.WebPageRetrievalService.RetrieveAsync( new Uri(this.URL), new WebPageRetrievalOptions { TimeoutSeconds = TIMEOUT_SECONDS, TargetChosenByUser = true, }); this.processStep = this.process[ReadWebContentSteps.PARSING]; this.StateHasChanged(); markdown = retrievedPage.ExtractedPage.Markdown; markdown = await this.PromptInjectionGuardService.SanitizeAsync(markdown, PromptInjectionSource.WebContent(this.URL)); if (this.PreselectContentCleanerAgent && this.providerSettings == AIStudio.Settings.Provider.NONE) { // // Say that the cleaning did not happen. The user asked for it, the page arrives, // and without a word they would take the raw markdown -- navigation, cookie banner // and advertising included -- for the cleaned result. // await this.MessageBus.SendError(new(Icons.Material.Filled.SettingsSuggest, T("The content was loaded, but not cleaned: no model is available for the content cleaner."))); } if (this.PreselectContentCleanerAgent && this.providerSettings != AIStudio.Settings.Provider.NONE) { this.AgentTextContentCleaner.ProviderSettings = this.providerSettings; var additionalData = new Dictionary { { "sourceURL", this.URL }, }; this.processStep = this.process[ReadWebContentSteps.CLEANING]; this.AgentIsRunning = true; await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning); this.StateHasChanged(); var contentBlock = await this.AgentTextContentCleaner.ProcessInput(new ContentBlock { Time = DateTimeOffset.UtcNow, ContentType = ContentType.TEXT, Role = ChatRole.USER, Content = new ContentText { Text = markdown, }, }, additionalData); markdown = contentBlock.Content is ContentText text ? text.Text : markdown; this.processStep = this.process[ReadWebContentSteps.DONE]; this.AgentIsRunning = false; await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning); this.StateHasChanged(); } } catch (Exception exception) { if (this.AgentIsRunning) { this.processStep = this.process[ReadWebContentSteps.START]; this.AgentIsRunning = false; await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning); this.StateHasChanged(); } // // Say why nothing was loaded. An empty text field looks like a page without content, // and the reasons a page cannot be read are things the user can act on: a link to a // PDF rather than a page, a host that does not answer, a server refusing the request. // this.Logger.LogWarning(exception, "Could not load the web content from '{ProvidedUrl}'.", this.URL); await this.MessageBus.SendError(new(Icons.Material.Filled.CloudOff, string.Format(this.T("The content of '{0}' could not be loaded: {1}"), this.URL, exception.Message))); } this.Content = markdown; await this.ContentChanged.InvokeAsync(this.Content); } /// /// Whether the content can be fetched. /// /// /// A missing model for the content cleaner is deliberately not part of this. Cleaning is an /// option of the fetch, not a condition for it: making it one would leave the user with a /// switch they turned on, no way to get their page, and a dead button to explain it. The page /// is fetched, and LoadFromWeb says that it arrived uncleaned. /// private bool IsReady => this.UrlIsValid; /// /// Whether the current URL can be loaded. /// /// /// Asked of the current value instead of remembered from the last validation run: the parent /// clears the URL when it resets its form, and the form validation does not run again at that /// point. The fetch button would otherwise stay enabled with an empty field. /// private bool UrlIsValid => this.ValidateURL(this.URL) is null; private async Task URLValueChanged(string url) { await this.URLChanged.InvokeAsync(url); } private async Task ShowWebContentReaderChanged(bool state) { await this.PreselectChanged.InvokeAsync(state); } private async Task UseContentCleanerAgentChanged(bool state) { await this.PreselectContentCleanerAgentChanged.InvokeAsync(state); } /// /// Says why the content cleaner has no model, or nothing when it has one. /// /// /// This is a hint, not a validation: the cleaner is an option of the reader, and an option /// nobody can use yet must not keep the assistant around it from running. It is also stated /// rather than remembered, so that choosing a model below makes it disappear at once. /// The two causes lead to different places, which is why they are told apart: either no model /// was chosen at all, or the chosen one is not trusted enough for this agent. /// private string? ContentCleanerHint { get { if(!this.PreselectContentCleanerAgent || this.providerSettings != AIStudio.Settings.Provider.NONE) return null; if(this.ProviderSettings == AIStudio.Settings.Provider.NONE) return T("The content cleaner uses the model of this assistant. Please select one below."); return T("The selected model does not meet the confidence requirements of the content cleaner. Please select another model, or configure an eligible one in the app settings."); } } private string? ValidateURL(string url) { if(string.IsNullOrWhiteSpace(url)) return T("Please provide a URL to load the content from."); var urlParsingResult = Uri.TryCreate(url, UriKind.Absolute, out var uriResult); if(!urlParsingResult) return T("Please provide a valid URL."); if(uriResult is not { Scheme: "http" or "https" }) return T("Please provide a valid HTTP or HTTPS URL."); return null; } }