Allow reading web content into assistants (#63)

app/MindWork AI Studio/Components/Blocks/Changelog.Logs.cs

@@ -13,6 +13,7 @@ public partial class Changelog
     
     public static readonly Log[] LOGS = 
     [
+        new (168, "v0.8.6, build 168 (2024-08-01 19:50 UTC)", "v0.8.6.md"),
         new (167, "v0.8.5, build 167 (2024-07-28 16:44 UTC)", "v0.8.5.md"),
         new (166, "v0.8.4, build 166 (2024-07-26 06:53 UTC)", "v0.8.4.md"),
         new (165, "v0.8.3, build 165 (2024-07-25 13:25 UTC)", "v0.8.3.md"),

app/MindWork AI Studio/Components/Blocks/ReadWebContent.razor

@@ -0,0 +1,31 @@
+<MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
+    <MudField Label="Read content from web?" Variant="Variant.Outlined" Class="mb-3" Disabled="@this.AgentIsRunning">
+        <MudSwitch T="bool" @bind-Value="@this.showWebContentReader" Color="Color.Primary" Disabled="@this.AgentIsRunning">
+            @(this.showWebContentReader ? "Show web content options" : "Hide web content options")
+        </MudSwitch>
+    </MudField>
+
+    @if (this.showWebContentReader)
+    {
+        <MudField Label="Cleanup content by using a LLM agent?" Variant="Variant.Outlined" Class="mb-3" Disabled="@this.AgentIsRunning">
+            <MudSwitch T="bool" @bind-Value="@this.useContentCleanerAgent" Color="Color.Primary" Validation="@this.ValidateProvider" Disabled="@this.AgentIsRunning">
+                @(this.useContentCleanerAgent ? "The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used." : "No content cleaning")
+            </MudSwitch>
+        </MudField>
+        <MudStack Row="@true" AlignItems="@AlignItems.Baseline" Class="mb-3">
+            <MudTextField T="string" Label="URL from which to load the content" @bind-Value="@this.providedURL" Validation="@this.ValidateURL" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Link" Placeholder="https://..." HelperText="Loads the content from your URL. Does not work when the content is hidden behind a paywall." Variant="Variant.Outlined" Immediate="@true" Disabled="@this.AgentIsRunning"/>
+            <MudButton Disabled="@(!this.IsReady || this.AgentIsRunning)" Variant="Variant.Filled" Size="Size.Large" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Download" OnClick="() => this.LoadFromWeb()">
+                Fetch
+            </MudButton>
+        </MudStack>
+        @if (this.AgentIsRunning)
+        {
+            <div class="pa-1">
+                <MudProgressLinear Color="Color.Primary" Indeterminate="true" Class=""/>
+            </div>
+            <div class="pa-6">
+                <MudSlider T="int" Disabled="@true" Value="@this.processStep" Min="@this.process.Min" Max="@this.process.Max" TickMarks="@true" Size="Size.Large" Variant="Variant.Filled" TickMarkLabels="@this.process.Labels" Class="mb-12"/>
+            </div>
+        }
+    }
+</MudPaper>

app/MindWork AI Studio/Components/Blocks/ReadWebContent.razor.cs

@@ -0,0 +1,198 @@
+using AIStudio.Agents;
+using AIStudio.Chat;
+using AIStudio.Settings;
+using AIStudio.Tools;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components.Blocks;
+
+public partial class ReadWebContent : ComponentBase
+{
+    [Inject]
+    private HTMLParser HTMLParser { get; init; } = null!;
+    
+    [Inject]
+    private AgentTextContentCleaner AgentTextContentCleaner { get; init; } = null!;
+    
+    [Inject]
+    protected SettingsManager SettingsManager { get; set; } = null!;
+    
+    [Inject]
+    protected IJSRuntime JsRuntime { get; init; } = null!;
+    
+    [Parameter]
+    public string Content { get; set; } = string.Empty;
+    
+    [Parameter]
+    public EventCallback<string> ContentChanged { get; set; }
+    
+    [Parameter]
+    public Settings.Provider ProviderSettings { get; set; }
+    
+    [Parameter]
+    public bool AgentIsRunning { get; set; }
+    
+    [Parameter]
+    public EventCallback<bool> AgentIsRunningChanged { get; set; }
+    
+    [Parameter]
+    public bool Preselect { get; set; }
+    
+    [Parameter]
+    public bool PreselectContentCleanerAgent { get; set; }
+
+    private Process<ReadWebContentSteps> process = Process<ReadWebContentSteps>.INSTANCE;
+    private ProcessStepValue processStep;
+    
+    private bool showWebContentReader;
+    private bool useContentCleanerAgent;
+    private string providedURL = string.Empty;
+    private bool urlIsValid;
+    private bool isProviderValid;
+
+    private Settings.Provider providerSettings;
+
+    #region Overrides of ComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        if(this.Preselect)
+            this.showWebContentReader = true;
+        
+        if(this.PreselectContentCleanerAgent)
+            this.useContentCleanerAgent = true;
+        
+        if (this.SettingsManager.ConfigurationData.PreselectAgentTextContentCleanerOptions)
+            this.providerSettings = this.SettingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == this.SettingsManager.ConfigurationData.PreselectedAgentTextContentCleanerProvider);
+        else
+            this.providerSettings = this.ProviderSettings;
+
+        await base.OnInitializedAsync();
+    }
+
+    protected override async Task OnParametersSetAsync()
+    {
+        if (!this.SettingsManager.ConfigurationData.PreselectAgentTextContentCleanerOptions)
+            this.providerSettings = this.ProviderSettings;
+        
+        await base.OnParametersSetAsync();
+    }
+
+    #endregion
+
+    private async Task LoadFromWeb()
+    {
+        if(!this.IsReady)
+            return;
+        
+        var markdown = string.Empty;
+        try
+        {
+            this.processStep = this.process[ReadWebContentSteps.LOADING];
+            this.StateHasChanged();
+            
+            var html = await this.HTMLParser.LoadWebContentHTML(new Uri(this.providedURL));
+            
+            this.processStep = this.process[ReadWebContentSteps.PARSING];
+            this.StateHasChanged();
+            markdown = this.HTMLParser.ParseToMarkdown(html);
+            
+            if (this.useContentCleanerAgent)
+            {
+                this.AgentTextContentCleaner.ProviderSettings = this.providerSettings;
+                var additionalData = new Dictionary<string, string>
+                {
+                    { "sourceURL", this.providedURL },
+                };
+            
+                this.processStep = this.process[ReadWebContentSteps.CLEANING];
+                this.AgentIsRunning = true;
+                await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning);
+                this.StateHasChanged();
+            
+                var contentBlock = await this.AgentTextContentCleaner.ProcessInput(new ContentBlock
+                {
+                    Time = DateTimeOffset.UtcNow,
+                    ContentType = ContentType.TEXT,
+                    Role = ChatRole.USER,
+                    Content = new ContentText
+                    {
+                        Text = markdown,
+                    },
+                }, additionalData);
+            
+                markdown = contentBlock.Content is ContentText text ? text.Text : markdown;
+            
+                this.processStep = this.process[ReadWebContentSteps.DONE];
+                this.AgentIsRunning = false;
+                await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning);
+                this.StateHasChanged();
+            }
+        }
+        catch
+        {
+            if (this.AgentIsRunning)
+            {
+                this.processStep = this.process[ReadWebContentSteps.START];
+                this.AgentIsRunning = false;
+                await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning);
+                this.StateHasChanged();
+            }
+        }
+
+        this.Content = markdown;
+        await this.ContentChanged.InvokeAsync(this.Content);
+    }
+
+    private bool IsReady
+    {
+        get
+        {
+            if(!this.urlIsValid)
+                return false;
+            
+            if(this.useContentCleanerAgent && !this.isProviderValid)
+                return false;
+            
+            return true;
+        }
+    }
+
+    private string? ValidateProvider(bool shouldUseAgent)
+    {
+        if(shouldUseAgent && this.providerSettings == default)
+        {
+            this.isProviderValid = false;
+            return "Please select a provider to use the cleanup agent.";
+        }
+
+        this.isProviderValid = true;
+        return null;
+    }
+    
+    private string? ValidateURL(string url)
+    {
+        if(string.IsNullOrWhiteSpace(url))
+        {
+            this.urlIsValid = false;
+            return "Please provide a URL to load the content from.";
+        }
+
+        var urlParsingResult = Uri.TryCreate(url, UriKind.Absolute, out var uriResult);
+        if(!urlParsingResult)
+        {
+            this.urlIsValid = false;
+            return "Please provide a valid URL.";
+        }
+
+        if(uriResult is not { Scheme: "http" or "https" })
+        {
+            this.urlIsValid = false;
+            return "Please provide a valid HTTP or HTTPS URL.";
+        }
+
+        this.urlIsValid = true;
+        return null;
+    }
+}

app/MindWork AI Studio/Components/Blocks/ReadWebContentSteps.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Components.Blocks;
+
+public enum ReadWebContentSteps
+{
+    START,
+    LOADING,
+    PARSING,
+    CLEANING,
+    DONE,
+}

app/MindWork AI Studio/Components/Blocks/Workspaces.razor.cs

@@ -16,13 +16,13 @@ namespace AIStudio.Components.Blocks;
 public partial class Workspaces : ComponentBase
 {
     [Inject]
-    private SettingsManager SettingsManager { get; set; } = null!;
+    private SettingsManager SettingsManager { get; init; } = null!;
     
     [Inject]
-    private IDialogService DialogService { get; set; } = null!;
+    private IDialogService DialogService { get; init; } = null!;
     
     [Inject]
-    public Random RNG { get; set; } = null!;
+    private ThreadSafeRandom RNG { get; init; } = null!;
     
     [Parameter]
     public ChatThread? CurrentChatThread { get; set; }