Allow reading web content into assistants (#63)

app/MindWork AI Studio/Agents/AgentBase.cs

@@ -0,0 +1,109 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools;
+
+// ReSharper disable MemberCanBePrivate.Global
+
+namespace AIStudio.Agents;
+
+public abstract class AgentBase(SettingsManager settingsManager, IJSRuntime jsRuntime, ThreadSafeRandom rng) : IAgent
+{
+    protected SettingsManager SettingsManager { get; init; } = settingsManager;
+
+    protected IJSRuntime JsRuntime { get; init; } = jsRuntime;
+
+    protected ThreadSafeRandom RNG { get; init; } = rng;
+
+    /// <summary>
+    /// Represents the type or category of this agent.
+    /// </summary>
+    protected abstract Type Type { get; }
+    
+    /// <summary>
+    /// The name of the agent.
+    /// </summary>
+    public abstract string Id { get; }
+
+    /// <summary>
+    /// The agent's job description. Will be used for the system prompt.
+    /// </summary>
+    protected abstract string JobDescription { get; }
+
+    /// <summary>
+    /// Represents the system prompt provided for the agent.
+    /// </summary>
+    protected abstract string SystemPrompt(string additionalData);
+    
+    #region Implementation of IAgent
+    
+    public abstract AIStudio.Settings.Provider? ProviderSettings { get; set; }
+    
+    public abstract Task<ChatThread> ProcessContext(ChatThread chatThread, IDictionary<string, string> additionalData);
+    
+    public abstract Task<ContentBlock> ProcessInput(ContentBlock input, IDictionary<string, string> additionalData);
+    
+    public abstract Task<bool> MadeDecision(ContentBlock input);
+    
+    public abstract IReadOnlyCollection<ContentBlock> GetContext();
+    
+    public abstract IReadOnlyCollection<ContentBlock> GetAnswers();
+    
+    #endregion
+    
+    protected ChatThread CreateChatThread(string systemPrompt) => new()
+    {
+        WorkspaceId = Guid.Empty,
+        ChatId = Guid.NewGuid(),
+        Name = string.Empty,
+        Seed = this.RNG.Next(),
+        SystemPrompt = systemPrompt,
+        Blocks = [],
+    };
+
+    protected DateTimeOffset AddUserRequest(ChatThread thread, string request)
+    {
+        var time = DateTimeOffset.Now;
+        thread.Blocks.Add(new ContentBlock
+        {
+            Time = time,
+            ContentType = ContentType.TEXT,
+            Role = ChatRole.USER,
+            Content = new ContentText
+            {
+                Text = request,
+            },
+        });
+        
+        return time;
+    }
+    
+    protected async Task AddAIResponseAsync(ChatThread thread, DateTimeOffset time)
+    {
+        if(this.ProviderSettings is null)
+            return;
+        
+        var providerSettings = this.ProviderSettings.Value;
+        var aiText = new ContentText
+        {
+            // We have to wait for the remote
+            // for the content stream: 
+            InitialRemoteWait = true,
+        };
+
+        var resultingContentBlock = new ContentBlock
+        {
+            Time = time,
+            ContentType = ContentType.TEXT,
+            Role = ChatRole.AI,
+            Content = aiText,
+        };
+        
+        thread.Blocks.Add(resultingContentBlock);
+        
+        // Use the selected provider to get the AI response.
+        // By awaiting this line, we wait for the entire
+        // content to be streamed.
+        await aiText.CreateFromProviderAsync(providerSettings.CreateProvider(), this.JsRuntime, this.SettingsManager, providerSettings.Model, thread);
+    }
+}

app/MindWork AI Studio/Agents/AgentTextContentCleaner.cs

@@ -0,0 +1,94 @@
+using AIStudio.Chat;
+using AIStudio.Settings;
+using AIStudio.Tools;
+
+namespace AIStudio.Agents;
+
+public sealed class AgentTextContentCleaner(SettingsManager settingsManager, IJSRuntime jsRuntime, ThreadSafeRandom rng) : AgentBase(settingsManager, jsRuntime, rng)
+{
+    private static readonly ContentBlock EMPTY_BLOCK = new()
+    {
+        Content = null,
+        ContentType = ContentType.NONE,
+        Role = ChatRole.AGENT,
+        Time = DateTimeOffset.UtcNow,
+    };
+    
+    private readonly List<ContentBlock> context = new();
+    private readonly List<ContentBlock> answers = new();
+    
+    #region Overrides of AgentBase
+
+    public override Settings.Provider? ProviderSettings { get; set; }
+
+    protected override Type Type => Type.SYSTEM;
+
+    public override string Id => "Text Content Cleaner";
+
+    protected override string JobDescription => 
+        """
+        You receive a Markdown document as input. Your goal is to identify the main content of the document
+        and return it including Markdown formatting. Remove areas that do not belong to the main part of the
+        document. For a blog article, return only the text of the article with its formatting. For a scientific
+        paper, only the contents of the paper. Delete elements of navigation, advertisements, HTML artifacts,
+        cookie banners, etc. If the content contains images, these images remain. The same applies to links.
+        Ensure that links and images are present as valid Markdown:
+        
+        - Syntax of links: [link text](URL)
+        - Syntax of images: ![alt text](URL)
+        
+        If you find relative links or images with relative paths, correct them to absolute paths. For this
+        purpose, here is the source URL:
+        """;
+
+    protected override string SystemPrompt(string additionalData) => $"{this.JobDescription} `{additionalData}`.";
+    
+    /// <inheritdoc />
+    public override async Task<ChatThread> ProcessContext(ChatThread chatThread, IDictionary<string, string> additionalData)
+    {
+        // We process the last block of the chat thread. Then, we add the result
+        // to the chat thread as the last block:
+        var answer = await this.ProcessInput(chatThread.Blocks[^1], additionalData);
+        chatThread.Blocks.Add(answer);
+        
+        this.context.Clear();
+        this.context.AddRange(chatThread.Blocks);
+        
+        return chatThread;
+    }
+
+    // <inheritdoc />
+    public override async Task<ContentBlock> ProcessInput(ContentBlock input, IDictionary<string, string> additionalData)
+    {
+        if (input.Content is not ContentText text)
+            return EMPTY_BLOCK;
+        
+        if(text.InitialRemoteWait || text.IsStreaming)
+            return EMPTY_BLOCK;
+        
+        if(string.IsNullOrWhiteSpace(text.Text))
+            return EMPTY_BLOCK;
+        
+        if(!additionalData.TryGetValue("sourceURL", out var sourceURL) || string.IsNullOrWhiteSpace(sourceURL))
+            return EMPTY_BLOCK;
+        
+        var thread = this.CreateChatThread(this.SystemPrompt(sourceURL));
+        var time = this.AddUserRequest(thread, text.Text);
+        await this.AddAIResponseAsync(thread, time);
+        
+        var answer = thread.Blocks[^1];
+        this.answers.Add(answer);
+        return answer;
+    }
+
+    // <inheritdoc />
+    public override Task<bool> MadeDecision(ContentBlock input) => Task.FromResult(true);
+
+    // <inheritdoc />
+    public override IReadOnlyCollection<ContentBlock> GetContext() => this.context;
+
+    // <inheritdoc />
+    public override IReadOnlyCollection<ContentBlock> GetAnswers() => this.answers;
+
+    #endregion
+}

app/MindWork AI Studio/Agents/IAgent.cs

@@ -0,0 +1,55 @@
+using AIStudio.Chat;
+
+namespace AIStudio.Agents;
+
+public interface IAgent
+{
+    /// <summary>
+    /// Gets the name of the agent.
+    /// </summary>
+    public string Id { get; }
+
+    /// <summary>
+    /// The provider to use for this agent.
+    /// </summary>
+    public AIStudio.Settings.Provider? ProviderSettings { get; set; }
+
+    /// <summary>
+    /// Processes a chat thread (i.e., context) and returns the updated thread.
+    /// </summary>
+    /// <param name="chatThread">The chat thread to process. The thread is the context for the agent.</param>
+    /// <param name="additionalData">Additional data to use for processing the chat thread.</param>
+    /// <returns>The updated chat thread. The last content block of the thread is the agent's response.</returns>
+    public Task<ChatThread> ProcessContext(ChatThread chatThread, IDictionary<string, string> additionalData);
+
+    /// <summary>
+    /// Processes the input content block and returns the agent's response.
+    /// </summary>
+    /// <param name="input">The content block to process. It represents the input.</param>
+    /// <param name="additionalData">Additional data to use for processing the input.</param>
+    /// <returns>The content block representing the agent's response.</returns>
+    public Task<ContentBlock> ProcessInput(ContentBlock input, IDictionary<string, string> additionalData);
+
+    /// <summary>
+    /// The agent makes a decision based on the input.
+    /// </summary>
+    /// <param name="input">The content block to process. Should be a question or a request.</param>
+    /// <returns>
+    /// True if a decision has been made based on the input, false otherwise.
+    /// </returns>
+    public Task<bool> MadeDecision(ContentBlock input);
+
+    /// <summary>
+    /// Retrieves the context of the agent.
+    /// </summary>
+    /// <returns>The collection of content blocks representing the agent's context. This includes the user's and the other agent's messages.</returns>
+    public IReadOnlyCollection<ContentBlock> GetContext();
+
+    /// <summary>
+    /// Retrieves the answers from the agent's context.
+    /// </summary>
+    /// <returns>
+    /// The collection of content blocks representing the answers provided by this agent.
+    /// </returns>
+    public IReadOnlyCollection<ContentBlock> GetAnswers();
+}

app/MindWork AI Studio/Agents/Type.cs

@@ -0,0 +1,27 @@
+namespace AIStudio.Agents;
+
+public enum Type
+{
+    /// <summary>
+    /// Represents an unspecified agent type.
+    /// </summary>
+    UNSPECIFIED = 0,
+    
+    /// <summary>
+    /// Represents a conversational agent who produces human-like responses and feedback (depending on the context and its job description).
+    /// For example, an expert agent for a specific domain. Answers might be detailed and informative.
+    /// </summary>
+    CONVERSATIONAL,
+
+    /// <summary>
+    /// Represents a worker agent type who performs tasks and provides information or services (depending on the context and its job description).
+    /// For example, a quality assurance agent who assesses the quality of a product or service. Answers might be short and concise.
+    /// </summary>
+    WORKER,
+
+    /// <summary>
+    /// Represents the system agent type who processes the input and provides a specific response (depending on the context and its job description).
+    /// For example, a HTML content agent who processes the arbitrary HTML content and provides a structured Markdown response. Answers might be structured and formatted.
+    /// </summary>
+    SYSTEM,
+}

app/MindWork AI Studio/Chat/ChatRole.cs

@@ -11,6 +11,7 @@ public enum ChatRole
     SYSTEM,
     USER,
     AI,
+    AGENT,
 }
 
 /// <summary>

app/MindWork AI Studio/Components/AssistantBase.razor.cs

@@ -1,6 +1,7 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools;
 
 using Microsoft.AspNetCore.Components;
 
@@ -15,7 +16,7 @@ public abstract partial class AssistantBase : ComponentBase
     protected IJSRuntime JsRuntime { get; init; } = null!;
 
     [Inject]
-    protected Random RNG { get; set; } = null!;
+    protected ThreadSafeRandom RNG { get; init; } = null!;
     
     protected abstract string Title { get; }
     

app/MindWork AI Studio/Components/Blocks/Changelog.Logs.cs

@@ -13,6 +13,7 @@ public partial class Changelog
     
     public static readonly Log[] LOGS = 
     [
+        new (168, "v0.8.6, build 168 (2024-08-01 19:50 UTC)", "v0.8.6.md"),
         new (167, "v0.8.5, build 167 (2024-07-28 16:44 UTC)", "v0.8.5.md"),
         new (166, "v0.8.4, build 166 (2024-07-26 06:53 UTC)", "v0.8.4.md"),
         new (165, "v0.8.3, build 165 (2024-07-25 13:25 UTC)", "v0.8.3.md"),

app/MindWork AI Studio/Components/Blocks/ReadWebContent.razor

@@ -0,0 +1,31 @@
+<MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
+    <MudField Label="Read content from web?" Variant="Variant.Outlined" Class="mb-3" Disabled="@this.AgentIsRunning">
+        <MudSwitch T="bool" @bind-Value="@this.showWebContentReader" Color="Color.Primary" Disabled="@this.AgentIsRunning">
+            @(this.showWebContentReader ? "Show web content options" : "Hide web content options")
+        </MudSwitch>
+    </MudField>
+
+    @if (this.showWebContentReader)
+    {
+        <MudField Label="Cleanup content by using a LLM agent?" Variant="Variant.Outlined" Class="mb-3" Disabled="@this.AgentIsRunning">
+            <MudSwitch T="bool" @bind-Value="@this.useContentCleanerAgent" Color="Color.Primary" Validation="@this.ValidateProvider" Disabled="@this.AgentIsRunning">
+                @(this.useContentCleanerAgent ? "The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used." : "No content cleaning")
+            </MudSwitch>
+        </MudField>
+        <MudStack Row="@true" AlignItems="@AlignItems.Baseline" Class="mb-3">
+            <MudTextField T="string" Label="URL from which to load the content" @bind-Value="@this.providedURL" Validation="@this.ValidateURL" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Link" Placeholder="https://..." HelperText="Loads the content from your URL. Does not work when the content is hidden behind a paywall." Variant="Variant.Outlined" Immediate="@true" Disabled="@this.AgentIsRunning"/>
+            <MudButton Disabled="@(!this.IsReady || this.AgentIsRunning)" Variant="Variant.Filled" Size="Size.Large" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Download" OnClick="() => this.LoadFromWeb()">
+                Fetch
+            </MudButton>
+        </MudStack>
+        @if (this.AgentIsRunning)
+        {
+            <div class="pa-1">
+                <MudProgressLinear Color="Color.Primary" Indeterminate="true" Class=""/>
+            </div>
+            <div class="pa-6">
+                <MudSlider T="int" Disabled="@true" Value="@this.processStep" Min="@this.process.Min" Max="@this.process.Max" TickMarks="@true" Size="Size.Large" Variant="Variant.Filled" TickMarkLabels="@this.process.Labels" Class="mb-12"/>
+            </div>
+        }
+    }
+</MudPaper>

app/MindWork AI Studio/Components/Blocks/ReadWebContent.razor.cs

@@ -0,0 +1,198 @@
+using AIStudio.Agents;
+using AIStudio.Chat;
+using AIStudio.Settings;
+using AIStudio.Tools;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components.Blocks;
+
+public partial class ReadWebContent : ComponentBase
+{
+    [Inject]
+    private HTMLParser HTMLParser { get; init; } = null!;
+    
+    [Inject]
+    private AgentTextContentCleaner AgentTextContentCleaner { get; init; } = null!;
+    
+    [Inject]
+    protected SettingsManager SettingsManager { get; set; } = null!;
+    
+    [Inject]
+    protected IJSRuntime JsRuntime { get; init; } = null!;
+    
+    [Parameter]
+    public string Content { get; set; } = string.Empty;
+    
+    [Parameter]
+    public EventCallback<string> ContentChanged { get; set; }
+    
+    [Parameter]
+    public Settings.Provider ProviderSettings { get; set; }
+    
+    [Parameter]
+    public bool AgentIsRunning { get; set; }
+    
+    [Parameter]
+    public EventCallback<bool> AgentIsRunningChanged { get; set; }
+    
+    [Parameter]
+    public bool Preselect { get; set; }
+    
+    [Parameter]
+    public bool PreselectContentCleanerAgent { get; set; }
+
+    private Process<ReadWebContentSteps> process = Process<ReadWebContentSteps>.INSTANCE;
+    private ProcessStepValue processStep;
+    
+    private bool showWebContentReader;
+    private bool useContentCleanerAgent;
+    private string providedURL = string.Empty;
+    private bool urlIsValid;
+    private bool isProviderValid;
+
+    private Settings.Provider providerSettings;
+
+    #region Overrides of ComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        if(this.Preselect)
+            this.showWebContentReader = true;
+        
+        if(this.PreselectContentCleanerAgent)
+            this.useContentCleanerAgent = true;
+        
+        if (this.SettingsManager.ConfigurationData.PreselectAgentTextContentCleanerOptions)
+            this.providerSettings = this.SettingsManager.ConfigurationData.Providers.FirstOrDefault(x => x.Id == this.SettingsManager.ConfigurationData.PreselectedAgentTextContentCleanerProvider);
+        else
+            this.providerSettings = this.ProviderSettings;
+
+        await base.OnInitializedAsync();
+    }
+
+    protected override async Task OnParametersSetAsync()
+    {
+        if (!this.SettingsManager.ConfigurationData.PreselectAgentTextContentCleanerOptions)
+            this.providerSettings = this.ProviderSettings;
+        
+        await base.OnParametersSetAsync();
+    }
+
+    #endregion
+
+    private async Task LoadFromWeb()
+    {
+        if(!this.IsReady)
+            return;
+        
+        var markdown = string.Empty;
+        try
+        {
+            this.processStep = this.process[ReadWebContentSteps.LOADING];
+            this.StateHasChanged();
+            
+            var html = await this.HTMLParser.LoadWebContentHTML(new Uri(this.providedURL));
+            
+            this.processStep = this.process[ReadWebContentSteps.PARSING];
+            this.StateHasChanged();
+            markdown = this.HTMLParser.ParseToMarkdown(html);
+            
+            if (this.useContentCleanerAgent)
+            {
+                this.AgentTextContentCleaner.ProviderSettings = this.providerSettings;
+                var additionalData = new Dictionary<string, string>
+                {
+                    { "sourceURL", this.providedURL },
+                };
+            
+                this.processStep = this.process[ReadWebContentSteps.CLEANING];
+                this.AgentIsRunning = true;
+                await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning);
+                this.StateHasChanged();
+            
+                var contentBlock = await this.AgentTextContentCleaner.ProcessInput(new ContentBlock
+                {
+                    Time = DateTimeOffset.UtcNow,
+                    ContentType = ContentType.TEXT,
+                    Role = ChatRole.USER,
+                    Content = new ContentText
+                    {
+                        Text = markdown,
+                    },
+                }, additionalData);
+            
+                markdown = contentBlock.Content is ContentText text ? text.Text : markdown;
+            
+                this.processStep = this.process[ReadWebContentSteps.DONE];
+                this.AgentIsRunning = false;
+                await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning);
+                this.StateHasChanged();
+            }
+        }
+        catch
+        {
+            if (this.AgentIsRunning)
+            {
+                this.processStep = this.process[ReadWebContentSteps.START];
+                this.AgentIsRunning = false;
+                await this.AgentIsRunningChanged.InvokeAsync(this.AgentIsRunning);
+                this.StateHasChanged();
+            }
+        }
+
+        this.Content = markdown;
+        await this.ContentChanged.InvokeAsync(this.Content);
+    }
+
+    private bool IsReady
+    {
+        get
+        {
+            if(!this.urlIsValid)
+                return false;
+            
+            if(this.useContentCleanerAgent && !this.isProviderValid)
+                return false;
+            
+            return true;
+        }
+    }
+
+    private string? ValidateProvider(bool shouldUseAgent)
+    {
+        if(shouldUseAgent && this.providerSettings == default)
+        {
+            this.isProviderValid = false;
+            return "Please select a provider to use the cleanup agent.";
+        }
+
+        this.isProviderValid = true;
+        return null;
+    }
+    
+    private string? ValidateURL(string url)
+    {
+        if(string.IsNullOrWhiteSpace(url))
+        {
+            this.urlIsValid = false;
+            return "Please provide a URL to load the content from.";
+        }
+
+        var urlParsingResult = Uri.TryCreate(url, UriKind.Absolute, out var uriResult);
+        if(!urlParsingResult)
+        {
+            this.urlIsValid = false;
+            return "Please provide a valid URL.";
+        }
+
+        if(uriResult is not { Scheme: "http" or "https" })
+        {
+            this.urlIsValid = false;
+            return "Please provide a valid HTTP or HTTPS URL.";
+        }
+
+        this.urlIsValid = true;
+        return null;
+    }
+}

app/MindWork AI Studio/Components/Blocks/ReadWebContentSteps.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Components.Blocks;
+
+public enum ReadWebContentSteps
+{
+    START,
+    LOADING,
+    PARSING,
+    CLEANING,
+    DONE,
+}

app/MindWork AI Studio/Components/Blocks/Workspaces.razor.cs

@@ -16,13 +16,13 @@ namespace AIStudio.Components.Blocks;
 public partial class Workspaces : ComponentBase
 {
     [Inject]
-    private SettingsManager SettingsManager { get; set; } = null!;
+    private SettingsManager SettingsManager { get; init; } = null!;
     
     [Inject]
-    private IDialogService DialogService { get; set; } = null!;
+    private IDialogService DialogService { get; init; } = null!;
     
     [Inject]
-    public Random RNG { get; set; } = null!;
+    private ThreadSafeRandom RNG { get; init; } = null!;
     
     [Parameter]
     public ChatThread? CurrentChatThread { get; set; }

app/MindWork AI Studio/Components/Pages/About.razor

@@ -47,6 +47,8 @@
                 <ThirdPartyComponent Name="arboard" Developer="Artur Kovacs, Avi Weinstock, 1Password & Open Source Community" LicenseName="MIT & Apache-2.0" LicenseUrl="https://github.com/1Password/arboard" RepositoryUrl="https://github.com/1Password/arboard" UseCase="To be able to use the responses of the LLM in other apps, we often use the clipboard of the respective operating system. Unfortunately, in .NET there is no solution that works with all operating systems. Therefore, I have opted for this library in Rust. This way, data transfer to other apps works on every system."/>
                 <ThirdPartyComponent Name="tokio" Developer="Alex Crichton & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/tokio-rs/tokio/blob/master/LICENSE" RepositoryUrl="https://github.com/tokio-rs/tokio" UseCase="Code in the Rust language can be specified as synchronous or asynchronous. Unlike .NET and the C# language, Rust cannot execute asynchronous code by itself. Rust requires support in the form of an executor for this. Tokio is one such executor."/>
                 <ThirdPartyComponent Name="flexi_logger" Developer="emabee & Open Source Community" LicenseName="MIT & Apache-2.0" LicenseUrl="https://github.com/emabee/flexi_logger" RepositoryUrl="https://github.com/emabee/flexi_logger" UseCase="This Rust library is used to output the app's messages to the terminal. This is helpful during development and troubleshooting. This feature is initially invisible; when the app is started via the terminal, the messages become visible."/>
+                <ThirdPartyComponent Name="HtmlAgilityPack" Developer="ZZZ Projects & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE" RepositoryUrl="https://github.com/zzzprojects/html-agility-pack" UseCase="We use the HtmlAgilityPack to extract content from the web. This is necessary, e.g., when you provide a URL as input for an assistant."/>
+                <ThirdPartyComponent Name="ReverseMarkdown" Developer="Babu Annamalai & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mysticmind/reversemarkdown-net/blob/master/LICENSE" RepositoryUrl="https://github.com/mysticmind/reversemarkdown-net" UseCase="This library is used to convert HTML to Markdown. This is necessary, e.g., when you provide a URL as input for an assistant."/>
             </MudGrid>
         </ExpansionPanel>
         <ExpansionPanel HeaderIcon="@Icons.Material.Filled.Verified" HeaderText="License: FSL-1.1-MIT">

app/MindWork AI Studio/Components/Pages/Chat.razor.cs

@@ -25,7 +25,7 @@ public partial class Chat : MSGComponentBase, IAsyncDisposable
     public IJSRuntime JsRuntime { get; init; } = null!;
 
     [Inject]
-    public Random RNG { get; set; } = null!;
+    private ThreadSafeRandom RNG { get; init; } = null!;
     
     [Inject]
     public IDialogService DialogService { get; set; } = null!;

app/MindWork AI Studio/Components/Pages/Settings.razor

@@ -82,7 +82,7 @@
         <ConfigurationSelect OptionDescription="Workspace behavior" SelectedValue="@(() => this.SettingsManager.ConfigurationData.WorkspaceStorageBehavior)" Data="@ConfigurationSelectDataFactory.GetWorkspaceStorageBehaviorData()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.WorkspaceStorageBehavior = selectedValue)" OptionHelp="Should we store your chats?"/>
         <ConfigurationSelect OptionDescription="Workspace maintenance" SelectedValue="@(() => this.SettingsManager.ConfigurationData.WorkspaceStorageTemporaryMaintenancePolicy)" Data="@ConfigurationSelectDataFactory.GetWorkspaceStorageTemporaryMaintenancePolicyData()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.WorkspaceStorageTemporaryMaintenancePolicy = selectedValue)" OptionHelp="If and when should we delete your temporary chats?"/>
 
-        <MudText Typo="Typo.h4" Class="mb-3">Assistants Options</MudText>
+        <MudText Typo="Typo.h4" Class="mb-3">Assistant Options</MudText>
         
         <MudText Typo="Typo.h5" Class="mb-3">Icon Finder Options</MudText>
         <MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
@@ -93,8 +93,11 @@
 
         <MudText Typo="Typo.h5" Class="mb-3">Translator Options</MudText>
         <ConfigurationSlider T="int" OptionDescription="How fast should the live translation react?" Min="500" Max="3_000" Step="100" Unit="milliseconds" Value="@(() => this.SettingsManager.ConfigurationData.LiveTranslationDebounceIntervalMilliseconds)" ValueUpdate="@(updatedValue => this.SettingsManager.ConfigurationData.LiveTranslationDebounceIntervalMilliseconds = updatedValue)"/>
+        <ConfigurationOption OptionDescription="Hide the web content reader?" LabelOn="Web content reader is hidden" LabelOff="Web content reader is shown" State="@(() => this.SettingsManager.ConfigurationData.HideWebContentReaderForTranslation)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.HideWebContentReaderForTranslation = updatedState)" OptionHelp="When activated, the web content reader is hidden and cannot be used. As a result, the user interface becomes a bit easier to use."/>
         <MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
             <ConfigurationOption OptionDescription="Preselect translator options?" LabelOn="Translator options are preselected" LabelOff="No translator options are preselected" State="@(() => this.SettingsManager.ConfigurationData.PreselectTranslationOptions)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.PreselectTranslationOptions = updatedState)" OptionHelp="When enabled, you can preselect the translator options. This is might be useful when you prefer a specific target language or LLM model."/>
+            <ConfigurationOption OptionDescription="Preselect the web content reader?" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectTranslationOptions || this.SettingsManager.ConfigurationData.HideWebContentReaderForTranslation)" LabelOn="Web content reader is preselected" LabelOff="Web content reader is not preselected" State="@(() => this.SettingsManager.ConfigurationData.PreselectWebContentReaderForTranslation)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.PreselectWebContentReaderForTranslation = updatedState)" OptionHelp="When enabled, the web content reader is preselected. This is might be useful when you prefer to load content from the web very often."/>
+            <ConfigurationOption OptionDescription="Preselect the content cleaner agent?" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectTranslationOptions || this.SettingsManager.ConfigurationData.HideWebContentReaderForTranslation)" LabelOn="Content cleaner agent is preselected" LabelOff="Content cleaner agent is not preselected" State="@(() => this.SettingsManager.ConfigurationData.PreselectContentCleanerAgentForTranslation)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.PreselectContentCleanerAgentForTranslation = updatedState)" OptionHelp="When enabled, the content cleaner agent is preselected. This is might be useful when you prefer to clean up the content before translating it."/>
             <ConfigurationOption OptionDescription="Preselect live translation?" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectTranslationOptions)" LabelOn="Live translation is preselected" LabelOff="Live translation is not preselected" State="@(() => this.SettingsManager.ConfigurationData.PreselectLiveTranslation)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.PreselectLiveTranslation = updatedState)" />
             <ConfigurationSelect OptionDescription="Preselect the target language" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectTranslationOptions)" SelectedValue="@(() => this.SettingsManager.ConfigurationData.PreselectedTranslationTargetLanguage)" Data="@ConfigurationSelectDataFactory.GetCommonLanguagesData()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.PreselectedTranslationTargetLanguage = selectedValue)" OptionHelp="Which target language should be preselected?"/>
             @if (this.SettingsManager.ConfigurationData.PreselectedTranslationTargetLanguage is CommonLanguages.OTHER)
@@ -117,8 +120,11 @@
         </MudPaper>
         
         <MudText Typo="Typo.h5" Class="mb-3">Text Summarizer Options</MudText>
+        <ConfigurationOption OptionDescription="Hide the web content reader?" LabelOn="Web content reader is hidden" LabelOff="Web content reader is shown" State="@(() => this.SettingsManager.ConfigurationData.HideWebContentReaderForTextSummarizer)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.HideWebContentReaderForTextSummarizer = updatedState)" OptionHelp="When activated, the web content reader is hidden and cannot be used. As a result, the user interface becomes a bit easier to use."/>
         <MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
             <ConfigurationOption OptionDescription="Preselect summarizer options?" LabelOn="Summarizer options are preselected" LabelOff="No summarizer options are preselected" State="@(() => this.SettingsManager.ConfigurationData.PreselectTextSummarizerOptions)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.PreselectTextSummarizerOptions = updatedState)" OptionHelp="When enabled, you can preselect the text summarizer options. This is might be useful when you prefer a specific language, complexity, or LLM."/>
+            <ConfigurationOption OptionDescription="Preselect the web content reader?" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectTextSummarizerOptions || this.SettingsManager.ConfigurationData.HideWebContentReaderForTextSummarizer)" LabelOn="Web content reader is preselected" LabelOff="Web content reader is not preselected" State="@(() => this.SettingsManager.ConfigurationData.PreselectWebContentReaderForTextSummarizer)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.PreselectWebContentReaderForTextSummarizer = updatedState)" OptionHelp="When enabled, the web content reader is preselected. This is might be useful when you prefer to load content from the web very often."/>
+            <ConfigurationOption OptionDescription="Preselect the content cleaner agent?" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectTextSummarizerOptions || this.SettingsManager.ConfigurationData.HideWebContentReaderForTextSummarizer)" LabelOn="Content cleaner agent is preselected" LabelOff="Content cleaner agent is not preselected" State="@(() => this.SettingsManager.ConfigurationData.PreselectContentCleanerAgentForTextSummarizer)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.PreselectContentCleanerAgentForTextSummarizer = updatedState)" OptionHelp="When enabled, the content cleaner agent is preselected. This is might be useful when you prefer to clean up the content before summarize it."/>
             <ConfigurationSelect OptionDescription="Preselect the target language" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectTextSummarizerOptions)" SelectedValue="@(() => this.SettingsManager.ConfigurationData.PreselectedTextSummarizerTargetLanguage)" Data="@ConfigurationSelectDataFactory.GetCommonLanguagesData()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.PreselectedTextSummarizerTargetLanguage = selectedValue)" OptionHelp="Which target language should be preselected?"/>
             @if (this.SettingsManager.ConfigurationData.PreselectedTextSummarizerTargetLanguage is CommonLanguages.OTHER)
             {
@@ -131,5 +137,17 @@
             }
             <ConfigurationProviderSelection Data="@this.availableProviders" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectTextSummarizerOptions)" SelectedValue="@(() => this.SettingsManager.ConfigurationData.PreselectedTextSummarizerProvider)" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.PreselectedTextSummarizerProvider = selectedValue)"/>
         </MudPaper>
+
+        <MudText Typo="Typo.h4" Class="mb-3">LLM Agent Options</MudText>
+        
+        <MudText Typo="Typo.h5" Class="mb-3">Text Content Cleaner Agent</MudText>
+        <MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
+            <MudText Typo="Typo.body1" Class="mb-3">
+                Use Case: this agent is used to clean up text content. It extracts the main content, removes advertisements and other irrelevant things,
+                and attempts to convert relative links into absolute links so that they can be used.
+            </MudText>
+            <ConfigurationOption OptionDescription="Preselect text content cleaner options?" LabelOn="Options are preselected" LabelOff="No options are preselected" State="@(() => this.SettingsManager.ConfigurationData.PreselectAgentTextContentCleanerOptions)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.PreselectAgentTextContentCleanerOptions = updatedState)" OptionHelp="When enabled, you can preselect some agent options. This is might be useful when you prefer a LLM."/>
+            <ConfigurationProviderSelection Data="@this.availableProviders" Disabled="@(() => !this.SettingsManager.ConfigurationData.PreselectAgentTextContentCleanerOptions)" SelectedValue="@(() => this.SettingsManager.ConfigurationData.PreselectedAgentTextContentCleanerProvider)" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.PreselectedAgentTextContentCleanerProvider = selectedValue)"/>
+        </MudPaper>
     </MudPaper>
 </InnerScrolling>

app/MindWork AI Studio/Components/Pages/TextSummarizer/AssistantTextSummarizer.razor

@@ -3,7 +3,12 @@
 @using AIStudio.Tools
 @inherits AssistantBaseCore
 
-<MudTextField T="string" @bind-Text="@this.inputText" Validation="@this.ValidatingText" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Label="Your input" Variant="Variant.Outlined" Lines="6" AutoGrow="@true" MaxLines="12" Class="mb-3" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
+@if (!this.SettingsManager.ConfigurationData.HideWebContentReaderForTextSummarizer)
+{
+    <ReadWebContent @bind-Content="@this.inputText" ProviderSettings="@this.providerSettings" @bind-AgentIsRunning="@this.isAgentRunning" Preselect="@(this.SettingsManager.ConfigurationData.PreselectTextSummarizerOptions && this.SettingsManager.ConfigurationData.PreselectWebContentReaderForTextSummarizer)" PreselectContentCleanerAgent="@(this.SettingsManager.ConfigurationData.PreselectTextSummarizerOptions && this.SettingsManager.ConfigurationData.PreselectContentCleanerAgentForTextSummarizer)"/>
+}
+
+<MudTextField T="string" Disabled="@this.isAgentRunning" @bind-Text="@this.inputText" Validation="@this.ValidatingText" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Label="Your input" Variant="Variant.Outlined" Lines="6" AutoGrow="@true" MaxLines="12" Class="mb-3" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
 
 <MudStack Row="@true" AlignItems="AlignItems.Center" Class="mb-3">
     <MudSelect T="CommonLanguages" @bind-Value="@this.selectedTargetLanguage" AdornmentIcon="@Icons.Material.Filled.Translate" Adornment="Adornment.Start" Label="Target language" Variant="Variant.Outlined" Margin="Margin.Dense">
@@ -31,13 +36,13 @@
     }
 </MudStack>
 
-<MudSelect T="Provider" @bind-Value="@this.providerSettings" Validation="@this.ValidatingProvider" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Apps" Margin="Margin.Dense" Label="Provider" Class="mb-3 rounded-lg" Variant="Variant.Outlined">
+<MudSelect T="Provider" Disabled="@this.isAgentRunning" @bind-Value="@this.providerSettings" Validation="@this.ValidatingProvider" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Apps" Margin="Margin.Dense" Label="Provider" Class="mb-3 rounded-lg" Variant="Variant.Outlined">
     @foreach (var provider in this.SettingsManager.ConfigurationData.Providers)
     {
         <MudSelectItem Value="@provider"/>
     }
 </MudSelect>
 
-<MudButton Variant="Variant.Filled" Class="mb-3" OnClick="() => this.SummarizeText()">
+<MudButton Variant="Variant.Filled" Class="mb-3" OnClick="() => this.SummarizeText()" Disabled="@this.isAgentRunning">
     Summarize
 </MudButton>

app/MindWork AI Studio/Components/Pages/TextSummarizer/AssistantTextSummarizer.razor.cs

@@ -24,6 +24,7 @@ public partial class AssistantTextSummarizer : AssistantBaseCore
         """;
     
     private string inputText = string.Empty;
+    private bool isAgentRunning;
     private CommonLanguages selectedTargetLanguage;
     private string customTargetLanguage = string.Empty;
     private Complexity selectedComplexity;

app/MindWork AI Studio/Components/Pages/Translation/AssistantTranslation.razor

@@ -3,6 +3,11 @@
 @using AIStudio.Tools
 @inherits AssistantBaseCore
 
+@if (!this.SettingsManager.ConfigurationData.HideWebContentReaderForTranslation)
+{
+    <ReadWebContent @bind-Content="@this.inputText" ProviderSettings="@this.providerSettings" @bind-AgentIsRunning="@this.isAgentRunning" Preselect="@(this.SettingsManager.ConfigurationData.PreselectTranslationOptions && this.SettingsManager.ConfigurationData.PreselectWebContentReaderForTranslation)" PreselectContentCleanerAgent="@(this.SettingsManager.ConfigurationData.PreselectTranslationOptions && this.SettingsManager.ConfigurationData.PreselectContentCleanerAgentForTranslation)"/>
+}
+
 <MudField Label="Live translation" Variant="Variant.Outlined" Class="mb-3">
     <MudSwitch T="bool" @bind-Value="@this.liveTranslation" Color="Color.Primary">
         @(this.liveTranslation ? "Live translation" : "No live translation")
@@ -11,11 +16,11 @@
 
 @if (this.liveTranslation)
 {
-    <MudTextField T="string" @bind-Text="@this.inputText" Validation="@this.ValidatingText" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Label="Your input" Variant="Variant.Outlined" Lines="6" AutoGrow="@true" MaxLines="12" Class="mb-3" Immediate="@true" DebounceInterval="1_000" OnDebounceIntervalElapsed="() => this.TranslateText(force: false)" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
+    <MudTextField T="string" Disabled="@this.isAgentRunning" @bind-Text="@this.inputText" Validation="@this.ValidatingText" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Label="Your input" Variant="Variant.Outlined" Lines="6" AutoGrow="@true" MaxLines="12" Class="mb-3" Immediate="@true" DebounceInterval="1_000" OnDebounceIntervalElapsed="() => this.TranslateText(force: false)" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
 }
 else
 {
-    <MudTextField T="string" @bind-Text="@this.inputText" Validation="@this.ValidatingText" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Label="Your input" Variant="Variant.Outlined" Lines="6" AutoGrow="@true" MaxLines="12" Class="mb-3" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
+    <MudTextField T="string" Disabled="@this.isAgentRunning" @bind-Text="@this.inputText" Validation="@this.ValidatingText" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Label="Your input" Variant="Variant.Outlined" Lines="6" AutoGrow="@true" MaxLines="12" Class="mb-3" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
 }
 
 <MudStack Row="@true" AlignItems="AlignItems.Center" Class="mb-3">
@@ -38,13 +43,13 @@ else
     }
 </MudStack>
 
-<MudSelect T="Provider" @bind-Value="@this.providerSettings" Validation="@this.ValidatingProvider" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Apps" Margin="Margin.Dense" Label="Provider" Class="mb-3 rounded-lg" Variant="Variant.Outlined">
+<MudSelect T="Provider" @bind-Value="@this.providerSettings" Disabled="@this.isAgentRunning" Validation="@this.ValidatingProvider" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Apps" Margin="Margin.Dense" Label="Provider" Class="mb-3 rounded-lg" Variant="Variant.Outlined">
     @foreach (var provider in this.SettingsManager.ConfigurationData.Providers)
     {
         <MudSelectItem Value="@provider"/>
     }
 </MudSelect>
 
-<MudButton Variant="Variant.Filled" Class="mb-3" OnClick="() => this.TranslateText(force: true)">
+<MudButton Disabled="@this.isAgentRunning" Variant="Variant.Filled" Class="mb-3" OnClick="() => this.TranslateText(force: true)">
     Translate
 </MudButton>

app/MindWork AI Studio/Components/Pages/Translation/AssistantTranslation.razor.cs

@@ -20,6 +20,7 @@ public partial class AssistantTranslation : AssistantBaseCore
         """;
     
     private bool liveTranslation;
+    private bool isAgentRunning;
     private string inputText = string.Empty;
     private string inputTextLastTranslation = string.Empty;
     private CommonLanguages selectedTargetLanguage;

app/MindWork AI Studio/MindWork AI Studio.csproj

@@ -45,9 +45,11 @@
     </ItemGroup>
 
     <ItemGroup>
+      <PackageReference Include="HtmlAgilityPack" Version="1.11.62" />
       <PackageReference Include="Microsoft.Extensions.FileProviders.Embedded" Version="8.0.7" />
       <PackageReference Include="MudBlazor" Version="7.4.0" />
       <PackageReference Include="MudBlazor.Markdown" Version="1.0.2" />
+      <PackageReference Include="ReverseMarkdown" Version="4.6.0" />
     </ItemGroup>
 
     <!-- Read the meta data file -->

app/MindWork AI Studio/Program.cs

@@ -1,4 +1,5 @@
 using AIStudio;
+using AIStudio.Agents;
 using AIStudio.Components;
 using AIStudio.Settings;
 using AIStudio.Tools;
@@ -29,7 +30,9 @@ builder.Services.AddSingleton(MessageBus.INSTANCE);
 builder.Services.AddSingleton<Rust>();
 builder.Services.AddMudMarkdownClipboardService<MarkdownClipboardService>();
 builder.Services.AddSingleton<SettingsManager>();
-builder.Services.AddSingleton<Random>();
+builder.Services.AddSingleton<ThreadSafeRandom>();
+builder.Services.AddTransient<HTMLParser>();
+builder.Services.AddTransient<AgentTextContentCleaner>();
 builder.Services.AddHostedService<UpdateService>();
 builder.Services.AddHostedService<TemporaryChatService>();
 builder.Services.AddRazorComponents()

app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs

@@ -41,6 +41,7 @@ public sealed class ProviderAnthropic() : BaseProvider("https://api.anthropic.co
                 {
                     ChatRole.USER => "user",
                     ChatRole.AI => "assistant",
+                    ChatRole.AGENT => "assistant",
 
                     _ => "user",
                 },

app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs

@@ -52,6 +52,7 @@ public class ProviderFireworks() : BaseProvider("https://api.fireworks.ai/infere
                 {
                     ChatRole.USER => "user",
                     ChatRole.AI => "assistant",
+                    ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
 
                     _ => "user",

app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs

@@ -51,6 +51,7 @@ public sealed class ProviderMistral() : BaseProvider("https://api.mistral.ai/v1/
                 {
                     ChatRole.USER => "user",
                     ChatRole.AI => "assistant",
+                    ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
 
                     _ => "user",

app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs

@@ -55,6 +55,7 @@ public sealed class ProviderOpenAI() : BaseProvider("https://api.openai.com/v1/"
                 {
                     ChatRole.USER => "user",
                     ChatRole.AI => "assistant",
+                    ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
 
                     _ => "user",

app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs

@@ -44,6 +44,7 @@ public sealed class ProviderSelfHosted(Settings.Provider provider) : BaseProvide
                 {
                     ChatRole.USER => "user",
                     ChatRole.AI => "assistant",
+                    ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
 
                     _ => "user",

app/MindWork AI Studio/Settings/DataModel/Data.cs

@@ -120,6 +120,21 @@ public sealed class Data
     /// </summary>
     public bool PreselectLiveTranslation { get; set; }
 
+    /// <summary>
+    /// Hide the web content reader?
+    /// </summary>
+    public bool HideWebContentReaderForTranslation { get; set; }
+
+    /// <summary>
+    /// Preselect the web content reader?
+    /// </summary>
+    public bool PreselectWebContentReaderForTranslation { get; set; }
+
+    /// <summary>
+    /// Preselect the content cleaner agent?
+    /// </summary>
+    public bool PreselectContentCleanerAgentForTranslation { get; set; }
+
     /// <summary>
     /// Preselect the target language?
     /// </summary>
@@ -173,6 +188,22 @@ public sealed class Data
     /// </summary>
     public bool PreselectTextSummarizerOptions { get; set; }
 
+    
+    /// <summary>
+    /// Hide the web content reader?
+    /// </summary>
+    public bool HideWebContentReaderForTextSummarizer { get; set; }
+
+    /// <summary>
+    /// Preselect the web content reader?
+    /// </summary>
+    public bool PreselectWebContentReaderForTextSummarizer { get; set; }
+
+    /// <summary>
+    /// Preselect the content cleaner agent?
+    /// </summary>
+    public bool PreselectContentCleanerAgentForTextSummarizer { get; set; }
+    
     /// <summary>
     /// Preselect the target language?
     /// </summary>
@@ -199,4 +230,18 @@ public sealed class Data
     public string PreselectedTextSummarizerProvider { get; set; } = string.Empty;
 
     #endregion
+
+    #region Agent: Text Content Cleaner Settings
+
+    /// <summary>
+    /// Preselect any text content cleaner options?
+    /// </summary>
+    public bool PreselectAgentTextContentCleanerOptions { get; set; }
+    
+    /// <summary>
+    /// Preselect a text content cleaner provider?
+    /// </summary>
+    public string PreselectedAgentTextContentCleanerProvider { get; set; } = string.Empty;
+
+    #endregion
 }

app/MindWork AI Studio/Tools/HTMLParser.cs

@@ -0,0 +1,57 @@
+using System.Net;
+using System.Text;
+
+using HtmlAgilityPack;
+
+using ReverseMarkdown;
+
+namespace AIStudio.Tools;
+
+public sealed class HTMLParser
+{
+    private static readonly Config MARKDOWN_PARSER_CONFIG = new()
+    {
+        UnknownTags = Config.UnknownTagsOption.Bypass,
+        RemoveComments = true,
+        SmartHrefHandling = true
+    };
+
+    /// <summary>
+    /// Loads the web content from the specified URL.
+    /// </summary>
+    /// <param name="url">The URL of the web page.</param>
+    /// <returns>The web content as text.</returns>
+    public async Task<string> LoadWebContentText(Uri url)
+    {
+        var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+        var parser = new HtmlWeb();
+        var doc = await parser.LoadFromWebAsync(url, Encoding.UTF8, new NetworkCredential(), cts.Token);
+        return doc.ParsedText;
+    }
+
+    /// <summary>
+    /// Loads the web content from the specified URL and returns it as an HTML string.
+    /// </summary>
+    /// <param name="url">The URL of the web page.</param>
+    /// <returns>The web content as an HTML string.</returns>
+    public async Task<string> LoadWebContentHTML(Uri url)
+    {
+        var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+        var parser = new HtmlWeb();
+        var doc = await parser.LoadFromWebAsync(url, Encoding.UTF8, new NetworkCredential(), cts.Token);
+        var innerHtml = doc.DocumentNode.InnerHtml;
+
+        return innerHtml;
+    }
+
+    /// <summary>
+    /// Converts HTML content to the Markdown format.
+    /// </summary>
+    /// <param name="html">The HTML content to parse.</param>
+    /// <returns>The converted Markdown content.</returns>
+    public string ParseToMarkdown(string html)
+    {
+        var markdownConverter = new Converter(MARKDOWN_PARSER_CONFIG);
+        return markdownConverter.Convert(html);
+    }
+}

app/MindWork AI Studio/Tools/Process.cs

@@ -0,0 +1,67 @@
+using System.Text;
+
+namespace AIStudio.Tools;
+
+public sealed class Process<T> where T : struct, Enum
+{
+    public static readonly Process<T> INSTANCE = new();
+    
+    private readonly Dictionary<T, ProcessStepValue> stepsData = [];
+    private readonly int min = int.MaxValue;
+    private readonly int max = int.MinValue;
+    private readonly string[] labels;
+
+    private Process()
+    {
+        var values = Enum.GetValues<T>();
+        this.labels = new string[values.Length];
+
+        for (var i = 0; i < values.Length; i++)
+        {
+            var value = values[i];
+            var stepValue = Convert.ToInt32(value);
+            var stepName = DeriveName(value);
+
+            this.labels[i] = stepName;
+            this.stepsData[value] = new ProcessStepValue(stepValue, stepName);
+
+            if (stepValue < this.min)
+                this.min = stepValue;
+
+            if (stepValue > this.max)
+                this.max = stepValue;
+        }
+    }
+    
+    private static string DeriveName(T value)
+    {
+        var text = value.ToString();
+        if (!text.Contains('_'))
+        {
+            text = text.ToLowerInvariant();
+            text = char.ToUpperInvariant(text[0]) + text[1..];
+        }
+        else
+        {
+            var parts = text.Split('_');
+            var sb = new StringBuilder();
+            foreach (var part in parts)
+            {
+                sb.Append(char.ToUpperInvariant(part[0]));
+                sb.Append(part[1..].ToLowerInvariant());
+            }
+            
+            text = sb.ToString();
+        }
+        
+        return text;
+    }
+    
+    public string[] Labels => this.labels;
+    
+    public int Min => this.min;
+    
+    public int Max => this.max;
+    
+    public ProcessStepValue this[T step] => this.stepsData[step];
+}

app/MindWork AI Studio/Tools/ProcessStepValue.cs

@@ -0,0 +1,258 @@
+using System.Globalization;
+using System.Numerics;
+
+namespace AIStudio.Tools;
+
+public readonly record struct ProcessStepValue(int Step, string Name) : INumber<ProcessStepValue>
+{
+    public static implicit operator int(ProcessStepValue process) => process.Step;
+    
+    #region INumber implementation
+    
+    #region Implementation of IComparable
+    
+    public int CompareTo(object? obj) => this.Step.CompareTo(obj);
+    
+    #endregion
+    
+    #region Implementation of IComparable<in ProcessStepValue>
+    
+    public int CompareTo(ProcessStepValue other) => this.Step.CompareTo(other.Step);
+    
+    #endregion
+    
+    #region Implementation of IFormattable
+    
+    public string ToString(string? format, IFormatProvider? formatProvider) => this.Step.ToString(format, formatProvider);
+    
+    #endregion
+    
+    #region Implementation of IParsable<ProcessStepValue>
+    
+    public static ProcessStepValue Parse(string s, IFormatProvider? provider) => new(int.Parse(s, provider), string.Empty);
+
+    public static bool TryParse(string? s, IFormatProvider? provider, out ProcessStepValue result)
+    {
+        if (int.TryParse(s, provider, out var stepValue))
+        {
+            result = new ProcessStepValue(stepValue, string.Empty);
+            return true;
+        }
+        
+        result = default;
+        return false;
+    }
+    #endregion
+    
+    #region Implementation of ISpanFormattable
+    
+    public bool TryFormat(Span<char> destination, out int charsWritten, ReadOnlySpan<char> format, IFormatProvider? provider) => this.Step.TryFormat(destination, out charsWritten, format, provider);
+
+    #endregion
+    
+    #region Implementation of ISpanParsable<ProcessStepValue>
+    
+    public static ProcessStepValue Parse(ReadOnlySpan<char> s, IFormatProvider? provider) => new(int.Parse(s, provider), string.Empty);
+
+    public static bool TryParse(ReadOnlySpan<char> s, IFormatProvider? provider, out ProcessStepValue result)
+    {
+        if (int.TryParse(s, provider, out int stepValue))
+        {
+            result = new ProcessStepValue(stepValue, string.Empty);
+            return true;
+        }
+        
+        result = default;
+        return false;
+    }
+    
+    #endregion
+    
+    #region Implementation of IAdditionOperators<ProcessStepValue,ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue operator +(ProcessStepValue left, ProcessStepValue right) => left with { Step = left.Step + right.Step };
+
+    #endregion
+    
+    #region Implementation of IAdditiveIdentity<ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue AdditiveIdentity => new(0, string.Empty);
+    
+    #endregion
+    
+    #region Implementation of IComparisonOperators<ProcessStepValue,ProcessStepValue,bool>
+    
+    public static bool operator >(ProcessStepValue left, ProcessStepValue right) => left.Step > right.Step;
+
+    public static bool operator >=(ProcessStepValue left, ProcessStepValue right) => left.Step >= right.Step;
+
+    public static bool operator <(ProcessStepValue left, ProcessStepValue right) => left.Step < right.Step;
+
+    public static bool operator <=(ProcessStepValue left, ProcessStepValue right) => left.Step <= right.Step;
+
+    #endregion
+    
+    #region Implementation of IDecrementOperators<ProcessStepValue>
+    
+    public static ProcessStepValue operator --(ProcessStepValue value) => value with { Step = value.Step - 1 };
+
+    #endregion
+
+    #region Implementation of IDivisionOperators<ProcessStepValue,ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue operator /(ProcessStepValue left, ProcessStepValue right) => left with { Step = left.Step / right.Step };
+
+    #endregion
+    
+    #region Implementation of IIncrementOperators<ProcessStepValue>
+    
+    public static ProcessStepValue operator ++(ProcessStepValue value) => value with { Step = value.Step + 1 };
+
+    #endregion
+    
+    #region Implementation of IModulusOperators<ProcessStepValue,ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue operator %(ProcessStepValue left, ProcessStepValue right) => left with { Step = left.Step % right.Step };
+
+    #endregion
+    
+    #region Implementation of IMultiplicativeIdentity<ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue MultiplicativeIdentity => new(1, string.Empty);
+    
+    #endregion
+    
+    #region Implementation of IMultiplyOperators<ProcessStepValue,ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue operator *(ProcessStepValue left, ProcessStepValue right) => left with { Step = left.Step * right.Step };
+
+    #endregion
+    
+    #region Implementation of ISubtractionOperators<ProcessStepValue,ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue operator -(ProcessStepValue left, ProcessStepValue right) => left with { Step = left.Step - right.Step };
+
+    #endregion
+    
+    #region Implementation of IUnaryNegationOperators<ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue operator -(ProcessStepValue value) => value with { Step = -value.Step };
+
+    #endregion
+    
+    #region Implementation of IUnaryPlusOperators<ProcessStepValue,ProcessStepValue>
+    
+    public static ProcessStepValue operator +(ProcessStepValue value) => value;
+
+    #endregion
+    
+    #region Implementation of INumberBase<ProcessStepValue>
+    
+    public static ProcessStepValue Abs(ProcessStepValue value) => value with { Step = Math.Abs(value.Step) };
+
+    public static bool IsCanonical(ProcessStepValue value) => true;
+    public static bool IsComplexNumber(ProcessStepValue value) => false;
+    public static bool IsEvenInteger(ProcessStepValue value) => value.Step % 2 == 0;
+    public static bool IsFinite(ProcessStepValue value) => true;
+    public static bool IsImaginaryNumber(ProcessStepValue value) => false;
+    public static bool IsInfinity(ProcessStepValue value) => false;
+    public static bool IsInteger(ProcessStepValue value) => true;
+    public static bool IsNaN(ProcessStepValue value) => false;
+    public static bool IsNegative(ProcessStepValue value) => value.Step < 0;
+    public static bool IsNegativeInfinity(ProcessStepValue value) => false;
+    public static bool IsNormal(ProcessStepValue value) => true;
+    public static bool IsOddInteger(ProcessStepValue value) => value.Step % 2 != 0;
+    public static bool IsPositive(ProcessStepValue value) => value.Step > 0;
+    public static bool IsPositiveInfinity(ProcessStepValue value) => false;
+    public static bool IsRealNumber(ProcessStepValue value) => true;
+    public static bool IsSubnormal(ProcessStepValue value) => false;
+    public static bool IsZero(ProcessStepValue value) => value.Step == 0;
+    public static ProcessStepValue MaxMagnitude(ProcessStepValue x, ProcessStepValue y)
+    {
+        return x with { Step = Math.Max(Math.Abs(x.Step), Math.Abs(y.Step)) };
+    }
+    
+    public static ProcessStepValue MaxMagnitudeNumber(ProcessStepValue x, ProcessStepValue y) => MaxMagnitude(x, y);
+
+    public static ProcessStepValue MinMagnitude(ProcessStepValue x, ProcessStepValue y) => x with { Step = Math.Min(Math.Abs(x.Step), Math.Abs(y.Step)) };
+
+    public static ProcessStepValue MinMagnitudeNumber(ProcessStepValue x, ProcessStepValue y) => MinMagnitude(x, y);
+
+    public static ProcessStepValue Parse(ReadOnlySpan<char> s, NumberStyles style, IFormatProvider? provider) => new(int.Parse(s, style, provider), string.Empty);
+
+    public static ProcessStepValue Parse(string s, NumberStyles style, IFormatProvider? provider) => new(int.Parse(s, style, provider), string.Empty);
+
+    public static bool TryConvertFromChecked<TOther>(TOther value, out ProcessStepValue result) where TOther : INumberBase<TOther>
+    {
+        if (TOther.TryConvertToChecked(value, out int intValue))
+        {
+            result = new ProcessStepValue(intValue, string.Empty);
+            return true;
+        }
+        
+        result = default;
+        return false;
+    }
+    
+    public static bool TryConvertFromSaturating<TOther>(TOther value, out ProcessStepValue result) where TOther : INumberBase<TOther>
+    {
+        if (TOther.TryConvertToSaturating(value, out int intValue))
+        {
+            result = new ProcessStepValue(intValue, string.Empty);
+            return true;
+        }
+        result = default;
+        return false;
+    }
+    
+    public static bool TryConvertFromTruncating<TOther>(TOther value, out ProcessStepValue result) where TOther : INumberBase<TOther>
+    {
+        if (TOther.TryConvertToTruncating(value, out int intValue))
+        {
+            result = new ProcessStepValue(intValue, string.Empty);
+            return true;
+        }
+        result = default;
+        return false;
+    }
+    
+    public static bool TryConvertToChecked<TOther>(ProcessStepValue value, out TOther result) where TOther : INumberBase<TOther> => TOther.TryConvertFromChecked(value.Step, out result!);
+
+    public static bool TryConvertToSaturating<TOther>(ProcessStepValue value, out TOther result) where TOther : INumberBase<TOther> => TOther.TryConvertFromSaturating(value.Step, out result!);
+
+    public static bool TryConvertToTruncating<TOther>(ProcessStepValue value, out TOther result) where TOther : INumberBase<TOther> => TOther.TryConvertFromTruncating(value.Step, out result!);
+
+    public static bool TryParse(ReadOnlySpan<char> s, NumberStyles style, IFormatProvider? provider, out ProcessStepValue result)
+    {
+        if (int.TryParse(s, style, provider, out var stepValue))
+        {
+            result = new ProcessStepValue(stepValue, string.Empty);
+            return true;
+        }
+        
+        result = default;
+        return false;
+    }
+    
+    public static bool TryParse(string? s, NumberStyles style, IFormatProvider? provider, out ProcessStepValue result)
+    {
+        if (int.TryParse(s, style, provider, out var stepValue))
+        {
+            result = new ProcessStepValue(stepValue, string.Empty);
+            return true;
+        }
+        
+        result = default;
+        return false;
+    }
+    
+    public static ProcessStepValue One => new(1, string.Empty);
+    
+    public static int Radix => 2;
+    
+    public static ProcessStepValue Zero => new(0, string.Empty);
+    
+    #endregion
+    
+    #endregion
+}

app/MindWork AI Studio/Tools/ThreadSafeRandom.cs

@@ -0,0 +1,88 @@
+namespace AIStudio.Tools;
+
+/// <inheritdoc />
+public sealed class ThreadSafeRandom : Random
+{
+    private static readonly object LOCK = new();
+
+    #region Overrides of Random
+
+    /// <inheritdoc />
+    public override int Next()
+    {
+        lock (LOCK)
+            return base.Next();
+    }
+
+    /// <inheritdoc />
+    public override int Next(int maxValue)
+    {
+        lock (LOCK)
+            return base.Next(maxValue);
+    }
+
+    /// <inheritdoc />
+    public override int Next(int minValue, int maxValue)
+    {
+        lock (LOCK)
+            return base.Next(minValue, maxValue);
+    }
+
+    /// <inheritdoc />
+    public override void NextBytes(byte[] buffer)
+    {
+        lock (LOCK)
+            base.NextBytes(buffer);
+    }
+
+    /// <inheritdoc />
+    public override void NextBytes(Span<byte> buffer)
+    {
+        lock (LOCK)
+            base.NextBytes(buffer);
+    }
+
+    /// <inheritdoc />
+    public override double NextDouble()
+    {
+        lock (LOCK)
+            return base.NextDouble();
+    }
+
+    /// <inheritdoc />
+    public override long NextInt64()
+    {
+        lock (LOCK)
+            return base.NextInt64();
+    }
+
+    /// <inheritdoc />
+    public override long NextInt64(long maxValue)
+    {
+        lock (LOCK)
+            return base.NextInt64(maxValue);
+    }
+
+    /// <inheritdoc />
+    public override long NextInt64(long minValue, long maxValue)
+    {
+        lock (LOCK)
+            return base.NextInt64(minValue, maxValue);
+    }
+
+    /// <inheritdoc />
+    public override float NextSingle()
+    {
+        lock (LOCK)
+            return base.NextSingle();
+    }
+
+    /// <inheritdoc />
+    protected override double Sample()
+    {
+        lock (LOCK)
+            return base.Sample();
+    }
+
+    #endregion
+}

app/MindWork AI Studio/packages.lock.json

@@ -2,6 +2,12 @@
   "version": 1,
   "dependencies": {
     "net8.0": {
+      "HtmlAgilityPack": {
+        "type": "Direct",
+        "requested": "[1.11.62, )",
+        "resolved": "1.11.62",
+        "contentHash": "KS4h7ZjWsO6YixRfQgYdR+PZMbTaZod1LBPi+1Ph7dJCARzQm7DOKe5HPhP/o0EWX5l7QCgAZHa4VOa4pQa8JQ=="
+      },
       "Microsoft.Extensions.FileProviders.Embedded": {
         "type": "Direct",
         "requested": "[8.0.7, )",
@@ -38,6 +44,15 @@
           "MudBlazor": "6.20.0"
         }
       },
+      "ReverseMarkdown": {
+        "type": "Direct",
+        "requested": "[4.6.0, )",
+        "resolved": "4.6.0",
+        "contentHash": "ehNpMz3yQwd7P/vHpwi4KyDlT8UtVmtiL+NTb6mFEPzbLqJXbRIGF4OxEA5tuBA5Cfwhzf537TX1UIB6dUpo7A==",
+        "dependencies": {
+          "HtmlAgilityPack": "1.11.61"
+        }
+      },
       "Markdig": {
         "type": "Transitive",
         "resolved": "0.37.0",
@@ -163,6 +178,6 @@
         "contentHash": "FHNOatmUq0sqJOkTx+UF/9YK1f180cnW5FVqnQMvYUN0elp6wFzbtPSiqbo1/ru8ICp43JM1i7kKkk6GsNGHlA=="
       }
     },
-    "net8.0/osx-x64": {}
+    "net8.0/osx-arm64": {}
   }
 }

app/MindWork AI Studio/wwwroot/changelog/v0.8.6.md

@@ -1,5 +1,9 @@
-# v0.8.6, build 168
+# v0.8.6, build 168 (2024-08-01 19:50 UTC)
 - Added possibility to configure a default provider for chats
+- Added architecture for future agent usage
+- Added a first agent to read, analyze and extract text from Markdown data
+- Added option to read web content into translation and text summarization assistants
 - Improved the readability of the `settings.json` file by using indentation and enum names instead of numbers
 - Improved assistant overview; assistants will now wrap to the next line if there are too many to fit on the row
-- Increased the default value for the live translation delay from 1,000 to 1,500 ms
+- Increased the default value for the live translation delay from 1,000 to 3_000 ms
+- Fixed random number generator usage to be thread-safe

metadata.txt

@@ -1,9 +1,9 @@
-0.8.5
+0.8.6
-2024-07-28 16:44:01 UTC
+2024-08-01 19:50:02 UTC
-167
+168
 8.0.107 (commit 1bdaef7265)
 8.0.7 (commit 2aade6beb0)
 1.80.0 (commit 051478957)
 7.4.0
 1.7.1
-c1b92a05fc7, release
+199be1e863f, release

runtime/Cargo.lock

@@ -2308,7 +2308,7 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
 [[package]]
 name = "mindwork-ai-studio"
-version = "0.8.5"
+version = "0.8.6"
 dependencies = [
  "arboard",
  "flexi_logger",

runtime/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mindwork-ai-studio"
-version = "0.8.5"
+version = "0.8.6"
 edition = "2021"
 description = "MindWork AI Studio"
 authors = ["Thorsten Sommer"]

runtime/tauri.conf.json

@@ -6,7 +6,7 @@
   },
   "package": {
     "productName": "MindWork AI Studio",
-	"version": "0.8.5"
+	"version": "0.8.6"
   },
   "tauri": {
     "allowlist": {