From 032c7ee56efb2e7e3e9af91c2db13e5333839670 Mon Sep 17 00:00:00 2001
From: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
Date: Sat, 8 Mar 2025 21:04:17 +0100
Subject: [PATCH] Added the retrieval process section for ERI data sources
 (#318)

---
 .../Dialogs/DataSourceERI-V1InfoDialog.razor  | 88 +++++++++----------
 .../DataSourceERI-V1InfoDialog.razor.cs       |  2 +
 .../Dialogs/DataSourceERI_V1Dialog.razor      | 16 +++-
 .../Dialogs/DataSourceERI_V1Dialog.razor.cs   | 26 ++++++
 .../Settings/DataModel/DataSourceERI_V1.cs    |  5 +-
 .../Settings/IERIDataSource.cs                |  5 ++
 .../Tools/Validation/DataSourceValidation.cs  |  8 ++
 .../wwwroot/changelog/v0.9.32.md              |  1 +
 8 files changed, 103 insertions(+), 48 deletions(-)

diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor
index 41344b9..dd6d277 100644
--- a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor	
+++ b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor	
@@ -42,53 +42,49 @@
         else
         {
             <MudExpansionPanels Class="mb-3">
-                @for (var index = 0; index < this.retrievalInfoformation.Count; index++)
-                {
-                    var info = this.retrievalInfoformation[index];
-                    <ExpansionPanel HeaderText="@this.RetrievalName(info)" HeaderIcon="@Icons.Material.Filled.Info" IsExpanded="index == 0">
-                        <TextInfoLines Label="Description" MaxLines="14" Value="@info.Description" ClipboardTooltipSubject="the retrieval description"/>
-                        <TextInfoLines Label="Parameters" MaxLines="14" Value="@this.RetrievalParameters(info)" ClipboardTooltipSubject="the retrieval parameters"/>
-                        
-                        @if (!string.IsNullOrWhiteSpace(info.Link))
-                        {
-                            <MudButton Href="@info.Link" Target="_blank" Class="mt-3" Color="Color.Primary" StartIcon="@Icons.Material.Filled.OpenInNew">
-                                Open web link, show more information
-                            </MudButton>
-                        }
+                <ExpansionPanel HeaderText="@this.RetrievalName(this.selectedRetrievalInfo)" HeaderIcon="@Icons.Material.Filled.Info">
+                    <TextInfoLines Label="Description" MaxLines="14" Value="@this.selectedRetrievalInfo.Description" ClipboardTooltipSubject="the retrieval description"/>
+                    <TextInfoLines Label="Parameters" MaxLines="14" Value="@this.RetrievalParameters(this.selectedRetrievalInfo)" ClipboardTooltipSubject="the retrieval parameters"/>
 
-                        <MudText Typo="Typo.h6" Class="mt-3">
-                            Embeddings
-                        </MudText>
-                        @* ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract *@
-                        @if (info.Embeddings is null || !info.Embeddings.Any())
-                        {
-                            <MudJustifiedText Typo="Typo.body1" Color="Color.Info" Class="mb-3">
-                                The data source does not provide any embedding information.
-                            </MudJustifiedText>
-                        }
-                        else
-                        {
-                            <MudExpansionPanels>
-                                @for (var embeddingIndex = 0; embeddingIndex < info.Embeddings.Count; embeddingIndex++)
-                                {
-                                    var embedding = info.Embeddings[embeddingIndex];
-                                    <ExpansionPanel HeaderText="@embedding.EmbeddingName" HeaderIcon="@Icons.Material.Filled.Info" IsExpanded="embeddingIndex == 0">
-                                        <TextInfoLine Icon="@Icons.Material.Filled.FormatShapes" Label="Type" Value="@embedding.EmbeddingType" ClipboardTooltipSubject="the embedding type"/>
-                                        <TextInfoLines Label="Description" MaxLines="14" Value="@embedding.Description" ClipboardTooltipSubject="the embedding description"/>
-                                        <TextInfoLines Label="When to use" MaxLines="3" Value="@embedding.UsedWhen" ClipboardTooltipSubject="when is the embedding used"/>
-                                        
-                                        @if (!string.IsNullOrWhiteSpace(embedding.Link))
-                                        {
-                                            <MudButton Href="@embedding.Link" Target="_blank" Class="mt-3" Color="Color.Primary" StartIcon="@Icons.Material.Filled.OpenInNew">
-                                                Open web link, show more information
-                                            </MudButton>
-                                        }
-                                    </ExpansionPanel>
-                                }
-                            </MudExpansionPanels>
-                        }
-                    </ExpansionPanel>
-                }
+                    @if (!string.IsNullOrWhiteSpace(this.selectedRetrievalInfo.Link))
+                    {
+                        <MudButton Href="@this.selectedRetrievalInfo.Link" Target="_blank" Class="mt-3" Color="Color.Primary" StartIcon="@Icons.Material.Filled.OpenInNew">
+                            Open web link, show more information
+                        </MudButton>
+                    }
+
+                    <MudText Typo="Typo.h6" Class="mt-3">
+                        Embeddings
+                    </MudText>
+                    @* ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract *@
+                    @if (this.selectedRetrievalInfo.Embeddings is null || !this.selectedRetrievalInfo.Embeddings.Any())
+                    {
+                        <MudJustifiedText Typo="Typo.body1" Color="Color.Info" Class="mb-3">
+                            The data source does not provide any embedding information.
+                        </MudJustifiedText>
+                    }
+                    else
+                    {
+                        <MudExpansionPanels>
+                            @for (var embeddingIndex = 0; embeddingIndex < this.selectedRetrievalInfo.Embeddings.Count; embeddingIndex++)
+                            {
+                                var embedding = this.selectedRetrievalInfo.Embeddings[embeddingIndex];
+                                <ExpansionPanel HeaderText="@embedding.EmbeddingName" HeaderIcon="@Icons.Material.Filled.Info" IsExpanded="embeddingIndex == 0">
+                                    <TextInfoLine Icon="@Icons.Material.Filled.FormatShapes" Label="Type" Value="@embedding.EmbeddingType" ClipboardTooltipSubject="the embedding type"/>
+                                    <TextInfoLines Label="Description" MaxLines="14" Value="@embedding.Description" ClipboardTooltipSubject="the embedding description"/>
+                                    <TextInfoLines Label="When to use" MaxLines="3" Value="@embedding.UsedWhen" ClipboardTooltipSubject="when is the embedding used"/>
+
+                                    @if (!string.IsNullOrWhiteSpace(embedding.Link))
+                                    {
+                                        <MudButton Href="@embedding.Link" Target="_blank" Class="mt-3" Color="Color.Primary" StartIcon="@Icons.Material.Filled.OpenInNew">
+                                            Open web link, show more information
+                                        </MudButton>
+                                    }
+                                </ExpansionPanel>
+                            }
+                        </MudExpansionPanels>
+                    }
+                </ExpansionPanel>
             </MudExpansionPanels>
         }
 
diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor.cs
index 5f5a442..af17351 100644
--- a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor.cs	
+++ b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor.cs	
@@ -42,6 +42,7 @@ public partial class DataSourceERI_V1InfoDialog : ComponentBase, IAsyncDisposabl
     private string serverDescription = string.Empty;
     private ProviderType securityRequirements = ProviderType.NONE;
     private IReadOnlyList<RetrievalInfo> retrievalInfoformation = [];
+    private RetrievalInfo selectedRetrievalInfo;
     
     private bool IsOperationInProgress { get; set; } = true;
     
@@ -130,6 +131,7 @@ public partial class DataSourceERI_V1InfoDialog : ComponentBase, IAsyncDisposabl
             }
             
             this.retrievalInfoformation = retrievalInfoResult.Data ?? [];
+            this.selectedRetrievalInfo = this.retrievalInfoformation.FirstOrDefault(x => x.Id == this.DataSource.SelectedRetrievalId);
             this.StateHasChanged();
         }
         catch (Exception e)
diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor
index fcc9fff..a37a2d4 100644
--- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor	
+++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor	
@@ -109,11 +109,25 @@
                     InputType="InputType.Password"
                     UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
             }
+            
+            @if (this.availableRetrievalProcesses.Count > 0)
+            {
+                <MudSelect @bind-Value="@this.dataSelectedRetrievalProcess" Text="@this.dataSelectedRetrievalProcess.Name" Label="Select one retrieval process" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateRetrievalProcess">
+                    @foreach (var retrievalProcess in this.availableRetrievalProcesses)
+                    {
+                        <MudSelectItem Value="@retrievalProcess">
+                            @retrievalProcess.Name
+                        </MudSelectItem>
+                    }
+                </MudSelect>
+            }
 
             <MudSelect @bind-Value="@this.dataSecurityPolicy" Text="@this.dataSecurityPolicy.ToSelectionText()" Label="Your security policy" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateSecurityPolicy">
                 @foreach (var policy in Enum.GetValues<DataSourceSecurity>())
                 {
-                    <MudSelectItem Value="@policy">@policy.ToSelectionText()</MudSelectItem>
+                    <MudSelectItem Value="@policy">
+                        @policy.ToSelectionText()
+                    </MudSelectItem>
                 }
             </MudSelect>
             
diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs
index e042f92..dd0f9ce 100644
--- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs	
+++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs	
@@ -8,6 +8,8 @@ using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
 
+using RetrievalInfo = AIStudio.Tools.ERIClient.DataModel.RetrievalInfo;
+
 // ReSharper disable InconsistentNaming
 namespace AIStudio.Dialogs;
 
@@ -59,6 +61,8 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
     private int dataPort;
     private AuthMethod dataAuthMethod;
     private string dataUsername = string.Empty;
+    private List<RetrievalInfo> availableRetrievalProcesses = [];
+    private RetrievalInfo dataSelectedRetrievalProcess;
     
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -91,6 +95,9 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
         // When editing, we need to load the data:
         if(this.IsEditing)
         {
+            //
+            // Assign the data to the form fields:
+            //
             this.dataEditingPreviousInstanceName = this.DataSource.Name.ToLowerInvariant();
             this.dataNum = this.DataSource.Num;
             this.dataId = this.DataSource.Id;
@@ -114,6 +121,12 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
                     await this.form.Validate();
                 }
             }
+            
+            // Load the data:
+            await this.TestConnection();
+            
+            // Select the retrieval process:
+            this.dataSelectedRetrievalProcess = this.availableRetrievalProcesses.FirstOrDefault(n => n.Id == this.DataSource.SelectedRetrievalId);
         }
         
         await base.OnInitializedAsync();
@@ -153,6 +166,7 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
             Username = this.dataUsername,
             Type = DataSourceType.ERI_V1,
             SecurityPolicy = this.dataSecurityPolicy,
+            SelectedRetrievalId = this.dataSelectedRetrievalProcess.Id,
         };
     }
     
@@ -224,6 +238,18 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
             
             this.dataSourceSecurityRequirements = securityRequirementsRequest.Data;
             
+            var retrievalInfoRequest = await client.GetRetrievalInfoAsync(cts.Token);
+            if (!retrievalInfoRequest.Successful)
+            {
+                await this.form.Validate();
+                
+                Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1);
+                this.dataIssues[^1] = retrievalInfoRequest.Message;
+                return;
+            }
+            
+            this.availableRetrievalProcesses = retrievalInfoRequest.Data ?? [];
+            
             this.connectionTested = true;
             this.connectionSuccessfulTested = true;
             this.Logger.LogInformation("Connection to the ERI v1 server was successful tested.");
diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs
index d02c6bf..c755be8 100644
--- a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs	
+++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs	
@@ -51,6 +51,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
     /// <inheritdoc />
     public ERIVersion Version { get; init; } = ERIVersion.V1;
     
+    /// <inheritdoc />
+    public string SelectedRetrievalId { get; init; }
+    
     /// <inheritdoc />
     public async Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastPrompt, ChatThread thread, CancellationToken token = default)
     {
@@ -74,7 +77,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                 
                 Thread = await thread.ToERIChatThread(token),
                 MaxMatches = 10,
-                RetrievalProcessId = null, // The ERI server selects the retrieval process when multiple processes are available
+                RetrievalProcessId = string.IsNullOrWhiteSpace(this.SelectedRetrievalId) ? null : this.SelectedRetrievalId,
                 Parameters = null, // The ERI server selects useful default parameters
             };
             
diff --git a/app/MindWork AI Studio/Settings/IERIDataSource.cs b/app/MindWork AI Studio/Settings/IERIDataSource.cs
index 35e3779..5513897 100644
--- a/app/MindWork AI Studio/Settings/IERIDataSource.cs	
+++ b/app/MindWork AI Studio/Settings/IERIDataSource.cs	
@@ -29,4 +29,9 @@ public interface IERIDataSource : IExternalDataSource
     /// The ERI specification to use.
     /// </summary>
     public ERIVersion Version { get; init; }
+    
+    /// <summary>
+    /// The ID of the selected retrieval process.
+    /// </summary>
+    public string SelectedRetrievalId { get; init; }
 }
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs
index b3f8542..d97edb3 100644
--- a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs	
+++ b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs	
@@ -92,6 +92,14 @@ public sealed class DataSourceValidation
         
         return null;
     }
+
+    public string? ValidateRetrievalProcess(RetrievalInfo retrievalInfo)
+    {
+        if(retrievalInfo == default)
+            return "Please select one retrieval process.";
+        
+        return null;
+    }
     
     public string? ValidatingName(string dataSourceName)
     {
diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.9.32.md b/app/MindWork AI Studio/wwwroot/changelog/v0.9.32.md
index fa8328e..977ec39 100644
--- a/app/MindWork AI Studio/wwwroot/changelog/v0.9.32.md	
+++ b/app/MindWork AI Studio/wwwroot/changelog/v0.9.32.md	
@@ -1,5 +1,6 @@
 # v0.9.32, build 207 (2025-03-xx xx:xx UTC)
 - Added the "Community & Code" section to the about page. It includes links to the GitHub repositories and the project website.
+- Added the retrieval process section for ERI data sources in the configuration dialog. Like all RAG functions, this is hidden behind the RAG feature flag in the app settings due to its preview status.
 - Improved data security by preventing the use of cloud LLMs after confidential data has been retrieved previously.
 - Improved the ERI client to expect JSON responses and send JSON requests using camel case.
 - Improved the ERI client to raise an error when the server responds with additional JSON data that is not expected.