Added an information view to all data sources (#273)

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2025-02-09 12:36:37 +01:00
1 parent 9d71978feb
commit c836bd7f33
61 files changed
+1596 -1372

No files matched your search

@@ -0,0 +1,101 @@
@using AIStudio.Tools.ERIClient.DataModel
<MudDialog>
<DialogContent>
<MudText Typo="Typo.h5">
Common data source information
</MudText>
<TextInfoLine Icon="@Icons.Material.Filled.Tag" Label="Data source name" Value="@this.DataSource.Name" ClipboardTooltipSubject="the data source name"/>
<TextInfoLine Icon="@Icons.Material.Filled.NetworkCheck" Label="ERI server hostname" Value="@this.DataSource.Hostname" ClipboardTooltipSubject="the ERI server hostname"/>
<TextInfoLine Icon="@Icons.Material.Filled.Tag" Label="ERI server port" Value="@this.Port" ClipboardTooltipSubject="the ERI server port"/>
@if (!this.IsConnectionEncrypted())
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
Please note: the connection to the ERI v1 server is not encrypted. This means that all
data sent to the server is transmitted in plain text. Please ask the ERI server administrator
to enable encryption.
</MudJustifiedText>
}
@if (this.DataSource.AuthMethod is AuthMethod.USERNAME_PASSWORD)
{
<TextInfoLine Icon="@Icons.Material.Filled.Person2" Label="Username" Value="@this.DataSource.Username" ClipboardTooltipSubject="the username"/>
}
<TextInfoLines Label="Server description" MaxLines="14" Value="@this.serverDescription" ClipboardTooltipSubject="the server description"/>
<TextInfoLines Label="Security requirements" MaxLines="3" Value="@this.securityRequirements.Explain()" ClipboardTooltipSubject="the security requirements"/>
<MudText Typo="Typo.h5" Class="mt-6">
Retrieval information
</MudText>
@if (!this.retrievalInfoformation.Any())
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Info" Class="mb-3">
The data source does not provide any retrieval information.
</MudJustifiedText>
}
else
{
<MudExpansionPanels Class="mb-3">
@for (var index = 0; index < this.retrievalInfoformation.Count; index++)
{
var info = this.retrievalInfoformation[index];
<ExpansionPanel HeaderText="@this.RetrievalName(info)" HeaderIcon="@Icons.Material.Filled.Info" IsExpanded="index == 0">
<TextInfoLines Label="Description" MaxLines="14" Value="@info.Description" ClipboardTooltipSubject="the retrieval description"/>
<TextInfoLines Label="Parameters" MaxLines="14" Value="@this.RetrievalParameters(info)" ClipboardTooltipSubject="the retrieval parameters"/>
@if (!string.IsNullOrWhiteSpace(info.Link))
{
<MudButton Href="@info.Link" Target="_blank" Class="mt-3" Color="Color.Primary" StartIcon="@Icons.Material.Filled.OpenInNew">
Open web link, show more information
</MudButton>
}
<MudText Typo="Typo.h6" Class="mt-3">
Embeddings
</MudText>
@if (!info.Embeddings.Any())
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Info" Class="mb-3">
The data source does not provide any embedding information.
</MudJustifiedText>
}
else
{
<MudExpansionPanels>
@for (var embeddingIndex = 0; embeddingIndex < info.Embeddings.Count; embeddingIndex++)
{
var embedding = info.Embeddings[embeddingIndex];
<ExpansionPanel HeaderText="@embedding.EmbeddingName" HeaderIcon="@Icons.Material.Filled.Info" IsExpanded="embeddingIndex == 0">
<TextInfoLine Icon="@Icons.Material.Filled.FormatShapes" Label="Type" Value="@embedding.EmbeddingType" ClipboardTooltipSubject="the embedding type"/>
<TextInfoLines Label="Description" MaxLines="14" Value="@embedding.Description" ClipboardTooltipSubject="the embedding description"/>
<TextInfoLines Label="When to use" MaxLines="3" Value="@embedding.UsedWhen" ClipboardTooltipSubject="when is the embedding used"/>
@if (!string.IsNullOrWhiteSpace(embedding.Link))
{
<MudButton Href="@embedding.Link" Target="_blank" Class="mt-3" Color="Color.Primary" StartIcon="@Icons.Material.Filled.OpenInNew">
Open web link, show more information
</MudButton>
}
</ExpansionPanel>
}
</MudExpansionPanels>
}
</ExpansionPanel>
}
</MudExpansionPanels>
}
<Issues IssuesData="@this.dataIssues"/>
</DialogContent>
<DialogActions>
@if (this.IsOperationInProgress)
{
<MudProgressLinear Color="Color.Primary" Indeterminate="true" Class="ml-5 mr-5"/>
}
<MudButton OnClick="@this.GetERIMetadata" Variant="Variant.Filled" Color="Color.Info">Reload</MudButton>
<MudButton OnClick="@this.Close" Variant="Variant.Filled">Close</MudButton>
</DialogActions>
</MudDialog>
@@ -0,0 +1,178 @@
// ReSharper disable InconsistentNaming
using System.Text;
using AIStudio.Assistants.ERI;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient;
using AIStudio.Tools.ERIClient.DataModel;
using Microsoft.AspNetCore.Components;
using RetrievalInfo = AIStudio.Tools.ERIClient.DataModel.RetrievalInfo;
namespace AIStudio.Dialogs;
public partial class DataSourceERI_V1InfoDialog : ComponentBase, IAsyncDisposable, ISecretId
{
[CascadingParameter]
private MudDialogInstance MudDialog { get; set; } = null!;
[Parameter]
public DataSourceERI_V1 DataSource { get; set; }
[Inject]
private RustService RustService { get; init; } = null!;
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
this.eriServerTasks.Add(this.GetERIMetadata());
await base.OnInitializedAsync();
}
#endregion
private readonly CancellationTokenSource cts = new();
private readonly List<Task> eriServerTasks = new();
private readonly List<string> dataIssues = [];
private string serverDescription = string.Empty;
private ProviderType securityRequirements = ProviderType.NONE;
private IReadOnlyList<RetrievalInfo> retrievalInfoformation = [];
private bool IsOperationInProgress { get; set; } = true;
private bool IsConnectionEncrypted() => this.DataSource.Hostname.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase);
private string Port => this.DataSource.Port == 0 ? string.Empty : $"{this.DataSource.Port}";
private string RetrievalName(RetrievalInfo retrievalInfo)
{
var hasId = !string.IsNullOrWhiteSpace(retrievalInfo.Id);
var hasName = !string.IsNullOrWhiteSpace(retrievalInfo.Name);
if (hasId && hasName)
return $"[{retrievalInfo.Id}] {retrievalInfo.Name}";
if (hasId)
return $"[{retrievalInfo.Id}] Unnamed retrieval process";
return hasName ? retrievalInfo.Name : "Unnamed retrieval process";
}
private string RetrievalParameters(RetrievalInfo retrievalInfo)
{
var parameters = retrievalInfo.ParametersDescription;
if (parameters is null || parameters.Count == 0)
return "This retrieval process has no parameters.";
var sb = new StringBuilder();
foreach (var (paramName, description) in parameters)
{
sb.Append("Parameter: ");
sb.AppendLine(paramName);
sb.AppendLine(description);
sb.AppendLine();
}
return sb.ToString();
}
private async Task GetERIMetadata()
{
this.dataIssues.Clear();
try
{
this.IsOperationInProgress = true;
this.StateHasChanged();
using var client = ERIClientFactory.Get(ERIVersion.V1, this.DataSource);
if(client is null)
{
this.dataIssues.Add("Failed to connect to the ERI v1 server. The server is not supported.");
return;
}
var loginResult = await client.AuthenticateAsync(this.DataSource, this.RustService);
if (!loginResult.Successful)
{
this.dataIssues.Add(loginResult.Message);
return;
}
var dataSourceInfo = await client.GetDataSourceInfoAsync(this.cts.Token);
if (!dataSourceInfo.Successful)
{
this.dataIssues.Add(dataSourceInfo.Message);
return;
}
this.serverDescription = dataSourceInfo.Data.Description;
var securityRequirementsResult = await client.GetSecurityRequirementsAsync(this.cts.Token);
if (!securityRequirementsResult.Successful)
{
this.dataIssues.Add(securityRequirementsResult.Message);
return;
}
this.securityRequirements = securityRequirementsResult.Data.AllowedProviderType;
var retrievalInfoResult = await client.GetRetrievalInfoAsync(this.cts.Token);
if (!retrievalInfoResult.Successful)
{
this.dataIssues.Add(retrievalInfoResult.Message);
return;
}
this.retrievalInfoformation = retrievalInfoResult.Data ?? [];
this.StateHasChanged();
}
catch (Exception e)
{
this.dataIssues.Add($"Failed to connect to the ERI v1 server. The message was: {e.Message}");
}
finally
{
this.IsOperationInProgress = false;
this.StateHasChanged();
}
}
private void Close()
{
this.cts.Cancel();
this.MudDialog.Close();
}
#region Implementation of ISecretId
public string SecretId => this.DataSource.Id;
public string SecretName => this.DataSource.Name;
#endregion
#region Implementation of IDisposable
public async ValueTask DisposeAsync()
{
try
{
await this.cts.CancelAsync();
await Task.WhenAll(this.eriServerTasks);
this.cts.Dispose();
}
catch
{
// ignored
}
}
#endregion
}
@@ -1,4 +1,4 @@
@using ERI_Client.V1
@using AIStudio.Tools.ERIClient.DataModel
<MudDialog>
<DialogContent>
<MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues">
@@ -1,9 +1,10 @@
using AIStudio.Assistants.ERI;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient;
using AIStudio.Tools.ERIClient.DataModel;
using AIStudio.Tools.Validation;
using ERI_Client.V1;
using Microsoft.AspNetCore.Components;
// ReSharper disable InconsistentNaming
@@ -43,7 +44,6 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
private string[] dataIssues = [];
private string dataSecretStorageIssue = string.Empty;
private string dataEditingPreviousInstanceName = string.Empty;
private HttpClient? httpClient;
private List<AuthMethod> availableAuthMethods = [];
private bool connectionTested;
private bool connectionSuccessfulTested;
@@ -167,31 +167,38 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
{
try
{
this.httpClient = new HttpClient
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(14));
var dataSource = new DataSourceERI_V1
{
BaseAddress = new Uri($"{this.dataHostname}:{this.dataPort}"),
Timeout = TimeSpan.FromSeconds(5),
Hostname = this.dataHostname,
Port = this.dataPort
};
using (this.httpClient)
using var client = ERIClientFactory.Get(ERIVersion.V1, dataSource);
if(client is null)
{
var client = new Client(this.httpClient);
var authSchemes = await client.GetAuthMethodsAsync();
if (authSchemes is null)
{
await this.form.Validate();
Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1);
this.dataIssues[^1] = "Failed to connect to the ERI v1 server. The server did not respond.";
return;
}
this.availableAuthMethods = authSchemes.Select(n => n.AuthMethod).ToList();
this.connectionTested = true;
this.connectionSuccessfulTested = true;
this.Logger.LogInformation("Connection to the ERI v1 server was successful tested.");
await this.form.Validate();
Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1);
this.dataIssues[^1] = "Failed to connect to the ERI v1 server. The server is not supported.";
return;
}
var authSchemes = await client.GetAuthMethodsAsync(cts.Token);
if (!authSchemes.Successful)
{
await this.form.Validate();
Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1);
this.dataIssues[^1] = authSchemes.Message;
return;
}
this.availableAuthMethods = authSchemes.Data!.Select(n => n.AuthMethod).ToList();
this.connectionTested = true;
this.connectionSuccessfulTested = true;
this.Logger.LogInformation("Connection to the ERI v1 server was successful tested.");
}
catch (Exception e)
{
@@ -0,0 +1,52 @@
<MudDialog>
<DialogContent>
<TextInfoLine Icon="@Icons.Material.Filled.Tag" Label="Data source name" Value="@this.DataSource.Name" ClipboardTooltipSubject="the data source name"/>
<TextInfoLine Icon="@Icons.Material.Filled.FolderOpen" Label="Path" Value="@this.DataSource.Path" ClipboardTooltipSubject="this path"/>
@if (!this.IsDirectoryAvailable)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
The directory chosen for the data source does not exist anymore. Please edit the data source and correct the path.
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
The directory chosen for the data source exists.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.Layers" Label="Embedding name" Value="@this.embeddingProvider.Name" ClipboardTooltipSubject="the embedding name"/>
@if (this.IsCloudEmbedding)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
The embedding runs in the cloud. All your data from the folder '@this.DataSource.Path' and all its subdirectories
will be sent to the cloud.
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
The embedding runs locally or in your organization. Your data is not sent to the cloud.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="Number of files" Value="@this.NumberFilesInDirectory" ClipboardTooltipSubject="the number of files in the directory"/>
<TextInfoLines Label="Files list" MaxLines="14" Value="@this.directoryFiles.ToString()" ClipboardTooltipSubject="the files list"/>
@if (this.directorySizeNumFiles > 100)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Warning" Class="mb-3">
For performance reasons, only the first 100 files are shown. The directory contains @this.NumberFilesInDirectory files in total.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="Total directory size" Value="@this.directorySizeBytes.FileSize()" ClipboardTooltipSubject="the total directory size"/>
</DialogContent>
<DialogActions>
@if (this.IsOperationInProgress)
{
<MudProgressLinear Color="Color.Primary" Indeterminate="true" Class="ml-5 mr-5"/>
}
<MudButton OnClick="@this.Close" Variant="Variant.Filled">Close</MudButton>
</DialogActions>
</MudDialog>
@@ -0,0 +1,112 @@
using System.Text;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using Microsoft.AspNetCore.Components;
using Timer = System.Timers.Timer;
namespace AIStudio.Dialogs;
public partial class DataSourceLocalDirectoryInfoDialog : ComponentBase, IAsyncDisposable
{
[CascadingParameter]
private MudDialogInstance MudDialog { get; set; } = null!;
[Parameter]
public DataSourceLocalDirectory DataSource { get; set; }
[Inject]
private SettingsManager SettingsManager { get; init; } = null!;
private readonly Timer refreshTimer = new(TimeSpan.FromSeconds(1.6))
{
AutoReset = true,
};
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
this.embeddingProvider = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.DataSource.EmbeddingId);
this.directoryInfo = new DirectoryInfo(this.DataSource.Path);
if (this.directoryInfo.Exists)
{
this.directorySizeTask = this.directoryInfo.DetermineContentSize(this.UpdateDirectorySize, this.UpdateDirectoryFiles, this.UpdateFileList, MAX_FILES_TO_SHOW, this.DirectoryOperationDone, this.cts.Token);
this.refreshTimer.Elapsed += (_, _) => this.InvokeAsync(this.StateHasChanged);
this.refreshTimer.Start();
}
await base.OnInitializedAsync();
}
#endregion
private const int MAX_FILES_TO_SHOW = 100;
private readonly CancellationTokenSource cts = new();
private EmbeddingProvider embeddingProvider;
private DirectoryInfo directoryInfo = null!;
private long directorySizeBytes;
private long directorySizeNumFiles;
private readonly StringBuilder directoryFiles = new();
private Task directorySizeTask = Task.CompletedTask;
private bool IsOperationInProgress { get; set; } = true;
private bool IsCloudEmbedding => !this.embeddingProvider.IsSelfHosted;
private bool IsDirectoryAvailable => this.directoryInfo.Exists;
private void UpdateFileList(string file)
{
this.directoryFiles.Append("- ");
this.directoryFiles.AppendLine(file);
}
private void UpdateDirectorySize(long size)
{
this.directorySizeBytes = size;
}
private void UpdateDirectoryFiles(long numFiles) => this.directorySizeNumFiles = numFiles;
private void DirectoryOperationDone()
{
this.refreshTimer.Stop();
this.IsOperationInProgress = false;
this.InvokeAsync(this.StateHasChanged);
}
private string NumberFilesInDirectory => $"{this.directorySizeNumFiles:###,###,###,###}";
private void Close()
{
this.cts.Cancel();
this.MudDialog.Close();
}
#region Implementation of IDisposable
public async ValueTask DisposeAsync()
{
try
{
await this.cts.CancelAsync();
await this.directorySizeTask;
this.cts.Dispose();
this.refreshTimer.Stop();
this.refreshTimer.Dispose();
}
catch
{
// ignored
}
}
#endregion
}
@@ -23,7 +23,6 @@ public partial class DataSourceLocalFileDialog : ComponentBase
[Inject]
private SettingsManager SettingsManager { get; init; } = null!;
private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
private readonly DataSourceValidation dataSourceValidation;
@@ -0,0 +1,39 @@
<MudDialog>
<DialogContent>
<TextInfoLine Icon="@Icons.Material.Filled.Tag" Label="Data source name" Value="@this.DataSource.Name" ClipboardTooltipSubject="the data source name"/>
<TextInfoLine Icon="@Icons.Material.Filled.FolderOpen" Label="File path" Value="@this.DataSource.FilePath" ClipboardTooltipSubject="this path"/>
@if (!this.IsFileAvailable)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
The file chosen for the data source does not exist anymore. Please edit the data source and choose another file or correct the path.
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
The file chosen for the data source exists.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.Layers" Label="Embedding name" Value="@this.embeddingProvider.Name" ClipboardTooltipSubject="the embedding name"/>
@if (this.IsCloudEmbedding)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
The embedding runs in the cloud. All your data within the
file '@this.DataSource.FilePath' will be sent to the cloud.
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
The embedding runs locally or in your organization. Your data is not sent to the cloud.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="File size" Value="@this.FileSize" ClipboardTooltipSubject="the file size"/>
</DialogContent>
<DialogActions>
<MudButton OnClick="@this.Close" Variant="Variant.Filled">Close</MudButton>
</DialogActions>
</MudDialog>
@@ -0,0 +1,40 @@
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Dialogs;
public partial class DataSourceLocalFileInfoDialog : ComponentBase
{
[CascadingParameter]
private MudDialogInstance MudDialog { get; set; } = null!;
[Parameter]
public DataSourceLocalFile DataSource { get; set; }
[Inject]
private SettingsManager SettingsManager { get; init; } = null!;
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
this.embeddingProvider = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.DataSource.EmbeddingId);
this.fileInfo = new FileInfo(this.DataSource.FilePath);
await base.OnInitializedAsync();
}
#endregion
private EmbeddingProvider embeddingProvider;
private FileInfo fileInfo = null!;
private bool IsCloudEmbedding => !this.embeddingProvider.IsSelfHosted;
private bool IsFileAvailable => this.fileInfo.Exists;
private string FileSize => this.fileInfo.FileSize();
private void Close() => this.MudDialog.Close();
}