Added an information view to all data sources (#273)

This commit is contained in:
Thorsten Sommer 2025-02-09 12:36:37 +01:00 committed by GitHub
parent 9d71978feb
commit c836bd7f33
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
61 changed files with 1596 additions and 1372 deletions

View File

@ -12,7 +12,7 @@ Things we are currently working on:
- [x] ~~App: Add an option to show preview features (PR [#222](https://github.com/MindWorkAI/AI-Studio/pull/222))~~
- [x] ~~App: Configure embedding providers (PR [#224](https://github.com/MindWorkAI/AI-Studio/pull/224))~~
- [x] ~~App: Implement an [ERI](https://github.com/MindWorkAI/ERI) server coding assistant (PR [#231](https://github.com/MindWorkAI/AI-Studio/pull/231))~~
- [x] ~~App: Management of data sources (local & external data via [ERI](https://github.com/MindWorkAI/ERI)) (PR [#259](https://github.com/MindWorkAI/AI-Studio/pull/259))~~
- [x] ~~App: Management of data sources (local & external data via [ERI](https://github.com/MindWorkAI/ERI)) (PR [#259](https://github.com/MindWorkAI/AI-Studio/pull/259), [#273](https://github.com/MindWorkAI/AI-Studio/pull/273))~~
- [ ] Runtime: Extract data from txt / md / pdf / docx / xlsx files
- [ ] (*Optional*) Runtime: Implement internal embedding provider through [fastembed-rs](https://github.com/Anush008/fastembed-rs)
- [ ] App: Implement external embedding providers

File diff suppressed because it is too large Load Diff

View File

@ -1,10 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<LangVersion>latest</LangVersion>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

View File

@ -2,10 +2,6 @@
Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MindWork AI Studio", "MindWork AI Studio\MindWork AI Studio.csproj", "{059FDFCC-7D0B-474E-9F20-B9C437DF1CDD}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ERIClients", "ERIClients", "{5C2AF789-287B-4FCB-B675-7273D8CD4579}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ERIClientV1", "ERIClientV1\ERIClientV1.csproj", "{9E35A273-0FA6-4BD5-8880-A1DDAC106926}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -16,12 +12,7 @@ Global
{059FDFCC-7D0B-474E-9F20-B9C437DF1CDD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{059FDFCC-7D0B-474E-9F20-B9C437DF1CDD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{059FDFCC-7D0B-474E-9F20-B9C437DF1CDD}.Release|Any CPU.Build.0 = Release|Any CPU
{9E35A273-0FA6-4BD5-8880-A1DDAC106926}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9E35A273-0FA6-4BD5-8880-A1DDAC106926}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9E35A273-0FA6-4BD5-8880-A1DDAC106926}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9E35A273-0FA6-4BD5-8880-A1DDAC106926}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{9E35A273-0FA6-4BD5-8880-A1DDAC106926} = {5C2AF789-287B-4FCB-B675-7273D8CD4579}
EndGlobalSection
EndGlobal

View File

@ -7,7 +7,7 @@ public static class ERIVersionExtensions
try
{
var url = version.SpecificationURL();
var response = await httpClient.GetAsync(url);
using var response = await httpClient.GetAsync(url);
return await response.Content.ReadAsStringAsync();
}
catch

View File

@ -23,7 +23,7 @@ public partial class Changelog : ComponentBase
private async Task ReadLogAsync()
{
var response = await this.HttpClient.GetAsync($"changelog/{this.SelectedLog.Filename}");
using var response = await this.HttpClient.GetAsync($"changelog/{this.SelectedLog.Filename}");
this.LogContent = await response.Content.ReadAsStringAsync();
}
}

View File

@ -1,4 +1,3 @@
@using AIStudio.Settings
@using AIStudio.Settings.DataModel
@inherits SettingsPanelBase
@ -37,12 +36,7 @@
<MudTd>@this.GetEmbeddingName(context)</MudTd>
<MudTd Style="text-align: left;">
@if (context is IERIDataSource)
{
@* <MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Info" Class="ma-2" OnClick="() => this.ShowInformation(context)"> *@
@* Show Information *@
@* </MudButton> *@
}
<MudIconButton Variant="Variant.Filled" Color="Color.Info" Icon="@Icons.Material.Filled.Info" Class="ma-2" OnClick="() => this.ShowInformation(context)"/>
<MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Edit" Class="ma-2" OnClick="() => this.EditDataSource(context)">
Edit
</MudButton>

View File

@ -1,8 +1,7 @@
using AIStudio.Dialogs;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using ERI_Client.V1;
using AIStudio.Tools.ERIClient.DataModel;
using Microsoft.AspNetCore.Components;
@ -216,10 +215,37 @@ public partial class SettingsPanelDataSources : SettingsPanelBase
}
}
private Task ShowInformation(IDataSource dataSource)
private async Task ShowInformation(IDataSource dataSource)
{
#warning Implement the information dialog for ERI data sources.
return Task.CompletedTask;
switch (dataSource)
{
case DataSourceLocalFile localFile:
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileInfoDialog>
{
{ x => x.DataSource, localFile },
};
await this.DialogService.ShowAsync<DataSourceLocalFileInfoDialog>("Local File Data Source Information", localFileDialogParameters, DialogOptions.FULLSCREEN);
break;
case DataSourceLocalDirectory localDirectory:
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryInfoDialog>
{
{ x => x.DataSource, localDirectory },
};
await this.DialogService.ShowAsync<DataSourceLocalDirectoryInfoDialog>("Local Directory Data Source Information", localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
break;
case DataSourceERI_V1 eriV1DataSource:
var eriV1DialogParameters = new DialogParameters<DataSourceERI_V1InfoDialog>
{
{ x => x.DataSource, eriV1DataSource },
};
await this.DialogService.ShowAsync<DataSourceERI_V1InfoDialog>("ERI v1 Data Source Information", eriV1DialogParameters, DialogOptions.FULLSCREEN);
break;
}
}
private async Task UpdateDataSources()

View File

@ -0,0 +1,19 @@
<MudStack Row="@true" AlignItems="AlignItems.Center">
<MudTextField
T="string"
ReadOnly="@true"
Label="@this.Label"
Text="@this.Value"
Variant="Variant.Outlined"
Margin="Margin.Dense"
Adornment="Adornment.Start"
AdornmentIcon="@this.Icon"
UserAttributes="@USER_INPUT_ATTRIBUTES" />
@if (this.ShowingCopyButton)
{
<MudTooltip Text="@this.ClipboardTooltip">
<MudIconButton Icon="@Icons.Material.Filled.ContentCopy" Size="Size.Medium" OnClick="@(() => this.CopyToClipboard(this.Value))"/>
</MudTooltip>
}
</MudStack>

View File

@ -0,0 +1,50 @@
using AIStudio.Settings;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
public partial class TextInfoLine : ComponentBase
{
[Parameter]
public string Label { get; set; } = string.Empty;
[Parameter]
public string Icon { get; set; } = Icons.Material.Filled.Info;
[Parameter]
public string Value { get; set; } = string.Empty;
[Parameter]
public string ClipboardTooltipSubject { get; set; } = "the text";
[Parameter]
public bool ShowingCopyButton { get; set; } = true;
[Inject]
private RustService RustService { get; init; } = null!;
[Inject]
private ISnackbar Snackbar { get; init; } = null!;
[Inject]
private SettingsManager SettingsManager { get; init; } = null!;
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
// Configure the spellchecking for the user input:
this.SettingsManager.InjectSpellchecking(USER_INPUT_ATTRIBUTES);
await base.OnInitializedAsync();
}
#endregion
private static readonly Dictionary<string, object?> USER_INPUT_ATTRIBUTES = new();
private string ClipboardTooltip => $"Copy {this.ClipboardTooltipSubject} to the clipboard";
private async Task CopyToClipboard(string content) => await this.RustService.CopyText2Clipboard(this.Snackbar, content);
}

View File

@ -0,0 +1,20 @@
<MudStack Row="@true" AlignItems="AlignItems.Start">
<MudTextField
T="string"
ReadOnly="@true"
Label="@this.Label"
Text="@this.Value"
Variant="Variant.Outlined"
Margin="Margin.Dense"
Lines="3"
MaxLines="@this.MaxLines"
AutoGrow="@true"
UserAttributes="@USER_INPUT_ATTRIBUTES" />
@if (this.ShowingCopyButton)
{
<MudTooltip Text="@this.ClipboardTooltip">
<MudIconButton Icon="@Icons.Material.Filled.ContentCopy" Size="Size.Medium" OnClick="@(() => this.CopyToClipboard(this.Value))"/>
</MudTooltip>
}
</MudStack>

View File

@ -0,0 +1,50 @@
using AIStudio.Settings;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
public partial class TextInfoLines : ComponentBase
{
[Parameter]
public string Label { get; set; } = string.Empty;
[Parameter]
public string Value { get; set; } = string.Empty;
[Parameter]
public string ClipboardTooltipSubject { get; set; } = "the text";
[Parameter]
public int MaxLines { get; set; } = 30;
[Parameter]
public bool ShowingCopyButton { get; set; } = true;
[Inject]
private RustService RustService { get; init; } = null!;
[Inject]
private ISnackbar Snackbar { get; init; } = null!;
[Inject]
private SettingsManager SettingsManager { get; init; } = null!;
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
// Configure the spellchecking for the user input:
this.SettingsManager.InjectSpellchecking(USER_INPUT_ATTRIBUTES);
await base.OnInitializedAsync();
}
#endregion
private static readonly Dictionary<string, object?> USER_INPUT_ATTRIBUTES = new();
private string ClipboardTooltip => $"Copy {this.ClipboardTooltipSubject} to the clipboard";
private async Task CopyToClipboard(string content) => await this.RustService.CopyText2Clipboard(this.Snackbar, content);
}

View File

@ -0,0 +1,101 @@
@using AIStudio.Tools.ERIClient.DataModel
<MudDialog>
<DialogContent>
<MudText Typo="Typo.h5">
Common data source information
</MudText>
<TextInfoLine Icon="@Icons.Material.Filled.Tag" Label="Data source name" Value="@this.DataSource.Name" ClipboardTooltipSubject="the data source name"/>
<TextInfoLine Icon="@Icons.Material.Filled.NetworkCheck" Label="ERI server hostname" Value="@this.DataSource.Hostname" ClipboardTooltipSubject="the ERI server hostname"/>
<TextInfoLine Icon="@Icons.Material.Filled.Tag" Label="ERI server port" Value="@this.Port" ClipboardTooltipSubject="the ERI server port"/>
@if (!this.IsConnectionEncrypted())
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
Please note: the connection to the ERI v1 server is not encrypted. This means that all
data sent to the server is transmitted in plain text. Please ask the ERI server administrator
to enable encryption.
</MudJustifiedText>
}
@if (this.DataSource.AuthMethod is AuthMethod.USERNAME_PASSWORD)
{
<TextInfoLine Icon="@Icons.Material.Filled.Person2" Label="Username" Value="@this.DataSource.Username" ClipboardTooltipSubject="the username"/>
}
<TextInfoLines Label="Server description" MaxLines="14" Value="@this.serverDescription" ClipboardTooltipSubject="the server description"/>
<TextInfoLines Label="Security requirements" MaxLines="3" Value="@this.securityRequirements.Explain()" ClipboardTooltipSubject="the security requirements"/>
<MudText Typo="Typo.h5" Class="mt-6">
Retrieval information
</MudText>
@if (!this.retrievalInfoformation.Any())
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Info" Class="mb-3">
The data source does not provide any retrieval information.
</MudJustifiedText>
}
else
{
<MudExpansionPanels Class="mb-3">
@for (var index = 0; index < this.retrievalInfoformation.Count; index++)
{
var info = this.retrievalInfoformation[index];
<ExpansionPanel HeaderText="@this.RetrievalName(info)" HeaderIcon="@Icons.Material.Filled.Info" IsExpanded="index == 0">
<TextInfoLines Label="Description" MaxLines="14" Value="@info.Description" ClipboardTooltipSubject="the retrieval description"/>
<TextInfoLines Label="Parameters" MaxLines="14" Value="@this.RetrievalParameters(info)" ClipboardTooltipSubject="the retrieval parameters"/>
@if (!string.IsNullOrWhiteSpace(info.Link))
{
<MudButton Href="@info.Link" Target="_blank" Class="mt-3" Color="Color.Primary" StartIcon="@Icons.Material.Filled.OpenInNew">
Open web link, show more information
</MudButton>
}
<MudText Typo="Typo.h6" Class="mt-3">
Embeddings
</MudText>
@if (!info.Embeddings.Any())
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Info" Class="mb-3">
The data source does not provide any embedding information.
</MudJustifiedText>
}
else
{
<MudExpansionPanels>
@for (var embeddingIndex = 0; embeddingIndex < info.Embeddings.Count; embeddingIndex++)
{
var embedding = info.Embeddings[embeddingIndex];
<ExpansionPanel HeaderText="@embedding.EmbeddingName" HeaderIcon="@Icons.Material.Filled.Info" IsExpanded="embeddingIndex == 0">
<TextInfoLine Icon="@Icons.Material.Filled.FormatShapes" Label="Type" Value="@embedding.EmbeddingType" ClipboardTooltipSubject="the embedding type"/>
<TextInfoLines Label="Description" MaxLines="14" Value="@embedding.Description" ClipboardTooltipSubject="the embedding description"/>
<TextInfoLines Label="When to use" MaxLines="3" Value="@embedding.UsedWhen" ClipboardTooltipSubject="when is the embedding used"/>
@if (!string.IsNullOrWhiteSpace(embedding.Link))
{
<MudButton Href="@embedding.Link" Target="_blank" Class="mt-3" Color="Color.Primary" StartIcon="@Icons.Material.Filled.OpenInNew">
Open web link, show more information
</MudButton>
}
</ExpansionPanel>
}
</MudExpansionPanels>
}
</ExpansionPanel>
}
</MudExpansionPanels>
}
<Issues IssuesData="@this.dataIssues"/>
</DialogContent>
<DialogActions>
@if (this.IsOperationInProgress)
{
<MudProgressLinear Color="Color.Primary" Indeterminate="true" Class="ml-5 mr-5"/>
}
<MudButton OnClick="@this.GetERIMetadata" Variant="Variant.Filled" Color="Color.Info">Reload</MudButton>
<MudButton OnClick="@this.Close" Variant="Variant.Filled">Close</MudButton>
</DialogActions>
</MudDialog>

View File

@ -0,0 +1,178 @@
// ReSharper disable InconsistentNaming
using System.Text;
using AIStudio.Assistants.ERI;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient;
using AIStudio.Tools.ERIClient.DataModel;
using Microsoft.AspNetCore.Components;
using RetrievalInfo = AIStudio.Tools.ERIClient.DataModel.RetrievalInfo;
namespace AIStudio.Dialogs;
public partial class DataSourceERI_V1InfoDialog : ComponentBase, IAsyncDisposable, ISecretId
{
[CascadingParameter]
private MudDialogInstance MudDialog { get; set; } = null!;
[Parameter]
public DataSourceERI_V1 DataSource { get; set; }
[Inject]
private RustService RustService { get; init; } = null!;
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
this.eriServerTasks.Add(this.GetERIMetadata());
await base.OnInitializedAsync();
}
#endregion
private readonly CancellationTokenSource cts = new();
private readonly List<Task> eriServerTasks = new();
private readonly List<string> dataIssues = [];
private string serverDescription = string.Empty;
private ProviderType securityRequirements = ProviderType.NONE;
private IReadOnlyList<RetrievalInfo> retrievalInfoformation = [];
private bool IsOperationInProgress { get; set; } = true;
private bool IsConnectionEncrypted() => this.DataSource.Hostname.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase);
private string Port => this.DataSource.Port == 0 ? string.Empty : $"{this.DataSource.Port}";
private string RetrievalName(RetrievalInfo retrievalInfo)
{
var hasId = !string.IsNullOrWhiteSpace(retrievalInfo.Id);
var hasName = !string.IsNullOrWhiteSpace(retrievalInfo.Name);
if (hasId && hasName)
return $"[{retrievalInfo.Id}] {retrievalInfo.Name}";
if (hasId)
return $"[{retrievalInfo.Id}] Unnamed retrieval process";
return hasName ? retrievalInfo.Name : "Unnamed retrieval process";
}
private string RetrievalParameters(RetrievalInfo retrievalInfo)
{
var parameters = retrievalInfo.ParametersDescription;
if (parameters is null || parameters.Count == 0)
return "This retrieval process has no parameters.";
var sb = new StringBuilder();
foreach (var (paramName, description) in parameters)
{
sb.Append("Parameter: ");
sb.AppendLine(paramName);
sb.AppendLine(description);
sb.AppendLine();
}
return sb.ToString();
}
private async Task GetERIMetadata()
{
this.dataIssues.Clear();
try
{
this.IsOperationInProgress = true;
this.StateHasChanged();
using var client = ERIClientFactory.Get(ERIVersion.V1, this.DataSource);
if(client is null)
{
this.dataIssues.Add("Failed to connect to the ERI v1 server. The server is not supported.");
return;
}
var loginResult = await client.AuthenticateAsync(this.DataSource, this.RustService);
if (!loginResult.Successful)
{
this.dataIssues.Add(loginResult.Message);
return;
}
var dataSourceInfo = await client.GetDataSourceInfoAsync(this.cts.Token);
if (!dataSourceInfo.Successful)
{
this.dataIssues.Add(dataSourceInfo.Message);
return;
}
this.serverDescription = dataSourceInfo.Data.Description;
var securityRequirementsResult = await client.GetSecurityRequirementsAsync(this.cts.Token);
if (!securityRequirementsResult.Successful)
{
this.dataIssues.Add(securityRequirementsResult.Message);
return;
}
this.securityRequirements = securityRequirementsResult.Data.AllowedProviderType;
var retrievalInfoResult = await client.GetRetrievalInfoAsync(this.cts.Token);
if (!retrievalInfoResult.Successful)
{
this.dataIssues.Add(retrievalInfoResult.Message);
return;
}
this.retrievalInfoformation = retrievalInfoResult.Data ?? [];
this.StateHasChanged();
}
catch (Exception e)
{
this.dataIssues.Add($"Failed to connect to the ERI v1 server. The message was: {e.Message}");
}
finally
{
this.IsOperationInProgress = false;
this.StateHasChanged();
}
}
private void Close()
{
this.cts.Cancel();
this.MudDialog.Close();
}
#region Implementation of ISecretId
public string SecretId => this.DataSource.Id;
public string SecretName => this.DataSource.Name;
#endregion
#region Implementation of IDisposable
public async ValueTask DisposeAsync()
{
try
{
await this.cts.CancelAsync();
await Task.WhenAll(this.eriServerTasks);
this.cts.Dispose();
}
catch
{
// ignored
}
}
#endregion
}

View File

@ -1,4 +1,4 @@
@using ERI_Client.V1
@using AIStudio.Tools.ERIClient.DataModel
<MudDialog>
<DialogContent>
<MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues">

View File

@ -1,9 +1,10 @@
using AIStudio.Assistants.ERI;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient;
using AIStudio.Tools.ERIClient.DataModel;
using AIStudio.Tools.Validation;
using ERI_Client.V1;
using Microsoft.AspNetCore.Components;
// ReSharper disable InconsistentNaming
@ -43,7 +44,6 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
private string[] dataIssues = [];
private string dataSecretStorageIssue = string.Empty;
private string dataEditingPreviousInstanceName = string.Empty;
private HttpClient? httpClient;
private List<AuthMethod> availableAuthMethods = [];
private bool connectionTested;
private bool connectionSuccessfulTested;
@ -167,31 +167,38 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId
{
try
{
this.httpClient = new HttpClient
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(14));
var dataSource = new DataSourceERI_V1
{
BaseAddress = new Uri($"{this.dataHostname}:{this.dataPort}"),
Timeout = TimeSpan.FromSeconds(5),
Hostname = this.dataHostname,
Port = this.dataPort
};
using (this.httpClient)
using var client = ERIClientFactory.Get(ERIVersion.V1, dataSource);
if(client is null)
{
var client = new Client(this.httpClient);
var authSchemes = await client.GetAuthMethodsAsync();
if (authSchemes is null)
{
await this.form.Validate();
Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1);
this.dataIssues[^1] = "Failed to connect to the ERI v1 server. The server did not respond.";
return;
}
this.availableAuthMethods = authSchemes.Select(n => n.AuthMethod).ToList();
this.connectionTested = true;
this.connectionSuccessfulTested = true;
this.Logger.LogInformation("Connection to the ERI v1 server was successful tested.");
await this.form.Validate();
Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1);
this.dataIssues[^1] = "Failed to connect to the ERI v1 server. The server is not supported.";
return;
}
var authSchemes = await client.GetAuthMethodsAsync(cts.Token);
if (!authSchemes.Successful)
{
await this.form.Validate();
Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1);
this.dataIssues[^1] = authSchemes.Message;
return;
}
this.availableAuthMethods = authSchemes.Data!.Select(n => n.AuthMethod).ToList();
this.connectionTested = true;
this.connectionSuccessfulTested = true;
this.Logger.LogInformation("Connection to the ERI v1 server was successful tested.");
}
catch (Exception e)
{

View File

@ -0,0 +1,52 @@
<MudDialog>
<DialogContent>
<TextInfoLine Icon="@Icons.Material.Filled.Tag" Label="Data source name" Value="@this.DataSource.Name" ClipboardTooltipSubject="the data source name"/>
<TextInfoLine Icon="@Icons.Material.Filled.FolderOpen" Label="Path" Value="@this.DataSource.Path" ClipboardTooltipSubject="this path"/>
@if (!this.IsDirectoryAvailable)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
The directory chosen for the data source does not exist anymore. Please edit the data source and correct the path.
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
The directory chosen for the data source exists.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.Layers" Label="Embedding name" Value="@this.embeddingProvider.Name" ClipboardTooltipSubject="the embedding name"/>
@if (this.IsCloudEmbedding)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
The embedding runs in the cloud. All your data from the folder '@this.DataSource.Path' and all its subdirectories
will be sent to the cloud.
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
The embedding runs locally or in your organization. Your data is not sent to the cloud.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="Number of files" Value="@this.NumberFilesInDirectory" ClipboardTooltipSubject="the number of files in the directory"/>
<TextInfoLines Label="Files list" MaxLines="14" Value="@this.directoryFiles.ToString()" ClipboardTooltipSubject="the files list"/>
@if (this.directorySizeNumFiles > 100)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Warning" Class="mb-3">
For performance reasons, only the first 100 files are shown. The directory contains @this.NumberFilesInDirectory files in total.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="Total directory size" Value="@this.directorySizeBytes.FileSize()" ClipboardTooltipSubject="the total directory size"/>
</DialogContent>
<DialogActions>
@if (this.IsOperationInProgress)
{
<MudProgressLinear Color="Color.Primary" Indeterminate="true" Class="ml-5 mr-5"/>
}
<MudButton OnClick="@this.Close" Variant="Variant.Filled">Close</MudButton>
</DialogActions>
</MudDialog>

View File

@ -0,0 +1,112 @@
using System.Text;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using Microsoft.AspNetCore.Components;
using Timer = System.Timers.Timer;
namespace AIStudio.Dialogs;
public partial class DataSourceLocalDirectoryInfoDialog : ComponentBase, IAsyncDisposable
{
[CascadingParameter]
private MudDialogInstance MudDialog { get; set; } = null!;
[Parameter]
public DataSourceLocalDirectory DataSource { get; set; }
[Inject]
private SettingsManager SettingsManager { get; init; } = null!;
private readonly Timer refreshTimer = new(TimeSpan.FromSeconds(1.6))
{
AutoReset = true,
};
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
this.embeddingProvider = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.DataSource.EmbeddingId);
this.directoryInfo = new DirectoryInfo(this.DataSource.Path);
if (this.directoryInfo.Exists)
{
this.directorySizeTask = this.directoryInfo.DetermineContentSize(this.UpdateDirectorySize, this.UpdateDirectoryFiles, this.UpdateFileList, MAX_FILES_TO_SHOW, this.DirectoryOperationDone, this.cts.Token);
this.refreshTimer.Elapsed += (_, _) => this.InvokeAsync(this.StateHasChanged);
this.refreshTimer.Start();
}
await base.OnInitializedAsync();
}
#endregion
private const int MAX_FILES_TO_SHOW = 100;
private readonly CancellationTokenSource cts = new();
private EmbeddingProvider embeddingProvider;
private DirectoryInfo directoryInfo = null!;
private long directorySizeBytes;
private long directorySizeNumFiles;
private readonly StringBuilder directoryFiles = new();
private Task directorySizeTask = Task.CompletedTask;
private bool IsOperationInProgress { get; set; } = true;
private bool IsCloudEmbedding => !this.embeddingProvider.IsSelfHosted;
private bool IsDirectoryAvailable => this.directoryInfo.Exists;
private void UpdateFileList(string file)
{
this.directoryFiles.Append("- ");
this.directoryFiles.AppendLine(file);
}
private void UpdateDirectorySize(long size)
{
this.directorySizeBytes = size;
}
private void UpdateDirectoryFiles(long numFiles) => this.directorySizeNumFiles = numFiles;
private void DirectoryOperationDone()
{
this.refreshTimer.Stop();
this.IsOperationInProgress = false;
this.InvokeAsync(this.StateHasChanged);
}
private string NumberFilesInDirectory => $"{this.directorySizeNumFiles:###,###,###,###}";
private void Close()
{
this.cts.Cancel();
this.MudDialog.Close();
}
#region Implementation of IDisposable
public async ValueTask DisposeAsync()
{
try
{
await this.cts.CancelAsync();
await this.directorySizeTask;
this.cts.Dispose();
this.refreshTimer.Stop();
this.refreshTimer.Dispose();
}
catch
{
// ignored
}
}
#endregion
}

View File

@ -23,7 +23,6 @@ public partial class DataSourceLocalFileDialog : ComponentBase
[Inject]
private SettingsManager SettingsManager { get; init; } = null!;
private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
private readonly DataSourceValidation dataSourceValidation;

View File

@ -0,0 +1,39 @@
<MudDialog>
<DialogContent>
<TextInfoLine Icon="@Icons.Material.Filled.Tag" Label="Data source name" Value="@this.DataSource.Name" ClipboardTooltipSubject="the data source name"/>
<TextInfoLine Icon="@Icons.Material.Filled.FolderOpen" Label="File path" Value="@this.DataSource.FilePath" ClipboardTooltipSubject="this path"/>
@if (!this.IsFileAvailable)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
The file chosen for the data source does not exist anymore. Please edit the data source and choose another file or correct the path.
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
The file chosen for the data source exists.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.Layers" Label="Embedding name" Value="@this.embeddingProvider.Name" ClipboardTooltipSubject="the embedding name"/>
@if (this.IsCloudEmbedding)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
The embedding runs in the cloud. All your data within the
file '@this.DataSource.FilePath' will be sent to the cloud.
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
The embedding runs locally or in your organization. Your data is not sent to the cloud.
</MudJustifiedText>
}
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="File size" Value="@this.FileSize" ClipboardTooltipSubject="the file size"/>
</DialogContent>
<DialogActions>
<MudButton OnClick="@this.Close" Variant="Variant.Filled">Close</MudButton>
</DialogActions>
</MudDialog>

View File

@ -0,0 +1,40 @@
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Dialogs;
public partial class DataSourceLocalFileInfoDialog : ComponentBase
{
[CascadingParameter]
private MudDialogInstance MudDialog { get; set; } = null!;
[Parameter]
public DataSourceLocalFile DataSource { get; set; }
[Inject]
private SettingsManager SettingsManager { get; init; } = null!;
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
this.embeddingProvider = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.DataSource.EmbeddingId);
this.fileInfo = new FileInfo(this.DataSource.FilePath);
await base.OnInitializedAsync();
}
#endregion
private EmbeddingProvider embeddingProvider;
private FileInfo fileInfo = null!;
private bool IsCloudEmbedding => !this.embeddingProvider.IsSelfHosted;
private bool IsFileAvailable => this.fileInfo.Exists;
private string FileSize => this.fileInfo.FileSize();
private void Close() => this.MudDialog.Close();
}

View File

@ -53,10 +53,6 @@
<PackageReference Include="ReverseMarkdown" Version="4.6.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ERIClientV1\ERIClientV1.csproj" />
</ItemGroup>
<!-- Read the meta data file -->
<Target Name="ReadMetaData" BeforeTargets="BeforeBuild">
<Error Text="The ../../metadata.txt file was not found!" Condition="!Exists('../../metadata.txt')" />

View File

@ -26,7 +26,7 @@ public partial class Home : ComponentBase
private async Task ReadLastChangeAsync()
{
var latest = Changelog.LOGS.MaxBy(n => n.Build);
var response = await this.HttpClient.GetAsync($"changelog/{latest.Filename}");
using var response = await this.HttpClient.GetAsync($"changelog/{latest.Filename}");
this.LastChangeContent = await response.Content.ReadAsStringAsync();
}

View File

@ -103,9 +103,14 @@ public abstract class BaseProvider : IProvider, ISecretId
{
using var request = await requestBuilder();
//
// Send the request with the ResponseHeadersRead option.
// This allows us to read the stream as soon as the headers are received.
// This is important because we want to stream the responses.
//
// Please notice: We do not dispose the response here. The caller is responsible
// for disposing the response object. This is important because the response
// object is used to read the stream.
var nextResponse = await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
if (nextResponse.IsSuccessStatusCode)
{

View File

@ -136,8 +136,8 @@ public class ProviderGoogle(ILogger logger) : BaseProvider("https://generativela
if (secretKey is null)
return default;
var request = new HttpRequestMessage(HttpMethod.Get, $"models?key={secretKey}");
var response = await this.httpClient.SendAsync(request, token);
using var request = new HttpRequestMessage(HttpMethod.Get, $"models?key={secretKey}");
using var response = await this.httpClient.SendAsync(request, token);
if(!response.IsSuccessStatusCode)
return default;

View File

@ -127,10 +127,10 @@ public class ProviderGroq(ILogger logger) : BaseProvider("https://api.groq.com/o
if (secretKey is null)
return [];
var request = new HttpRequestMessage(HttpMethod.Get, "models");
using var request = new HttpRequestMessage(HttpMethod.Get, "models");
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", secretKey);
var response = await this.httpClient.SendAsync(request, token);
using var response = await this.httpClient.SendAsync(request, token);
if(!response.IsSuccessStatusCode)
return [];

View File

@ -138,10 +138,10 @@ public sealed class ProviderMistral(ILogger logger) : BaseProvider("https://api.
if (secretKey is null)
return default;
var request = new HttpRequestMessage(HttpMethod.Get, "models");
using var request = new HttpRequestMessage(HttpMethod.Get, "models");
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", secretKey);
var response = await this.httpClient.SendAsync(request, token);
using var response = await this.httpClient.SendAsync(request, token);
if(!response.IsSuccessStatusCode)
return default;

View File

@ -154,10 +154,10 @@ public sealed class ProviderOpenAI(ILogger logger) : BaseProvider("https://api.o
if (secretKey is null)
return [];
var request = new HttpRequestMessage(HttpMethod.Get, "models");
using var request = new HttpRequestMessage(HttpMethod.Get, "models");
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", secretKey);
var response = await this.httpClient.SendAsync(request, token);
using var response = await this.httpClient.SendAsync(request, token);
if(!response.IsSuccessStatusCode)
return [];

View File

@ -154,11 +154,11 @@ public sealed class ProviderSelfHosted(ILogger logger, Host host, string hostnam
}
};
var lmStudioRequest = new HttpRequestMessage(HttpMethod.Get, "models");
using var lmStudioRequest = new HttpRequestMessage(HttpMethod.Get, "models");
if(secretKey is not null)
lmStudioRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKeyProvisional);
var lmStudioResponse = await this.httpClient.SendAsync(lmStudioRequest, token);
using var lmStudioResponse = await this.httpClient.SendAsync(lmStudioRequest, token);
if(!lmStudioResponse.IsSuccessStatusCode)
return [];

View File

@ -127,10 +127,10 @@ public sealed class ProviderX(ILogger logger) : BaseProvider("https://api.x.ai/v
if (secretKey is null)
return [];
var request = new HttpRequestMessage(HttpMethod.Get, "models");
using var request = new HttpRequestMessage(HttpMethod.Get, "models");
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", secretKey);
var response = await this.httpClient.SendAsync(request, token);
using var response = await this.httpClient.SendAsync(request, token);
if(!response.IsSuccessStatusCode)
return [];

View File

@ -1,6 +1,7 @@
using ERI_Client.V1;
// ReSharper disable InconsistentNaming
using AIStudio.Tools.ERIClient.DataModel;
namespace AIStudio.Settings.DataModel;
/// <summary>
@ -24,23 +25,15 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
/// <inheritdoc />
public DataSourceType Type { get; init; } = DataSourceType.NONE;
/// <summary>
/// The hostname of the ERI server.
/// </summary>
/// <inheritdoc />
public string Hostname { get; init; } = string.Empty;
/// <summary>
/// The port of the ERI server.
/// </summary>
/// <inheritdoc />
public int Port { get; init; }
/// <summary>
/// The authentication method to use.
/// </summary>
/// <inheritdoc />
public AuthMethod AuthMethod { get; init; } = AuthMethod.NONE;
/// <summary>
/// The username to use for authentication, when the auth. method is USERNAME_PASSWORD.
/// </summary>
/// <inheritdoc />
public string Username { get; init; } = string.Empty;
}

View File

@ -1,12 +1,26 @@
using ERI_Client.V1;
using AIStudio.Tools.ERIClient.DataModel;
namespace AIStudio.Settings;
public interface IERIDataSource : IExternalDataSource
{
/// <summary>
/// The hostname of the ERI server.
/// </summary>
public string Hostname { get; init; }
/// <summary>
/// The port of the ERI server.
/// </summary>
public int Port { get; init; }
/// <summary>
/// The authentication method to use.
/// </summary>
public AuthMethod AuthMethod { get; init; }
/// <summary>
/// The username to use for authentication, when the auth. method is USERNAME_PASSWORD.
/// </summary>
public string Username { get; init; }
}

View File

@ -1,4 +1,4 @@
using ERI_Client.V1;
using AIStudio.Tools.ERIClient.DataModel;
namespace AIStudio.Tools;

View File

@ -0,0 +1,66 @@
namespace AIStudio.Tools;
public static class DirectoryInfoExtensions
{
private static readonly EnumerationOptions ENUMERATION_OPTIONS = new()
{
IgnoreInaccessible = true,
RecurseSubdirectories = true,
ReturnSpecialDirectories = false,
};
/// <summary>
/// Determines the size of the directory and all its subdirectories, as well as the number of files. When desired,
/// it can report the found files up to a certain limit.
/// </summary>
/// <remarks>
/// You might set reportMaxFiles to a negative value to report all files. Any positive value will limit the number
/// of reported files. The cancellation token can be used to stop the operation. The cancellation operation is also able
/// to cancel slow operations, e.g., when the directory is on a slow network drive.
///
/// After stopping the operation, the total size and number of files are reported as they were at the time of cancellation.
///
/// Please note that the entire operation is done on a background thread. Thus, when reporting the found files or the
/// current total size, you need to use the appropriate dispatcher to update the UI. Usually, you can use the InvokeAsync
/// method to update the UI from a background thread.
/// </remarks>
/// <param name="directoryInfo">The root directory to determine the size of.</param>
/// <param name="reportCurrentTotalSize">The callback to report the current total size of the directory.</param>
/// <param name="reportCurrentNumFiles">The callback to report the current number of files found.</param>
/// <param name="reportNextFile">The callback to report the next file found. The file name is relative to the root directory.</param>
/// <param name="reportMaxFiles">The maximum number of files to report. A negative value reports all files.</param>
/// <param name="done">The callback to report that the operation is done.</param>
/// <param name="cancellationToken">The cancellation token to stop the operation.</param>
public static async Task DetermineContentSize(this DirectoryInfo directoryInfo, Action<long> reportCurrentTotalSize, Action<long> reportCurrentNumFiles, Action<string> reportNextFile, int reportMaxFiles = -1, Action? done = null, CancellationToken cancellationToken = default)
{
var rootDirectoryLen = directoryInfo.FullName.Length;
long totalSize = 0;
long numFiles = 0;
await Task.Factory.StartNew(() => {
foreach (var file in directoryInfo.EnumerateFiles("*", ENUMERATION_OPTIONS))
{
if (cancellationToken.IsCancellationRequested)
return;
totalSize += file.Length;
numFiles++;
if (numFiles % 100 == 0)
{
reportCurrentTotalSize(totalSize);
reportCurrentNumFiles(numFiles);
}
if (reportMaxFiles < 0 || numFiles <= reportMaxFiles)
reportNextFile(file.FullName[rootDirectoryLen..]);
}
}, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default);
reportCurrentTotalSize(totalSize);
reportCurrentNumFiles(numFiles);
if(done is not null)
done();
}
}

View File

@ -0,0 +1,19 @@
namespace AIStudio.Tools.ERIClient;
public sealed class APIResponse<T>
{
/// <summary>
/// Was the API call successful?
/// </summary>
public bool Successful { get; set; }
/// <summary>
/// When the API call was not successful, this will contain the error message.
/// </summary>
public string Message { get; set; } = string.Empty;
/// <summary>
/// The data returned by the API call.
/// </summary>
public T? Data { get; set; }
}

View File

@ -0,0 +1,13 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// An authentication field.
/// </summary>
public enum AuthField
{
NONE,
USERNAME,
PASSWORD,
TOKEN,
KERBEROS_TICKET,
}

View File

@ -0,0 +1,8 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// The mapping between an AuthField and the field name in the authentication request.
/// </summary>
/// <param name="AuthField">The AuthField that is mapped to the field name.</param>
/// <param name="FieldName">The field name in the authentication request.</param>
public record AuthFieldMapping(AuthField AuthField, string FieldName);

View File

@ -0,0 +1,9 @@
namespace AIStudio.Tools.ERIClient.DataModel;
public enum AuthMethod
{
NONE,
KERBEROS,
USERNAME_PASSWORD,
TOKEN,
}

View File

@ -0,0 +1,9 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// The response to an authentication request.
/// </summary>
/// <param name="Success">True, when the authentication was successful.</param>
/// <param name="Token">The token to use for further requests.</param>
/// <param name="Message">When the authentication was not successful, this contains the reason.</param>
public readonly record struct AuthResponse(bool Success, string? Token, string? Message);

View File

@ -0,0 +1,9 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// Describes one authentication scheme for this data source.
/// </summary>
/// <param name="AuthMethod">The method used for authentication, e.g., "API Key," "Username/Password," etc.</param>
/// <param name="AuthFieldMappings">A list of field mappings for the authentication method. The client must know,
/// e.g., how the password field is named in the request.</param>
public readonly record struct AuthScheme(AuthMethod AuthMethod, List<AuthFieldMapping> AuthFieldMappings);

View File

@ -0,0 +1,7 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// A chat thread, which is a list of content blocks.
/// </summary>
/// <param name="ContentBlocks">The content blocks in this chat thread.</param>
public readonly record struct ChatThread(List<ContentBlock> ContentBlocks);

View File

@ -0,0 +1,12 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// A block of content of a chat thread.
/// </summary>
/// <remarks>
/// Images and other media are base64 encoded.
/// </remarks>
/// <param name="Content">The content of the block. Remember that images and other media are base64 encoded.</param>
/// <param name="Role">The role of the content in the chat thread.</param>
/// <param name="Type">The type of the content, e.g., text, image, video, etc.</param>
public readonly record struct ContentBlock(string Content, Role Role, ContentType Type);

View File

@ -0,0 +1,16 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// The type of content.
/// </summary>
public enum ContentType
{
NONE,
UNKNOWN,
TEXT,
IMAGE,
VIDEO,
AUDIO,
SPEECH,
}

View File

@ -0,0 +1,27 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// Matching context returned by the data source as a result of a retrieval request.
/// </summary>
/// <param name="Name">The name of the source, e.g., a document name, database name,
/// collection name, etc.</param>
/// <param name="Category">What are the contents of the source? For example, is it a
/// dictionary, a book chapter, business concept, a paper, etc.</param>
/// <param name="Path">The path to the content, e.g., a URL, a file path, a path in a
/// graph database, etc.</param>
/// <param name="Type">The type of the content, e.g., text, image, video, audio, speech, etc.</param>
/// <param name="MatchedContent">The content that matched the user prompt. For text, you
/// return the matched text and, e.g., three words before and after it.</param>
/// <param name="SurroundingContent">The surrounding content of the matched content.
/// For text, you may return, e.g., one sentence or paragraph before and after
/// the matched content.</param>
/// <param name="Links">Links to related content, e.g., links to Wikipedia articles,
/// links to sources, etc.</param>
public readonly record struct Context(
string Name,
string Category,
string? Path,
ContentType Type,
string MatchedContent,
string[] SurroundingContent,
string[] Links);

View File

@ -0,0 +1,9 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// Information about the data source.
/// </summary>
/// <param name="Name">The name of the data source, e.g., "Internal Organization Documents."</param>
/// <param name="Description">A short description of the data source. What kind of data does it contain?
/// What is the data source used for?</param>
public readonly record struct DataSourceInfo(string Name, string Description);

View File

@ -0,0 +1,20 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// Represents information about the used embedding for this data source. The purpose of this information is to give the
/// interested user an idea of what kind of embedding is used and what it does.
/// </summary>
/// <param name="EmbeddingType">What kind of embedding is used. For example, "Transformer Embedding," "Contextual Word
/// Embedding," "Graph Embedding," etc.</param>
/// <param name="EmbeddingName">Name the embedding used. This can be a library, a framework, or the name of the used
/// algorithm.</param>
/// <param name="Description">A short description of the embedding. Describe what the embedding is doing.</param>
/// <param name="UsedWhen">Describe when the embedding is used. For example, when the user prompt contains certain
/// keywords, or anytime?</param>
/// <param name="Link">A link to the embedding's documentation or the source code. Might be null.</param>
public readonly record struct EmbeddingInfo(
string EmbeddingType,
string EmbeddingName,
string Description,
string UsedWhen,
string? Link);

View File

@ -0,0 +1,22 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// Known types of providers that can process data.
/// </summary>
public enum ProviderType
{
/// <summary>
/// The related data is not allowed to be sent to any provider.
/// </summary>
NONE,
/// <summary>
/// The related data can be sent to any provider.
/// </summary>
ANY,
/// <summary>
/// The related data can be sent to a provider that is hosted by the same organization, either on-premises or locally.
/// </summary>
SELF_HOSTED,
}

View File

@ -0,0 +1,13 @@
namespace AIStudio.Tools.ERIClient.DataModel;
public static class ProviderTypeExtensions
{
public static string Explain(this ProviderType providerType) => providerType switch
{
ProviderType.NONE => "The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment.",
ProviderType.ANY => "The related data can be sent to any provider, regardless of where it is hosted (cloud or self-hosted).",
ProviderType.SELF_HOSTED => "The related data can be sent to a provider that is hosted by the same organization, either on-premises or locally. Cloud-based providers are not allowed.",
_ => "Unknown configuration. This data source cannot be used at the moment.",
};
}

View File

@ -0,0 +1,20 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// Information about a retrieval process, which this data source implements.
/// </summary>
/// <param name="Id">A unique identifier for the retrieval process. This can be a GUID, a unique name, or an increasing integer.</param>
/// <param name="Name">The name of the retrieval process, e.g., "Keyword-Based Wikipedia Article Retrieval".</param>
/// <param name="Description">A short description of the retrieval process. What kind of retrieval process is it?</param>
/// <param name="Link">A link to the retrieval process's documentation, paper, Wikipedia article, or the source code. Might be null.</param>
/// <param name="ParametersDescription">A dictionary that describes the parameters of the retrieval process. The key is the parameter name,
/// and the value is a description of the parameter. Although each parameter will be sent as a string, the description should indicate the
/// expected type and range, e.g., 0.0 to 1.0 for a float parameter.</param>
/// <param name="Embeddings">A list of embeddings used in this retrieval process. It might be empty in case no embedding is used.</param>
public readonly record struct RetrievalInfo(
string Id,
string Name,
string Description,
string? Link,
Dictionary<string, string>? ParametersDescription,
List<EmbeddingInfo> Embeddings);

View File

@ -0,0 +1,25 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// The retrieval request sent by AI Studio.
/// </summary>
/// <remarks>
/// Images and other media are base64 encoded.
/// </remarks>
/// <param name="LatestUserPrompt">The latest user prompt that AI Studio received.</param>
/// <param name="LatestUserPromptType">The type of the latest user prompt, e.g., text, image, etc.</param>
/// <param name="Thread">The chat thread that the user is currently in.</param>
/// <param name="RetrievalProcessId">Optional. The ID of the retrieval process that the data source should use.
/// When null, the data source chooses an appropriate retrieval process. Selecting a retrieval process is optional
/// for AI Studio users. Most users do not specify a retrieval process.</param>
/// <param name="Parameters">A dictionary of parameters that the data source should use for the retrieval process.
/// Although each parameter will be sent as a string, the retrieval process specifies the expected type and range.</param>
/// <param name="MaxMatches">The maximum number of matches that the data source should return. AI Studio uses
/// any value below 1 to indicate that the data source should return as many matches as appropriate.</param>
public readonly record struct RetrievalRequest(
string LatestUserPrompt,
ContentType LatestUserPromptType,
ChatThread Thread,
string? RetrievalProcessId,
Dictionary<string, string>? Parameters,
int MaxMatches);

View File

@ -0,0 +1,15 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// Possible roles of any chat thread.
/// </summary>
public enum Role
{
NONE,
UNKNOW,
SYSTEM,
USER,
AI,
AGENT,
}

View File

@ -0,0 +1,7 @@
namespace AIStudio.Tools.ERIClient.DataModel;
/// <summary>
/// Represents the security requirements for this data source.
/// </summary>
/// <param name="AllowedProviderType">Which provider types are allowed to process the data?</param>
public readonly record struct SecurityRequirements(ProviderType AllowedProviderType);

View File

@ -0,0 +1,36 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace AIStudio.Tools.ERIClient;
public abstract class ERIClientBase(string baseAddress) : IDisposable
{
protected static readonly JsonSerializerOptions JSON_OPTIONS = new()
{
WriteIndented = true,
AllowTrailingCommas = true,
PropertyNamingPolicy = null,
DictionaryKeyPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true,
Converters =
{
new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseUpper),
}
};
protected readonly HttpClient httpClient = new()
{
BaseAddress = new Uri(baseAddress),
};
protected string securityToken = string.Empty;
#region Implementation of IDisposable
public void Dispose()
{
this.httpClient.Dispose();
}
#endregion
}

View File

@ -0,0 +1,14 @@
using AIStudio.Assistants.ERI;
using AIStudio.Settings;
namespace AIStudio.Tools.ERIClient;
public static class ERIClientFactory
{
public static IERIClient? Get(ERIVersion version, IERIDataSource dataSource) => version switch
{
ERIVersion.V1 => new ERIClientV1($"{dataSource.Hostname}:{dataSource.Port}"),
_ => null
};
}

View File

@ -0,0 +1,344 @@
using System.Text;
using System.Text.Json;
using AIStudio.Settings;
using AIStudio.Tools.ERIClient.DataModel;
namespace AIStudio.Tools.ERIClient;
public class ERIClientV1(string baseAddress) : ERIClientBase(baseAddress), IERIClient
{
#region Implementation of IERIClient
public async Task<APIResponse<List<AuthScheme>>> GetAuthMethodsAsync(CancellationToken cancellationToken = default)
{
using var response = await this.httpClient.GetAsync("/auth/methods", cancellationToken);
if(!response.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to retrieve the authentication methods: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}"
};
}
var authMethods = await response.Content.ReadFromJsonAsync<List<AuthScheme>>(JSON_OPTIONS, cancellationToken);
if(authMethods is null)
{
return new()
{
Successful = false,
Message = "Failed to retrieve the authentication methods: the ERI server did not return a valid response."
};
}
return new()
{
Successful = true,
Data = authMethods
};
}
public async Task<APIResponse<AuthResponse>> AuthenticateAsync(IERIDataSource dataSource, RustService rustService, CancellationToken cancellationToken = default)
{
var authMethod = dataSource.AuthMethod;
var username = dataSource.Username;
switch (dataSource.AuthMethod)
{
case AuthMethod.NONE:
using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}"))
{
using var noneAuthResponse = await this.httpClient.SendAsync(request, cancellationToken);
if(!noneAuthResponse.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to authenticate with the ERI server. Code: {noneAuthResponse.StatusCode}, Reason: {noneAuthResponse.ReasonPhrase}"
};
}
var noneAuthResult = await noneAuthResponse.Content.ReadFromJsonAsync<AuthResponse>(JSON_OPTIONS, cancellationToken);
if(noneAuthResult == default)
{
return new()
{
Successful = false,
Message = "Failed to authenticate with the ERI server: the response was invalid."
};
}
this.securityToken = noneAuthResult.Token ?? string.Empty;
return new()
{
Successful = true,
Data = noneAuthResult
};
}
case AuthMethod.USERNAME_PASSWORD:
var passwordResponse = await rustService.GetSecret(dataSource);
if (!passwordResponse.Success)
{
return new()
{
Successful = false,
Message = "Failed to retrieve the password."
};
}
var password = await passwordResponse.Secret.Decrypt(Program.ENCRYPTION);
using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}"))
{
// We must send both values inside the header. The username field is named 'user'.
// The password field is named 'password'.
request.Headers.Add("user", username);
request.Headers.Add("password", password);
using var usernamePasswordAuthResponse = await this.httpClient.SendAsync(request, cancellationToken);
if(!usernamePasswordAuthResponse.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to authenticate with the ERI server. Code: {usernamePasswordAuthResponse.StatusCode}, Reason: {usernamePasswordAuthResponse.ReasonPhrase}"
};
}
var usernamePasswordAuthResult = await usernamePasswordAuthResponse.Content.ReadFromJsonAsync<AuthResponse>(JSON_OPTIONS, cancellationToken);
if(usernamePasswordAuthResult == default)
{
return new()
{
Successful = false,
Message = "Failed to authenticate with the server: the response was invalid."
};
}
this.securityToken = usernamePasswordAuthResult.Token ?? string.Empty;
return new()
{
Successful = true,
Data = usernamePasswordAuthResult
};
}
case AuthMethod.TOKEN:
var tokenResponse = await rustService.GetSecret(dataSource);
if (!tokenResponse.Success)
{
return new()
{
Successful = false,
Message = "Failed to retrieve the access token."
};
}
var token = await tokenResponse.Secret.Decrypt(Program.ENCRYPTION);
using (var request = new HttpRequestMessage(HttpMethod.Post, $"auth?authMethod={authMethod}"))
{
request.Headers.Add("Authorization", $"Bearer {token}");
using var tokenAuthResponse = await this.httpClient.SendAsync(request, cancellationToken);
if(!tokenAuthResponse.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to authenticate with the ERI server. Code: {tokenAuthResponse.StatusCode}, Reason: {tokenAuthResponse.ReasonPhrase}"
};
}
var tokenAuthResult = await tokenAuthResponse.Content.ReadFromJsonAsync<AuthResponse>(JSON_OPTIONS, cancellationToken);
if(tokenAuthResult == default)
{
return new()
{
Successful = false,
Message = "Failed to authenticate with the ERI server: the response was invalid."
};
}
this.securityToken = tokenAuthResult.Token ?? string.Empty;
return new()
{
Successful = true,
Data = tokenAuthResult
};
}
default:
this.securityToken = string.Empty;
return new()
{
Successful = false,
Message = "The authentication method is not supported yet."
};
}
}
public async Task<APIResponse<DataSourceInfo>> GetDataSourceInfoAsync(CancellationToken cancellationToken = default)
{
using var request = new HttpRequestMessage(HttpMethod.Get, "/dataSource");
request.Headers.Add("token", this.securityToken);
using var response = await this.httpClient.SendAsync(request, cancellationToken);
if(!response.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to retrieve the data source information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}"
};
}
var dataSourceInfo = await response.Content.ReadFromJsonAsync<DataSourceInfo>(JSON_OPTIONS, cancellationToken);
if(dataSourceInfo == default)
{
return new()
{
Successful = false,
Message = "Failed to retrieve the data source information: the ERI server did not return a valid response."
};
}
return new()
{
Successful = true,
Data = dataSourceInfo
};
}
public async Task<APIResponse<List<EmbeddingInfo>>> GetEmbeddingInfoAsync(CancellationToken cancellationToken = default)
{
using var request = new HttpRequestMessage(HttpMethod.Get, "/embedding/info");
request.Headers.Add("token", this.securityToken);
using var response = await this.httpClient.SendAsync(request, cancellationToken);
if(!response.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to retrieve the embedding information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}"
};
}
var embeddingInfo = await response.Content.ReadFromJsonAsync<List<EmbeddingInfo>>(JSON_OPTIONS, cancellationToken);
if(embeddingInfo is null)
{
return new()
{
Successful = false,
Message = "Failed to retrieve the embedding information: the ERI server did not return a valid response."
};
}
return new()
{
Successful = true,
Data = embeddingInfo
};
}
public async Task<APIResponse<List<RetrievalInfo>>> GetRetrievalInfoAsync(CancellationToken cancellationToken = default)
{
using var request = new HttpRequestMessage(HttpMethod.Get, "/retrieval/info");
request.Headers.Add("token", this.securityToken);
using var response = await this.httpClient.SendAsync(request, cancellationToken);
if(!response.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to retrieve the retrieval information: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}"
};
}
var retrievalInfo = await response.Content.ReadFromJsonAsync<List<RetrievalInfo>>(JSON_OPTIONS, cancellationToken);
if(retrievalInfo is null)
{
return new()
{
Successful = false,
Message = "Failed to retrieve the retrieval information: the ERI server did not return a valid response."
};
}
return new()
{
Successful = true,
Data = retrievalInfo
};
}
public async Task<APIResponse<List<Context>>> ExecuteRetrievalAsync(RetrievalRequest request, CancellationToken cancellationToken = default)
{
using var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/retrieval");
requestMessage.Headers.Add("token", this.securityToken);
using var content = new StringContent(JsonSerializer.Serialize(request, JSON_OPTIONS), Encoding.UTF8, "application/json");
requestMessage.Content = content;
using var response = await this.httpClient.SendAsync(requestMessage, cancellationToken);
if(!response.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to execute the retrieval request: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}"
};
}
var contexts = await response.Content.ReadFromJsonAsync<List<Context>>(JSON_OPTIONS, cancellationToken);
if(contexts is null)
{
return new()
{
Successful = false,
Message = "Failed to execute the retrieval request: the ERI server did not return a valid response."
};
}
return new()
{
Successful = true,
Data = contexts
};
}
public async Task<APIResponse<SecurityRequirements>> GetSecurityRequirementsAsync(CancellationToken cancellationToken = default)
{
using var request = new HttpRequestMessage(HttpMethod.Get, "/security/requirements");
request.Headers.Add("token", this.securityToken);
using var response = await this.httpClient.SendAsync(request, cancellationToken);
if(!response.IsSuccessStatusCode)
{
return new()
{
Successful = false,
Message = $"Failed to retrieve the security requirements: there was an issue communicating with the ERI server. Code: {response.StatusCode}, Reason: {response.ReasonPhrase}"
};
}
var securityRequirements = await response.Content.ReadFromJsonAsync<SecurityRequirements>(JSON_OPTIONS, cancellationToken);
if(securityRequirements == default)
{
return new()
{
Successful = false,
Message = "Failed to retrieve the security requirements: the ERI server did not return a valid response."
};
}
return new()
{
Successful = true,
Data = securityRequirements
};
}
#endregion
}

View File

@ -0,0 +1,62 @@
using AIStudio.Settings;
using AIStudio.Tools.ERIClient.DataModel;
namespace AIStudio.Tools.ERIClient;
public interface IERIClient : IDisposable
{
/// <summary>
/// Retrieves the available authentication methods from the ERI server.
/// </summary>
/// <remarks>
/// No authentication is required to retrieve the available authentication methods.
/// </remarks>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>The available authentication methods.</returns>
public Task<APIResponse<List<AuthScheme>>> GetAuthMethodsAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Authenticate the user to the ERI server.
/// </summary>
/// <param name="dataSource">The data source to use.</param>
/// <param name="rustService">The Rust service.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>The authentication response.</returns>
public Task<APIResponse<AuthResponse>> AuthenticateAsync(IERIDataSource dataSource, RustService rustService, CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves the data source information from the ERI server.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>The data source information.</returns>
public Task<APIResponse<DataSourceInfo>> GetDataSourceInfoAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves the embedding information from the ERI server.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A list of embedding information.</returns>
public Task<APIResponse<List<EmbeddingInfo>>> GetEmbeddingInfoAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves the retrieval information from the ERI server.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A list of retrieval information.</returns>
public Task<APIResponse<List<RetrievalInfo>>> GetRetrievalInfoAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Executes a retrieval request on the ERI server.
/// </summary>
/// <param name="request">The retrieval request.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>The retrieved contexts to use for augmentation and generation.</returns>
public Task<APIResponse<List<Context>>> ExecuteRetrievalAsync(RetrievalRequest request, CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves the security requirements from the ERI server.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>The security requirements.</returns>
public Task<APIResponse<SecurityRequirements>> GetSecurityRequirementsAsync(CancellationToken cancellationToken = default);
}

View File

@ -0,0 +1,17 @@
namespace AIStudio.Tools;
public static class FileInfoExtensions
{
/// <summary>
/// Returns the file size in human-readable format.
/// </summary>
/// <param name="fileInfo">The file info object.</param>
/// <returns>The file size in human-readable format.</returns>
public static string FileSize(this FileInfo fileInfo)
{
if (!fileInfo.Exists)
return "N/A";
return fileInfo.Length.FileSize();
}
}

View File

@ -0,0 +1,22 @@
namespace AIStudio.Tools;
public static class LongExtensions
{
/// <summary>
/// Formats the file size in a human-readable format.
/// </summary>
/// <param name="sizeBytes">The size in bytes.</param>
/// <returns>The formatted file size.</returns>
public static string FileSize(this long sizeBytes)
{
string[] sizes = { "B", "kB", "MB", "GB", "TB" };
var order = 0;
while (sizeBytes >= 1024 && order < sizes.Length - 1)
{
order++;
sizeBytes /= 1024;
}
return $"{sizeBytes:0.##} {sizes[order]}";
}
}

View File

@ -1,4 +1,4 @@
using ERI_Client.V1;
using AIStudio.Tools.ERIClient.DataModel;
namespace AIStudio.Tools.Validation;

View File

@ -210,6 +210,6 @@
"type": "Project"
}
},
"net8.0/osx-x64": {}
"net8.0/osx-arm64": {}
}
}

View File

@ -0,0 +1,4 @@
# v0.9.28, build 203 (2025-0x-xx xx:xx UTC)
- Added an information view to all data sources to the data source configuration page. The data source configuration is a preview feature behind the RAG feature flag.
- Added a ERI ((E)xternal (R)etrieval (I)nterface) client for communication with any ERI server.
- Improved the resource handling when loading models.